From b3032007fd41b3371cfd2dae23984389b8129da9 Mon Sep 17 00:00:00 2001 From: Ewout Kramer Date: Mon, 31 Jul 2023 17:29:48 +0200 Subject: [PATCH 1/3] Add cycle detection --- .../Terminology/ValueSetExpanderSettings.cs | 11 +- .../Terminology/ValueSetExpander.cs | 131 +++++++++--------- .../Source/TerminologyTests.cs | 39 +++++- 3 files changed, 109 insertions(+), 72 deletions(-) diff --git a/src/Hl7.Fhir.Base/Specification/Terminology/ValueSetExpanderSettings.cs b/src/Hl7.Fhir.Base/Specification/Terminology/ValueSetExpanderSettings.cs index 78ca235800..43d2f5f6e2 100644 --- a/src/Hl7.Fhir.Base/Specification/Terminology/ValueSetExpanderSettings.cs +++ b/src/Hl7.Fhir.Base/Specification/Terminology/ValueSetExpanderSettings.cs @@ -36,6 +36,12 @@ public class ValueSetExpanderSettings /// public bool IncludeDesignations { get; set; } + //// + //// Controls whether concepts are included that are marked as inactive. This setting overrides + //// ValueSet.compose.inactive. + //// + //public bool IncludeInactive { get; set; } + /// Default constructor. Creates a new instance with default property values. public ValueSetExpanderSettings() { } @@ -57,13 +63,14 @@ public void CopyTo(ValueSetExpanderSettings other) other.MaxExpansionSize = MaxExpansionSize; other.ValueSetSource = ValueSetSource; other.IncludeDesignations = IncludeDesignations; + // other.IncludeInactive = IncludeInactive; } /// Creates a new object that is a copy of the current instance. - public ValueSetExpanderSettings Clone() => new ValueSetExpanderSettings(this); + public ValueSetExpanderSettings Clone() => new(this); /// Creates a new instance with default property values. - public static ValueSetExpanderSettings CreateDefault() => new ValueSetExpanderSettings(); + public static ValueSetExpanderSettings CreateDefault() => new(); } } diff --git a/src/Hl7.Fhir.STU3/Specification/Terminology/ValueSetExpander.cs b/src/Hl7.Fhir.STU3/Specification/Terminology/ValueSetExpander.cs index 6387864739..155d898a25 100644 --- a/src/Hl7.Fhir.STU3/Specification/Terminology/ValueSetExpander.cs +++ b/src/Hl7.Fhir.STU3/Specification/Terminology/ValueSetExpander.cs @@ -1,4 +1,6 @@ -/* +#nullable enable + +/* * Copyright (c) 2016, Firely (info@fire.ly) and contributors * See the file CONTRIBUTORS for details. * @@ -12,23 +14,32 @@ using System; using System.Collections.Generic; using System.Linq; -using T=System.Threading.Tasks; +using T = System.Threading.Tasks; namespace Hl7.Fhir.Specification.Terminology { + /// + /// Expands valuesets by processing their include and exclude filters. Will create an in-place expansion. + /// public class ValueSetExpander { - -//ValueSetExpander keeps throwing TerminologyService Exceptions to not change the public interface. -#pragma warning disable 0618 - + /// + /// Settings to control the behaviour of the expansion. + /// public ValueSetExpanderSettings Settings { get; } + /// + /// Create a new expander with specific settings. + /// + /// public ValueSetExpander(ValueSetExpanderSettings settings) { Settings = settings; } + /// + /// Create a new expander with default settings + /// public ValueSetExpander() : this(ValueSetExpanderSettings.CreateDefault()) { // nothing @@ -37,8 +48,14 @@ public ValueSetExpander() : this(ValueSetExpanderSettings.CreateDefault()) [Obsolete("ValueSetExpander now works best with asynchronous resolvers. Use ExpandAsync() instead.")] public void Expand(ValueSet source) => TaskHelper.Await(() => ExpandAsync(source)); + /// + /// Expand the include and exclude filters. Creates the + /// + /// + /// + public T.Task ExpandAsync(ValueSet source) => ExpandAsync(source, new()); - public async T.Task ExpandAsync(ValueSet source) + internal async T.Task ExpandAsync(ValueSet source, Stack inclusionChain) { // Note we are expanding the valueset in-place, so it's up to the caller to decide whether // to clone the valueset, depending on store and performance requirements. @@ -47,7 +64,8 @@ public async T.Task ExpandAsync(ValueSet source) try { - await handleCompose(source).ConfigureAwait(false); + inclusionChain.Push(source.Url); + await handleCompose(source, inclusionChain).ConfigureAwait(false); } catch (Exception) { @@ -55,59 +73,40 @@ public async T.Task ExpandAsync(ValueSet source) source.Expansion = null; throw; } - + finally + { + inclusionChain.Pop(); + } } private void setExpansionParameters(ValueSet vs) { vs.Expansion.Parameter = new List(); - if(Settings.IncludeDesignations) + if (Settings.IncludeDesignations) { vs.Expansion.Parameter.Add(new ValueSet.ParameterComponent { Name = "includeDesignations", Value = new FhirBoolean(true) - }); + }); } //TODO add more parameters to the valuset here when we implement them. } - - //private int copyToExpansion(string system, string version, IEnumerable source, List dest) - //{ - // int added = 0; - - // foreach (var concept in source) - // { - // bool isDeprecated = concept.GetDeprecated() ?? false; - - // if (!isDeprecated) - // { - // var newContains = addToExpansion(system, version, concept.Code, concept.Display, concept.Abstract, dest); - // added += 1; - - // if (concept.Concept != null && concept.Concept.Any()) - // added += copyToExpansion(system, version, concept.Concept, newContains.Contains); - // } - // } - - // return added; - //} - - private async T.Task handleCompose(ValueSet source) + private async T.Task handleCompose(ValueSet source, Stack inclusionChain) { if (source.Compose == null) return; // handleImport(source); - await handleInclude(source).ConfigureAwait(false); - await handleExclude(source).ConfigureAwait(false); + await handleInclude(source, inclusionChain).ConfigureAwait(false); + await handleExclude(source, inclusionChain).ConfigureAwait(false); } - - private async T.Task> collectConcepts(ValueSet.ConceptSetComponent conceptSet) + private async T.Task> collectConcepts(ValueSet.ConceptSetComponent conceptSet, Stack inclusionChain) { - List result = new List(); + List result = new(); + // vsd-1 if (!conceptSet.ValueSet.Any() && conceptSet.System == null) throw Error.InvalidOperation($"Encountered a ConceptSet with neither a 'system' nor a 'valueset'"); @@ -122,7 +121,7 @@ private async T.Task handleCompose(ValueSet source) { // We'd probably really have to look this code up in the original ValueSet (by system) to know something about 'abstract' // and what would we do with a hierarchy if we encountered that in the include? - if(Settings.IncludeDesignations) + if (Settings.IncludeDesignations) { result.Add(conceptSet.System, conceptSet.Version, concept.Code, concept.Display, concept.Designation); } @@ -130,7 +129,7 @@ private async T.Task handleCompose(ValueSet source) { result.Add(conceptSet.System, conceptSet.Version, concept.Code, concept.Display); } - + } } else @@ -149,10 +148,10 @@ private async T.Task handleCompose(ValueSet source) throw new ValueSetExpansionTooComplexException($"ConceptSets with combined 'system' and 'valueset'(s) are not yet supported."); var importedVs = conceptSet.ValueSet.Single(); - var concepts = await getExpansionForValueSet(importedVs).ConfigureAwait(false); + var concepts = await getExpansionForValueSet(importedVs, inclusionChain).ConfigureAwait(false); import(result, concepts, importedVs); } - + return result; void import(List dest, List source, string importeeUrl) @@ -164,14 +163,14 @@ void import(List dest, List inclusionChain) { if (!source.Compose.Include.Any()) return; int csIndex = 0; foreach (var include in source.Compose.Include) { - var includedConcepts = await collectConcepts(include).ConfigureAwait(false); + var includedConcepts = await collectConcepts(include, inclusionChain).ConfigureAwait(false); // Yes, exclusion could make this smaller again, but alas, before we have processed those we might have run out of memory if (source.Expansion.Total + includedConcepts.Count > Settings.MaxExpansionSize) @@ -186,13 +185,13 @@ private async T.Task handleInclude(ValueSet source) } } - private async T.Task handleExclude(ValueSet source) + private async T.Task handleExclude(ValueSet source, Stack inclusionChain) { if (!source.Compose.Exclude.Any()) return; foreach (var exclude in source.Compose.Exclude) { - var excludedConcepts = await collectConcepts(exclude).ConfigureAwait(false); + var excludedConcepts = await collectConcepts(exclude, inclusionChain).ConfigureAwait(false); source.Expansion.Contains.Remove(excludedConcepts); @@ -201,22 +200,22 @@ private async T.Task handleExclude(ValueSet source) } } - - private async T.Task> getExpansionForValueSet(string uri) + private async T.Task> getExpansionForValueSet(string uri, Stack inclusionChain) { + if (inclusionChain.Contains(uri)) + throw new TerminologyServiceException($"ValueSet expansion encountered a cycling dependency from {inclusionChain.Peek()} back to {uri}."); + if (Settings.ValueSetSource == null) throw Error.InvalidOperation($"No valueset resolver available to resolve valueset '{uri}', " + "set ValueSetExpander.Settings.ValueSetSource to fix."); - var importedVs = await Settings.ValueSetSource.AsAsync().FindValueSetAsync(uri).ConfigureAwait(false); - if (importedVs == null) throw new ValueSetUnknownException($"Cannot resolve canonical reference '{uri}' to ValueSet"); - - if (!importedVs.HasExpansion) await ExpandAsync(importedVs).ConfigureAwait(false); + var importedVs = await Settings.ValueSetSource.AsAsync().FindValueSetAsync(uri).ConfigureAwait(false) + ?? throw new ValueSetUnknownException($"Cannot resolve canonical reference '{uri}' to ValueSet"); + if (!importedVs.HasExpansion) await ExpandAsync(importedVs, inclusionChain).ConfigureAwait(false); - if (importedVs.HasExpansion) - return importedVs.Expansion.Contains; - else - throw new ValueSetUnknownException($"Expansion returned neither an error, nor an expansion for ValueSet with canonical reference '{uri}'"); + return importedVs.HasExpansion + ? importedVs.Expansion.Contains + : throw new ValueSetUnknownException($"Expansion returned neither an error, nor an expansion for ValueSet with canonical reference '{uri}'"); } private async T.Task> getConceptsFromCodeSystem(string uri) @@ -225,9 +224,8 @@ private async T.Task handleExclude(ValueSet source) throw Error.InvalidOperation($"No terminology service available to resolve references to codesystem '{uri}', " + "set ValueSetExpander.Settings.ValueSetSource to fix."); - var importedCs = await Settings.ValueSetSource.AsAsync().FindCodeSystemAsync(uri).ConfigureAwait(false); - if (importedCs == null) throw new ValueSetUnknownException($"Cannot resolve canonical reference '{uri}' to CodeSystem"); - + var importedCs = await Settings.ValueSetSource.AsAsync().FindCodeSystemAsync(uri).ConfigureAwait(false) + ?? throw new ValueSetUnknownException($"Cannot resolve canonical reference '{uri}' to CodeSystem"); var result = new List(); result.AddRange(importedCs.Concept.Select(c => c.ToContainsComponent(importedCs, Settings))); @@ -238,7 +236,7 @@ private async T.Task handleExclude(ValueSet source) public static class ContainsSetExtensions { - public static ValueSet.ContainsComponent Add(this List dest, string system, string version, string code, string display, List designations = null, IEnumerable children = null) + public static ValueSet.ContainsComponent Add(this List dest, string system, string version, string code, string display, List? designations = null, IEnumerable? children = null) { var newContains = new ValueSet.ContainsComponent { @@ -294,9 +292,9 @@ internal static ValueSet.ContainsComponent ToContainsComponent(this CodeSystem.C newContains.System = system.Url; newContains.Version = system.Version; newContains.Code = source.Code; - newContains.Display = source.Display; - if(settings.IncludeDesignations) - newContains.Designation = source.Designation.ToValueSetDesignations(); + newContains.Display = source.Display; + if (settings.IncludeDesignations) + newContains.Designation = source.Designation.ToValueSetDesignations(); var abstractProperty = source.ListConceptProperties(system, CodeSystem.CONCEPTPROPERTY_NOT_SELECTABLE).SingleOrDefault(); if (abstractProperty?.Value is FhirBoolean isAbstract) @@ -316,7 +314,7 @@ internal static ValueSet.ContainsComponent ToContainsComponent(this CodeSystem.C private static List ToValueSetDesignations(this List csDesignations) { var vsDesignations = new List(); - csDesignations.ForEach(d => vsDesignations.Add(d.ToValueSetDesignation())); + csDesignations.ForEach(d => vsDesignations.Add(d.ToValueSetDesignation())); return vsDesignations; } @@ -331,5 +329,6 @@ private static ValueSet.DesignationComponent ToValueSetDesignation(this CodeSyst } } - #pragma warning restore } + +#nullable restore \ No newline at end of file diff --git a/src/Hl7.Fhir.Specification.STU3.Tests/Source/TerminologyTests.cs b/src/Hl7.Fhir.Specification.STU3.Tests/Source/TerminologyTests.cs index 6c8c07a955..862b088612 100644 --- a/src/Hl7.Fhir.Specification.STU3.Tests/Source/TerminologyTests.cs +++ b/src/Hl7.Fhir.Specification.STU3.Tests/Source/TerminologyTests.cs @@ -16,7 +16,7 @@ public class TerminologyTests { private readonly IAsyncResourceResolver _resolver = new CachedResolver(ZipSource.CreateValidationSource()); - private static Uri _externalTerminologyServerEndpoint = new("https://ontoserver.csiro.au/stu3-latest"); + private static readonly Uri _externalTerminologyServerEndpoint = new("https://ontoserver.csiro.au/stu3-latest"); // Use here a FhirPackageSource without the expansion package. private readonly IAsyncResourceResolver _resolverWithoutExpansions = new CachedResolver(ZipSource.CreateValidationSource()); @@ -60,6 +60,37 @@ public async T.Task ExpansionOfWholeSystem() //Assert.Equal("http://hl7.org/fhir/ValueSet/issue-type?version=3.14", ((FhirUri)versionParam.Value).Value); } + [Fact] + public async T.Task CatchesCyclicExpansions() + { + var resolver = new InMemoryResourceResolver(); + + var vs1 = new ValueSet() + { + Url = "http://nu.nl/refers-to-other", + Version = "2022-08-01", + Status = PublicationStatus.Active, + Compose = new() + { + Include = new() { new() { ValueSet = new[] { "http://nu.nl/other" } } } + } + }; + + var vs2 = new ValueSet() + { + Url = "http://nu.nl/other", + Version = "2022-08-01", + Status = PublicationStatus.Active, + Compose = new() + { + Include = new() { new() { ValueSet = new[] { vs1.Url } } } + } + }; + + resolver.Add(vs1, vs2); + var expander = new ValueSetExpander(new ValueSetExpanderSettings { ValueSetSource = resolver }); + await Assert.ThrowsAsync(() => expander.ExpandAsync(vs1)); + } [Fact] public async T.Task ExpansionOfComposeInclude() @@ -923,7 +954,7 @@ public async T.Task FallbackServiceValidateCodeTestWithVS() } #region helper functions - private async T.Task validateCodedValue(ITerminologyService service, string url = null, string context = null, string code = null, + private static async T.Task validateCodedValue(ITerminologyService service, string url = null, string context = null, string code = null, string system = null, string version = null, string display = null, Coding coding = null, CodeableConcept codeableConcept = null) { @@ -964,14 +995,14 @@ public async Task ResolveByCanonicalUriAsync(string uri) private class OnlyCodeSystemResolver : IAsyncResourceResolver, IConformanceSource { - private CodeSystem _onlyCs; + private readonly CodeSystem _onlyCs; public OnlyCodeSystemResolver(string csUrl) { _onlyCs = createCodeSystem(csUrl); } - private CodeSystem createCodeSystem(string csUrl) + private static CodeSystem createCodeSystem(string csUrl) { return new CodeSystem { From b2d25eb087f53f4a996fa14f0c1d40d4b5bd4933 Mon Sep 17 00:00:00 2001 From: Ewout Kramer Date: Mon, 31 Jul 2023 18:26:47 +0200 Subject: [PATCH 2/3] Add support for multiple compose.ValueSet and compose.System. --- .../Terminology/ValueSetExpander.cs | 335 ------------------ .../Terminology/ValueSetExpander.cs | 43 ++- 2 files changed, 28 insertions(+), 350 deletions(-) delete mode 100644 src/Hl7.Fhir.Conformance/Specification/Terminology/ValueSetExpander.cs rename src/{Hl7.Fhir.STU3 => Hl7.Fhir.Shims.STU3AndUp}/Specification/Terminology/ValueSetExpander.cs (88%) diff --git a/src/Hl7.Fhir.Conformance/Specification/Terminology/ValueSetExpander.cs b/src/Hl7.Fhir.Conformance/Specification/Terminology/ValueSetExpander.cs deleted file mode 100644 index b3a4e923ec..0000000000 --- a/src/Hl7.Fhir.Conformance/Specification/Terminology/ValueSetExpander.cs +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright (c) 2016, Firely (info@fire.ly) and contributors - * See the file CONTRIBUTORS for details. - * - * This file is licensed under the BSD 3-Clause license - * available at https://raw.githubusercontent.com/FirelyTeam/firely-net-sdk/master/LICENSE - */ - -using Hl7.Fhir.Model; -using Hl7.Fhir.Specification.Source; -using Hl7.Fhir.Utility; -using System; -using System.Collections.Generic; -using System.Linq; -using T = System.Threading.Tasks; - -namespace Hl7.Fhir.Specification.Terminology -{ - public class ValueSetExpander - { - - //ValueSetExpander keeps throwing TerminologyService Exceptions to not change the public interface. -#pragma warning disable 0618 - - public ValueSetExpanderSettings Settings { get; } - - public ValueSetExpander(ValueSetExpanderSettings settings) - { - Settings = settings; - } - - public ValueSetExpander() : this(ValueSetExpanderSettings.CreateDefault()) - { - // nothing - } - - [Obsolete("ValueSetExpander now works best with asynchronous resolvers. Use ExpandAsync() instead.")] - public void Expand(ValueSet source) => TaskHelper.Await(() => ExpandAsync(source)); - - - public async T.Task ExpandAsync(ValueSet source) - { - // Note we are expanding the valueset in-place, so it's up to the caller to decide whether - // to clone the valueset, depending on store and performance requirements. - source.Expansion = ValueSet.ExpansionComponent.Create(); - setExpansionParameters(source); - - try - { - await handleCompose(source).ConfigureAwait(false); - } - catch (Exception) - { - // Expansion failed - remove (partial) expansion - source.Expansion = null; - throw; - } - - } - - private void setExpansionParameters(ValueSet vs) - { - vs.Expansion.Parameter = new List(); - if (Settings.IncludeDesignations) - { - vs.Expansion.Parameter.Add(new ValueSet.ParameterComponent - { - Name = "includeDesignations", - Value = new FhirBoolean(true) - }); - } - //TODO add more parameters to the valuset here when we implement them. - } - - - //private int copyToExpansion(string system, string version, IEnumerable source, List dest) - //{ - // int added = 0; - - // foreach (var concept in source) - // { - // bool isDeprecated = concept.GetDeprecated() ?? false; - - // if (!isDeprecated) - // { - // var newContains = addToExpansion(system, version, concept.Code, concept.Display, concept.Abstract, dest); - // added += 1; - - // if (concept.Concept != null && concept.Concept.Any()) - // added += copyToExpansion(system, version, concept.Concept, newContains.Contains); - // } - // } - - // return added; - //} - - private async T.Task handleCompose(ValueSet source) - { - if (source.Compose == null) return; - - // handleImport(source); - await handleInclude(source).ConfigureAwait(false); - await handleExclude(source).ConfigureAwait(false); - } - - - private async T.Task> collectConcepts(ValueSet.ConceptSetComponent conceptSet) - { - List result = new List(); - - if (!conceptSet.ValueSet.Any() && conceptSet.System == null) - throw Error.InvalidOperation($"Encountered a ConceptSet with neither a 'system' nor a 'valueset'"); - - if (conceptSet.System != null) - { - if (conceptSet.Filter.Any()) - throw new ValueSetExpansionTooComplexException($"ConceptSets with a filter are not yet supported."); - - if (conceptSet.Concept.Any()) - { - foreach (var concept in conceptSet.Concept) - { - // We'd probably really have to look this code up in the original ValueSet (by system) to know something about 'abstract' - // and what would we do with a hierarchy if we encountered that in the include? - if (Settings.IncludeDesignations) - { - result.Add(conceptSet.System, conceptSet.Version, concept.Code, concept.Display, concept.Designation); - } - else - { - result.Add(conceptSet.System, conceptSet.Version, concept.Code, concept.Display); - } - - } - } - else - { - // Do a full import of the codesystem - var importedConcepts = await getConceptsFromCodeSystem(conceptSet.System).ConfigureAwait(false); - import(result, importedConcepts, conceptSet.System); - } - } - - if (conceptSet.ValueSet.Any()) - { - if (conceptSet.ValueSet.Count() > 1) - throw new ValueSetExpansionTooComplexException($"ConceptSets with multiple valuesets are not yet supported."); - if (conceptSet.System != null) - throw new ValueSetExpansionTooComplexException($"ConceptSets with combined 'system' and 'valueset'(s) are not yet supported."); - - var importedVs = conceptSet.ValueSet.Single(); - var concepts = await getExpansionForValueSet(importedVs).ConfigureAwait(false); - import(result, concepts, importedVs); - } - - return result; - - void import(List dest, List source, string importeeUrl) - { - if (dest.Count + source.Count > Settings.MaxExpansionSize) - throw new ValueSetExpansionTooBigException($"Import of '{importeeUrl}' ({source.Count} concepts) would be larger than the set maximum size ({Settings.MaxExpansionSize})"); - - dest.AddRange(source); - } - } - - private async T.Task handleInclude(ValueSet source) - { - if (!source.Compose.Include.Any()) return; - - int csIndex = 0; - foreach (var include in source.Compose.Include) - { - var includedConcepts = await collectConcepts(include).ConfigureAwait(false); - - // Yes, exclusion could make this smaller again, but alas, before we have processed those we might have run out of memory - if (source.Expansion.Total + includedConcepts.Count > Settings.MaxExpansionSize) - throw new ValueSetExpansionTooBigException($"Inclusion of {includedConcepts.Count} concepts from conceptset #{csIndex}' to " + - $"valueset '{source.Url}' ({source.Expansion.Total} concepts) would be larger than the set maximum size ({Settings.MaxExpansionSize})"); - - source.Expansion.Contains.AddRange(includedConcepts); - - var original = source.Expansion.Total ?? 0; - source.Expansion.Total = original + includedConcepts.CountConcepts(); - csIndex += 1; - } - } - - private async T.Task handleExclude(ValueSet source) - { - if (!source.Compose.Exclude.Any()) return; - - foreach (var exclude in source.Compose.Exclude) - { - var excludedConcepts = await collectConcepts(exclude).ConfigureAwait(false); - - source.Expansion.Contains.Remove(excludedConcepts); - - var original = source.Expansion.Total ?? 0; - source.Expansion.Total = original - excludedConcepts.CountConcepts(); - } - } - - - private async T.Task> getExpansionForValueSet(string uri) - { - if (Settings.ValueSetSource == null) - throw Error.InvalidOperation($"No valueset resolver available to resolve valueset '{uri}', " + - "set ValueSetExpander.Settings.ValueSetSource to fix."); - - var importedVs = await Settings.ValueSetSource.AsAsync().FindValueSetAsync(uri).ConfigureAwait(false); - if (importedVs == null) throw new ValueSetUnknownException($"The ValueSet expander cannot find system '{uri}', so the expansion cannot be completed."); - - if (!importedVs.HasExpansion) await ExpandAsync(importedVs).ConfigureAwait(false); - - if (importedVs.HasExpansion) - return importedVs.Expansion.Contains; - else - throw new ValueSetUnknownException($"Expansion returned neither an error, nor an expansion for ValueSet with canonical reference '{uri}'"); - } - - private async T.Task> getConceptsFromCodeSystem(string uri) - { - if (Settings.ValueSetSource == null) - throw Error.InvalidOperation($"No terminology service available to resolve references to codesystem '{uri}', " + - "set ValueSetExpander.Settings.ValueSetSource to fix."); - - var importedCs = await Settings.ValueSetSource.AsAsync().FindCodeSystemAsync(uri).ConfigureAwait(false); - if (importedCs == null) throw new ValueSetUnknownException($"The ValueSet expander cannot find system '{uri}', so the expansion cannot be completed."); - - var result = new List(); - result.AddRange(importedCs.Concept.Select(c => c.ToContainsComponent(importedCs, Settings))); - - return result; - } - } - - - public static class ContainsSetExtensions - { - public static ValueSet.ContainsComponent Add(this List dest, string system, string version, string code, string display, List designations = null, IEnumerable children = null) - { - var newContains = new ValueSet.ContainsComponent - { - System = system, - Code = code, - Display = display, - Version = version - }; - - newContains.System = system; - newContains.Code = code; - newContains.Display = display; - newContains.Version = version; - newContains.Designation = designations; - - if (children != null) - newContains.Contains = new List(children); - - dest.Add(newContains); - - return newContains; - } - - public static void Remove(this List dest, string system, string code) - { - dest.RemoveAll(c => c.System == system && c.Code == code); - - // Look for this code in children too - foreach (var component in dest) - { - if (component.Contains.Any()) - component.Contains.Remove(system, code); - } - } - - public static void Remove(this List dest, List source) - { - //TODO: Pretty unclear what to do with child concepts in the source - they all need to be removed from dest? - foreach (var sourceConcept in source) - dest.Remove(sourceConcept.System, sourceConcept.Code); - } - - internal static ValueSet.ContainsComponent ToContainsComponent(this CodeSystem.ConceptDefinitionComponent source, CodeSystem system, ValueSetExpanderSettings settings) - { - var newContains = new ValueSet.ContainsComponent - { - System = system.Url, - Version = system.Version, - Code = source.Code, - Display = source.Display - }; - - newContains.System = system.Url; - newContains.Version = system.Version; - newContains.Code = source.Code; - newContains.Display = source.Display; - if (settings.IncludeDesignations) - newContains.Designation = source.Designation.ToValueSetDesignations(); - - var abstractProperty = source.ListConceptProperties(system, CodeSystem.CONCEPTPROPERTY_NOT_SELECTABLE).SingleOrDefault(); - if (abstractProperty?.Value is FhirBoolean isAbstract) - newContains.Abstract = isAbstract.Value; - - var inactiveProperty = source.ListConceptProperties(system, CodeSystem.CONCEPTPROPERTY_STATUS).SingleOrDefault(); - if (inactiveProperty?.Value is FhirBoolean isInactive) - newContains.Inactive = isInactive.Value; - - if (source.Concept.Any()) - newContains.Contains.AddRange( - source.Concept.Select(c => c.ToContainsComponent(system, settings))); - - return newContains; - } - - private static List ToValueSetDesignations(this List csDesignations) - { - var vsDesignations = new List(); - csDesignations.ForEach(d => vsDesignations.Add(d.ToValueSetDesignation())); - return vsDesignations; - } - - private static ValueSet.DesignationComponent ToValueSetDesignation(this CodeSystem.DesignationComponent csDesignation) - { - return new ValueSet.DesignationComponent - { - Language = csDesignation.Language, - Use = csDesignation.Use, - Value = csDesignation.Value - }; - } - - } -#pragma warning restore -} diff --git a/src/Hl7.Fhir.STU3/Specification/Terminology/ValueSetExpander.cs b/src/Hl7.Fhir.Shims.STU3AndUp/Specification/Terminology/ValueSetExpander.cs similarity index 88% rename from src/Hl7.Fhir.STU3/Specification/Terminology/ValueSetExpander.cs rename to src/Hl7.Fhir.Shims.STU3AndUp/Specification/Terminology/ValueSetExpander.cs index b4fad7e060..0129c77bb9 100644 --- a/src/Hl7.Fhir.STU3/Specification/Terminology/ValueSetExpander.cs +++ b/src/Hl7.Fhir.Shims.STU3AndUp/Specification/Terminology/ValueSetExpander.cs @@ -102,6 +102,22 @@ private async T.Task handleCompose(ValueSet source, Stack inclusionChain await handleExclude(source, inclusionChain).ConfigureAwait(false); } + + private class SystemAndCodeComparer : IEqualityComparer + { + public bool Equals(ValueSet.ContainsComponent? x, ValueSet.ContainsComponent? y) + { + if (ReferenceEquals(x, y)) return true; + if (x is null || y is null) return false; + + return x.Code == y.Code && x.System == y.System; + } + + public int GetHashCode(ValueSet.ContainsComponent obj) => (obj.Code ?? "").GetHashCode() ^ (obj.System ?? "").GetHashCode(); + } + + private static readonly IEqualityComparer systemAndCodeComparer = new SystemAndCodeComparer(); + private async T.Task> collectConcepts(ValueSet.ConceptSetComponent conceptSet, Stack inclusionChain) { List result = new(); @@ -142,14 +158,14 @@ private async T.Task handleCompose(ValueSet source, Stack inclusionChain if (conceptSet.ValueSet.Any()) { - if (conceptSet.ValueSet.Count() > 1) - throw new ValueSetExpansionTooComplexException($"ConceptSets with multiple valuesets are not yet supported."); - if (conceptSet.System != null) - throw new ValueSetExpansionTooComplexException($"ConceptSets with combined 'system' and 'valueset'(s) are not yet supported."); - - var importedVs = conceptSet.ValueSet.Single(); - var concepts = await getExpansionForValueSet(importedVs, inclusionChain).ConfigureAwait(false); - import(result, concepts, importedVs); + var expanded = await T.Task.WhenAll(conceptSet.ValueSet.Select(vs => getExpansionForValueSet(vs, inclusionChain))).ConfigureAwait(false); + var concepts = expanded.SelectMany(concept => concept); + + if (conceptSet.System is not null) + concepts = concepts.Where(c => c.System == conceptSet.System); + + concepts = concepts.Distinct(systemAndCodeComparer); + import(result, concepts.ToList(), string.Join(",", conceptSet.ValueSet)); } return result; @@ -210,7 +226,7 @@ private async T.Task handleExclude(ValueSet source, Stack inclusionChain "set ValueSetExpander.Settings.ValueSetSource to fix."); var importedVs = await Settings.ValueSetSource.AsAsync().FindValueSetAsync(uri).ConfigureAwait(false) - ?? throw new ValueSetUnknownException($"Cannot resolve canonical reference '{uri}' to ValueSet"); + ?? throw new ValueSetUnknownException($"The ValueSet expander cannot find system '{uri}', so the expansion cannot be completed."); if (!importedVs.HasExpansion) await ExpandAsync(importedVs, inclusionChain).ConfigureAwait(false); return importedVs.HasExpansion @@ -224,13 +240,10 @@ private async T.Task handleExclude(ValueSet source, Stack inclusionChain throw Error.InvalidOperation($"No terminology service available to resolve references to codesystem '{uri}', " + "set ValueSetExpander.Settings.ValueSetSource to fix."); - var importedCs = await Settings.ValueSetSource.AsAsync().FindCodeSystemAsync(uri).ConfigureAwait(false); - if (importedCs == null) throw new ValueSetUnknownException($"The ValueSet expander cannot find system '{uri}', so the expansion cannot be completed."); - - var result = new List(); - result.AddRange(importedCs.Concept.Select(c => c.ToContainsComponent(importedCs, Settings))); + var importedCs = await Settings.ValueSetSource.AsAsync().FindCodeSystemAsync(uri).ConfigureAwait(false) + ?? throw new ValueSetUnknownException($"The ValueSet expander cannot find system '{uri}', so the expansion cannot be completed."); - return result; + return importedCs.Concept.Select(c => c.ToContainsComponent(importedCs, Settings)).ToList(); } } From 9e93e69abde6ea99144c8810165b0df73e5fefbd Mon Sep 17 00:00:00 2001 From: Ewout Kramer Date: Tue, 1 Aug 2023 17:35:52 +0200 Subject: [PATCH 3/3] Reworked the expander to more closely follow the expansion algorithm specified in the spec. - Using more delayed processing, so processing valuesets that are too big is actually stopped before attempting it. - Documented all steps more clearly, referring to the spec. - specifying both System and ValueSet will now do an intersection. --- .../Terminology/ValueSetExpander.cs | 348 ++++++++++++++++ .../Terminology/ValueSetExpander.cs | 387 ++++++++++++++++++ .../Source/TerminologyTests.cs | 104 ++++- 3 files changed, 838 insertions(+), 1 deletion(-) create mode 100644 src/Hl7.Fhir.Conformance/Specification/Terminology/ValueSetExpander.cs create mode 100644 src/Hl7.Fhir.STU3/Specification/Terminology/ValueSetExpander.cs diff --git a/src/Hl7.Fhir.Conformance/Specification/Terminology/ValueSetExpander.cs b/src/Hl7.Fhir.Conformance/Specification/Terminology/ValueSetExpander.cs new file mode 100644 index 0000000000..0129c77bb9 --- /dev/null +++ b/src/Hl7.Fhir.Conformance/Specification/Terminology/ValueSetExpander.cs @@ -0,0 +1,348 @@ +#nullable enable + +/* + * Copyright (c) 2016, Firely (info@fire.ly) and contributors + * See the file CONTRIBUTORS for details. + * + * This file is licensed under the BSD 3-Clause license + * available at https://raw.githubusercontent.com/FirelyTeam/firely-net-sdk/master/LICENSE + */ + +using Hl7.Fhir.Model; +using Hl7.Fhir.Specification.Source; +using Hl7.Fhir.Utility; +using System; +using System.Collections.Generic; +using System.Linq; +using T = System.Threading.Tasks; + +namespace Hl7.Fhir.Specification.Terminology +{ + /// + /// Expands valuesets by processing their include and exclude filters. Will create an in-place expansion. + /// + public class ValueSetExpander + { + /// + /// Settings to control the behaviour of the expansion. + /// + public ValueSetExpanderSettings Settings { get; } + + /// + /// Create a new expander with specific settings. + /// + /// + public ValueSetExpander(ValueSetExpanderSettings settings) + { + Settings = settings; + } + + /// + /// Create a new expander with default settings + /// + public ValueSetExpander() : this(ValueSetExpanderSettings.CreateDefault()) + { + // nothing + } + + [Obsolete("ValueSetExpander now works best with asynchronous resolvers. Use ExpandAsync() instead.")] + public void Expand(ValueSet source) => TaskHelper.Await(() => ExpandAsync(source)); + + /// + /// Expand the include and exclude filters. Creates the + /// + /// + /// + public T.Task ExpandAsync(ValueSet source) => ExpandAsync(source, new()); + + internal async T.Task ExpandAsync(ValueSet source, Stack inclusionChain) + { + // Note we are expanding the valueset in-place, so it's up to the caller to decide whether + // to clone the valueset, depending on store and performance requirements. + source.Expansion = ValueSet.ExpansionComponent.Create(); + setExpansionParameters(source); + + try + { + inclusionChain.Push(source.Url); + await handleCompose(source, inclusionChain).ConfigureAwait(false); + } + catch (Exception) + { + // Expansion failed - remove (partial) expansion + source.Expansion = null; + throw; + } + finally + { + inclusionChain.Pop(); + } + } + + private void setExpansionParameters(ValueSet vs) + { + vs.Expansion.Parameter = new List(); + if (Settings.IncludeDesignations) + { + vs.Expansion.Parameter.Add(new ValueSet.ParameterComponent + { + Name = "includeDesignations", + Value = new FhirBoolean(true) + }); + } + //TODO add more parameters to the valuset here when we implement them. + } + + private async T.Task handleCompose(ValueSet source, Stack inclusionChain) + { + if (source.Compose == null) return; + + // handleImport(source); + await handleInclude(source, inclusionChain).ConfigureAwait(false); + await handleExclude(source, inclusionChain).ConfigureAwait(false); + } + + + private class SystemAndCodeComparer : IEqualityComparer + { + public bool Equals(ValueSet.ContainsComponent? x, ValueSet.ContainsComponent? y) + { + if (ReferenceEquals(x, y)) return true; + if (x is null || y is null) return false; + + return x.Code == y.Code && x.System == y.System; + } + + public int GetHashCode(ValueSet.ContainsComponent obj) => (obj.Code ?? "").GetHashCode() ^ (obj.System ?? "").GetHashCode(); + } + + private static readonly IEqualityComparer systemAndCodeComparer = new SystemAndCodeComparer(); + + private async T.Task> collectConcepts(ValueSet.ConceptSetComponent conceptSet, Stack inclusionChain) + { + List result = new(); + + // vsd-1 + if (!conceptSet.ValueSet.Any() && conceptSet.System == null) + throw Error.InvalidOperation($"Encountered a ConceptSet with neither a 'system' nor a 'valueset'"); + + if (conceptSet.System != null) + { + if (conceptSet.Filter.Any()) + throw new ValueSetExpansionTooComplexException($"ConceptSets with a filter are not yet supported."); + + if (conceptSet.Concept.Any()) + { + foreach (var concept in conceptSet.Concept) + { + // We'd probably really have to look this code up in the original ValueSet (by system) to know something about 'abstract' + // and what would we do with a hierarchy if we encountered that in the include? + if (Settings.IncludeDesignations) + { + result.Add(conceptSet.System, conceptSet.Version, concept.Code, concept.Display, concept.Designation); + } + else + { + result.Add(conceptSet.System, conceptSet.Version, concept.Code, concept.Display); + } + + } + } + else + { + // Do a full import of the codesystem + var importedConcepts = await getConceptsFromCodeSystem(conceptSet.System).ConfigureAwait(false); + import(result, importedConcepts, conceptSet.System); + } + } + + if (conceptSet.ValueSet.Any()) + { + var expanded = await T.Task.WhenAll(conceptSet.ValueSet.Select(vs => getExpansionForValueSet(vs, inclusionChain))).ConfigureAwait(false); + var concepts = expanded.SelectMany(concept => concept); + + if (conceptSet.System is not null) + concepts = concepts.Where(c => c.System == conceptSet.System); + + concepts = concepts.Distinct(systemAndCodeComparer); + import(result, concepts.ToList(), string.Join(",", conceptSet.ValueSet)); + } + + return result; + + void import(List dest, List source, string importeeUrl) + { + if (dest.Count + source.Count > Settings.MaxExpansionSize) + throw new ValueSetExpansionTooBigException($"Import of '{importeeUrl}' ({source.Count} concepts) would be larger than the set maximum size ({Settings.MaxExpansionSize})"); + + dest.AddRange(source); + } + } + + private async T.Task handleInclude(ValueSet source, Stack inclusionChain) + { + if (!source.Compose.Include.Any()) return; + + int csIndex = 0; + foreach (var include in source.Compose.Include) + { + var includedConcepts = await collectConcepts(include, inclusionChain).ConfigureAwait(false); + + // Yes, exclusion could make this smaller again, but alas, before we have processed those we might have run out of memory + if (source.Expansion.Total + includedConcepts.Count > Settings.MaxExpansionSize) + throw new ValueSetExpansionTooBigException($"Inclusion of {includedConcepts.Count} concepts from conceptset #{csIndex}' to " + + $"valueset '{source.Url}' ({source.Expansion.Total} concepts) would be larger than the set maximum size ({Settings.MaxExpansionSize})"); + + source.Expansion.Contains.AddRange(includedConcepts); + + var original = source.Expansion.Total ?? 0; + source.Expansion.Total = original + includedConcepts.CountConcepts(); + csIndex += 1; + } + } + + private async T.Task handleExclude(ValueSet source, Stack inclusionChain) + { + if (!source.Compose.Exclude.Any()) return; + + foreach (var exclude in source.Compose.Exclude) + { + var excludedConcepts = await collectConcepts(exclude, inclusionChain).ConfigureAwait(false); + + source.Expansion.Contains.Remove(excludedConcepts); + + var original = source.Expansion.Total ?? 0; + source.Expansion.Total = original - excludedConcepts.CountConcepts(); + } + } + + private async T.Task> getExpansionForValueSet(string uri, Stack inclusionChain) + { + if (inclusionChain.Contains(uri)) + throw new TerminologyServiceException($"ValueSet expansion encountered a cycling dependency from {inclusionChain.Peek()} back to {uri}."); + + if (Settings.ValueSetSource == null) + throw Error.InvalidOperation($"No valueset resolver available to resolve valueset '{uri}', " + + "set ValueSetExpander.Settings.ValueSetSource to fix."); + + var importedVs = await Settings.ValueSetSource.AsAsync().FindValueSetAsync(uri).ConfigureAwait(false) + ?? throw new ValueSetUnknownException($"The ValueSet expander cannot find system '{uri}', so the expansion cannot be completed."); + if (!importedVs.HasExpansion) await ExpandAsync(importedVs, inclusionChain).ConfigureAwait(false); + + return importedVs.HasExpansion + ? importedVs.Expansion.Contains + : throw new ValueSetUnknownException($"Expansion returned neither an error, nor an expansion for ValueSet with canonical reference '{uri}'"); + } + + private async T.Task> getConceptsFromCodeSystem(string uri) + { + if (Settings.ValueSetSource == null) + throw Error.InvalidOperation($"No terminology service available to resolve references to codesystem '{uri}', " + + "set ValueSetExpander.Settings.ValueSetSource to fix."); + + var importedCs = await Settings.ValueSetSource.AsAsync().FindCodeSystemAsync(uri).ConfigureAwait(false) + ?? throw new ValueSetUnknownException($"The ValueSet expander cannot find system '{uri}', so the expansion cannot be completed."); + + return importedCs.Concept.Select(c => c.ToContainsComponent(importedCs, Settings)).ToList(); + } + } + + + public static class ContainsSetExtensions + { + public static ValueSet.ContainsComponent Add(this List dest, string system, string version, string code, string display, List? designations = null, IEnumerable? children = null) + { + var newContains = new ValueSet.ContainsComponent + { + System = system, + Code = code, + Display = display, + Version = version + }; + + newContains.System = system; + newContains.Code = code; + newContains.Display = display; + newContains.Version = version; + newContains.Designation = designations; + + if (children != null) + newContains.Contains = new List(children); + + dest.Add(newContains); + + return newContains; + } + + public static void Remove(this List dest, string system, string code) + { + dest.RemoveAll(c => c.System == system && c.Code == code); + + // Look for this code in children too + foreach (var component in dest) + { + if (component.Contains.Any()) + component.Contains.Remove(system, code); + } + } + + public static void Remove(this List dest, List source) + { + //TODO: Pretty unclear what to do with child concepts in the source - they all need to be removed from dest? + foreach (var sourceConcept in source) + dest.Remove(sourceConcept.System, sourceConcept.Code); + } + + internal static ValueSet.ContainsComponent ToContainsComponent(this CodeSystem.ConceptDefinitionComponent source, CodeSystem system, ValueSetExpanderSettings settings) + { + var newContains = new ValueSet.ContainsComponent + { + System = system.Url, + Version = system.Version, + Code = source.Code, + Display = source.Display + }; + + newContains.System = system.Url; + newContains.Version = system.Version; + newContains.Code = source.Code; + newContains.Display = source.Display; + if (settings.IncludeDesignations) + newContains.Designation = source.Designation.ToValueSetDesignations(); + + var abstractProperty = source.ListConceptProperties(system, CodeSystem.CONCEPTPROPERTY_NOT_SELECTABLE).SingleOrDefault(); + if (abstractProperty?.Value is FhirBoolean isAbstract) + newContains.Abstract = isAbstract.Value; + + var inactiveProperty = source.ListConceptProperties(system, CodeSystem.CONCEPTPROPERTY_STATUS).SingleOrDefault(); + if (inactiveProperty?.Value is FhirBoolean isInactive) + newContains.Inactive = isInactive.Value; + + if (source.Concept.Any()) + newContains.Contains.AddRange( + source.Concept.Select(c => c.ToContainsComponent(system, settings))); + + return newContains; + } + + private static List ToValueSetDesignations(this List csDesignations) + { + var vsDesignations = new List(); + csDesignations.ForEach(d => vsDesignations.Add(d.ToValueSetDesignation())); + return vsDesignations; + } + + private static ValueSet.DesignationComponent ToValueSetDesignation(this CodeSystem.DesignationComponent csDesignation) + { + return new ValueSet.DesignationComponent + { + Language = csDesignation.Language, + Use = csDesignation.Use, + Value = csDesignation.Value + }; + } + + } +} + +#nullable restore \ No newline at end of file diff --git a/src/Hl7.Fhir.STU3/Specification/Terminology/ValueSetExpander.cs b/src/Hl7.Fhir.STU3/Specification/Terminology/ValueSetExpander.cs new file mode 100644 index 0000000000..17394a34bd --- /dev/null +++ b/src/Hl7.Fhir.STU3/Specification/Terminology/ValueSetExpander.cs @@ -0,0 +1,387 @@ +#nullable enable + +/* + * Copyright (c) 2016, Firely (info@fire.ly) and contributors + * See the file CONTRIBUTORS for details. + * + * This file is licensed under the BSD 3-Clause license + * available at https://raw.githubusercontent.com/FirelyTeam/firely-net-sdk/master/LICENSE + */ + +using Hl7.Fhir.Model; +using Hl7.Fhir.Specification.Source; +using Hl7.Fhir.Utility; +using System; +using System.Collections.Generic; +using System.Linq; +using T = System.Threading.Tasks; + +namespace Hl7.Fhir.Specification.Terminology +{ + /// + /// Expands valuesets by processing their include and exclude filters. Will create an in-place expansion. + /// + public class ValueSetExpander + { + /// + /// Settings to control the behaviour of the expansion. + /// + public ValueSetExpanderSettings Settings { get; } + + /// + /// Create a new expander with specific settings. + /// + /// + public ValueSetExpander(ValueSetExpanderSettings settings) + { + Settings = settings; + } + + /// + /// Create a new expander with default settings + /// + public ValueSetExpander() : this(ValueSetExpanderSettings.CreateDefault()) + { + // nothing + } + + [Obsolete("ValueSetExpander now works best with asynchronous resolvers. Use ExpandAsync() instead.")] + public void Expand(ValueSet source) => TaskHelper.Await(() => ExpandAsync(source)); + + /// + /// Expand the include and exclude filters. Creates the + /// + /// + /// + public T.Task ExpandAsync(ValueSet source) => ExpandAsync(source, new()); + + internal async T.Task ExpandAsync(ValueSet source, Stack inclusionChain) + { + // Note we are expanding the valueset in-place, so it's up to the caller to decide whether + // to clone the valueset, depending on store and performance requirements. + source.Expansion = ValueSet.ExpansionComponent.Create(); + setExpansionParameters(source); + + try + { + inclusionChain.Push(source.Url); + await handleCompose(source, inclusionChain).ConfigureAwait(false); + } + catch (Exception) + { + // Expansion failed - remove (partial) expansion + source.Expansion = null; + throw; + } + finally + { + inclusionChain.Pop(); + } + } + + private void setExpansionParameters(ValueSet vs) + { + vs.Expansion.Parameter = new List(); + if (Settings.IncludeDesignations) + { + vs.Expansion.Parameter.Add(new ValueSet.ParameterComponent + { + Name = "includeDesignations", + Value = new FhirBoolean(true) + }); + } + //TODO add more parameters to the valuset here when we implement them. + } + + private async T.Task handleCompose(ValueSet source, Stack inclusionChain) + { + if (source.Compose == null) return; + + await handleInclude(source, inclusionChain).ConfigureAwait(false); + await handleExclude(source, inclusionChain).ConfigureAwait(false); + } + + + private class SystemAndCodeComparer : IEqualityComparer + { + public bool Equals(ValueSet.ContainsComponent? x, ValueSet.ContainsComponent? y) + { + if (ReferenceEquals(x, y)) return true; + if (x is null || y is null) return false; + + return x.Code == y.Code && x.System == y.System; + } + + public int GetHashCode(ValueSet.ContainsComponent obj) => (obj.Code ?? "").GetHashCode() ^ (obj.System ?? "").GetHashCode(); + } + + private static readonly IEqualityComparer _systemAndCodeComparer = new SystemAndCodeComparer(); + + // This function contains the main logic of expanding an include/exclude ConceptSet. + // It processes mainly two parts, which each return 0..* expanded ContainsComponents: + // * The "System" group (system + filter + concepts). + // * The "ValueSet" group (valueset) + // The results of both of these parts are then intersected. + private async T.Task> processConceptSet(ValueSet.ConceptSetComponent conceptSet, Stack inclusionChain) + { + // vsd-1 + if (!conceptSet.ValueSetElement.Any() && conceptSet.System == null) + throw Error.InvalidOperation($"Encountered a ConceptSet with neither a 'system' nor a 'valueset'"); + + // Process the system group + var systemResult = await processSystemGroup(conceptSet).ConfigureAwait(false); + + // Process the ValueSet group + var valueSetResult = await processValueSetGroup(conceptSet, inclusionChain).ConfigureAwait(false); + + // > For each compose.include: (...) Add the intersection of the result set from the system(step 1) and all of the result sets from the value sets(step 2) to the expansion. + // Most of the time, the expansion contains stuff from either the system (+enumerated concepts) or the valuesets. + // If that is the case, return the result directly. If both were specified, we need to calculate the intersection. + return (systemResult, valueSetResult) switch + { + { systemResult.Count: 0, valueSetResult.Count: 0 } => systemResult, // just return an empty list + { systemResult.Count: > 0, valueSetResult.Count: 0 } => systemResult, + { systemResult.Count: 0, valueSetResult.Count: > 0 } => valueSetResult, + _ => systemResult.Intersect(valueSetResult, _systemAndCodeComparer).ToList() + }; + } + + // > For each valueSet, find the referenced value set by ValueSet.url, expand that + // > to produce a collection of result sets.This means that expansion across imports is a recursive process. + private async T.Task> processValueSetGroup(ValueSet.ConceptSetComponent conceptSet, Stack inclusionChain) + { + var result = new List(); + + if (conceptSet.ValueSetElement.Any()) + { + // > valueSet(s) only: Codes are 'selected' for inclusion if they are in all the referenced value sets + // "all the referenced sets" means we need to calculate the intersection of the expanded valuesets. + var expanded = await T.Task.WhenAll(conceptSet.ValueSet.Select(vs => expandValueSetAndFilterOnSystem(vs))).ConfigureAwait(false); + var concepts = expanded.Length == 1 ? expanded.Single() : expanded.Aggregate((l, r) => l.Intersect(r, _systemAndCodeComparer)); + + addCapped(result, concepts, $"Import of valuesets '{string.Join(",", conceptSet.ValueSet)}' would result in an expqansion larger than the maximum expansion size."); + + // > valueSet and System: Codes are 'selected' for inclusion if they are selected by the code system selection (after checking for concept and filter) and if they are in all the referenced value sets + // If a System was specified, simulate a intersection between the codesystem and the valuesets by filtering on the + // codesystem's canonical. See previous if. + IEnumerable filterOnSystem(IEnumerable concepts) => + conceptSet.System is not null ? concepts.Where(c => c.System == conceptSet.System) : concepts; + + async T.Task> expandValueSetAndFilterOnSystem(string canonical) + { + var expansion = await getExpansionForValueSet(canonical, inclusionChain).ConfigureAwait(false); + return filterOnSystem(expansion); + } + } + + return result; + } + + // > If there is a system, identify the correct version of the code system, and then: + // > * If there are no codes or filters, add every code in the code system to the result set. + // > * If codes are listed, check that they are valid, and check their active status, and if ok, add them to the result set(the parameters to the $expand operation may be used to control whether active codes are included). + // > * If any filters are present, process them in order(as explained above), and add the intersection of their results to the result set. + private async T.Task> processSystemGroup(ValueSet.ConceptSetComponent conceptSet) + { + var result = new List(); + + if (conceptSet.System != null) + { + // We should probably really have to look this code up in the original codesystem to know something about 'abstract' + // and what would we do with a hierarchy if we encountered that in the include? + // Filter and Concept are mutually exclusive (vsd-3) + if (conceptSet.Filter.Any()) + throw new ValueSetExpansionTooComplexException($"ConceptSets with a filter are not yet supported."); + else if (conceptSet.Concept.Any()) + { + var convertedConcepts = conceptSet.Concept.Select(c => + ContainsSetExtensions.BuildContainsComponent(conceptSet.System, conceptSet.Version, c.Code, c.Display, Settings.IncludeDesignations ? c.Designation : null)); + + addCapped(result, convertedConcepts, $"Adding the enumerated concepts to the expansion would result in a valueset larger than the maximum expansion size."); + } + else if (!conceptSet.ValueSetElement.Any()) + { + // Do a full import of the codesystem. Conceptually, if a ValueSet is specified, we should include the + // *intersection* of the ValueSets and the System. That is computationally expensive, so instead we will not + // include the Codesystem at all if there are valuesets, but include the ValueSets instead, filtering them + // on the given system instead (see next if). This is not the same if there are codes in the valueset that + // use a system, but are not actually defined within that codesystem, but that sounds illegal to me anyway. + var importedConcepts = await getAllConceptsFromCodeSystem(conceptSet.System).ConfigureAwait(false); + addCapped(result, importedConcepts, $"Import of full codesystem '{conceptSet.System}' would result in an expansion larger than the maximum expansion size."); + } + } + + return result; + } + + private void addCapped(List dest, IEnumerable source, string message) + { + var capacityLeft = Settings.MaxExpansionSize - dest.Count; + var cappedSource = source.Take(capacityLeft + 1).ToList(); + + if (cappedSource.Count == capacityLeft + 1) + throw new ValueSetExpansionTooBigException(message); + + dest.AddRange(cappedSource); + } + + private async T.Task handleInclude(ValueSet source, Stack inclusionChain) + { + if (!source.Compose.Include.Any()) return; + + int csIndex = 0; + foreach (var include in source.Compose.Include) + { + var includedConcepts = await processConceptSet(include, inclusionChain).ConfigureAwait(false); + + // Yes, exclusion could make this smaller again, but alas, before we have processed those we might have run out of memory + addCapped(source.Expansion.Contains, includedConcepts, $"Inclusion of {includedConcepts.Count} concepts from conceptset #{csIndex}' to " + + $"valueset '{source.Url}' ({source.Expansion.Total} concepts) would be larger than the set maximum size ({Settings.MaxExpansionSize})"); + + var original = source.Expansion.Total ?? 0; + source.Expansion.Total = original + includedConcepts.CountConcepts(); + csIndex += 1; + } + } + + private async T.Task handleExclude(ValueSet source, Stack inclusionChain) + { + if (!source.Compose.Exclude.Any()) return; + + foreach (var exclude in source.Compose.Exclude) + { + var excludedConcepts = await processConceptSet(exclude, inclusionChain).ConfigureAwait(false); + + source.Expansion.Contains.Remove(excludedConcepts); + + var original = source.Expansion.Total ?? 0; + source.Expansion.Total = original - excludedConcepts.CountConcepts(); + } + } + + private async T.Task> getExpansionForValueSet(string uri, Stack inclusionChain) + { + if (inclusionChain.Contains(uri)) + throw new TerminologyServiceException($"ValueSet expansion encountered a cycling dependency from {inclusionChain.Peek()} back to {uri}."); + + if (Settings.ValueSetSource == null) + throw Error.InvalidOperation($"No valueset resolver available to resolve valueset '{uri}', so the expansion cannot be completed."); + + var importedVs = await Settings.ValueSetSource.AsAsync().FindValueSetAsync(uri).ConfigureAwait(false) + ?? throw new ValueSetUnknownException($"The ValueSet expander cannot find valueset '{uri}', so the expansion cannot be completed."); + if (!importedVs.HasExpansion) await ExpandAsync(importedVs, inclusionChain).ConfigureAwait(false); + + return importedVs.HasExpansion + ? importedVs.Expansion.Contains + : throw new ValueSetUnknownException($"Expansion returned neither an error, nor an expansion for ValueSet with canonical reference '{uri}'"); + } + + private async T.Task> getAllConceptsFromCodeSystem(string uri) + { + if (Settings.ValueSetSource == null) + throw Error.InvalidOperation($"No valueset resolver available to resolve codesystem '{uri}', so the expansion cannot be completed."); + + var importedCs = await Settings.ValueSetSource.AsAsync().FindCodeSystemAsync(uri).ConfigureAwait(false) + ?? throw new ValueSetUnknownException($"The ValueSet expander cannot find codesystem '{uri}', so the expansion cannot be completed."); + + return importedCs.Concept.Select(c => c.ToContainsComponent(importedCs, Settings)); + } + } + + + public static class ContainsSetExtensions + { + internal static ValueSet.ContainsComponent BuildContainsComponent(string system, string version, string code, string display, List? designations = null, IEnumerable? children = null) + { + return new ValueSet.ContainsComponent + { + System = system, + Code = code, + Display = display, + Version = version, + Designation = designations, + Contains = children?.ToList() + }; + + } + + public static ValueSet.ContainsComponent Add(this List dest, string system, string version, string code, string display, List? designations = null, IEnumerable? children = null) + { + var newContains = BuildContainsComponent(system, version, code, display, designations, children); + dest.Add(newContains); + + return newContains; + } + + public static void Remove(this List dest, string system, string code) + { + dest.RemoveAll(c => c.System == system && c.Code == code); + + // Look for this code in children too + foreach (var component in dest) + { + if (component.Contains.Any()) + component.Contains.Remove(system, code); + } + } + + public static void Remove(this List dest, List source) + { + //TODO: Pretty unclear what to do with child concepts in the source - they all need to be removed from dest? + foreach (var sourceConcept in source) + dest.Remove(sourceConcept.System, sourceConcept.Code); + } + + internal static ValueSet.ContainsComponent ToContainsComponent(this CodeSystem.ConceptDefinitionComponent source, CodeSystem system, ValueSetExpanderSettings settings) + { + var newContains = new ValueSet.ContainsComponent + { + System = system.Url, + Version = system.Version, + Code = source.Code, + Display = source.Display + }; + + newContains.System = system.Url; + newContains.Version = system.Version; + newContains.Code = source.Code; + newContains.Display = source.Display; + if (settings.IncludeDesignations) + newContains.Designation = source.Designation.ToValueSetDesignations(); + + var abstractProperty = source.ListConceptProperties(system, CodeSystem.CONCEPTPROPERTY_NOT_SELECTABLE).SingleOrDefault(); + if (abstractProperty?.Value is FhirBoolean isAbstract) + newContains.Abstract = isAbstract.Value; + + var inactiveProperty = source.ListConceptProperties(system, CodeSystem.CONCEPTPROPERTY_STATUS).SingleOrDefault(); + if (inactiveProperty?.Value is FhirBoolean isInactive) + newContains.Inactive = isInactive.Value; + + if (source.Concept.Any()) + newContains.Contains.AddRange( + source.Concept.Select(c => c.ToContainsComponent(system, settings))); + + return newContains; + } + + private static List ToValueSetDesignations(this List csDesignations) + { + var vsDesignations = new List(); + csDesignations.ForEach(d => vsDesignations.Add(d.ToValueSetDesignation())); + return vsDesignations; + } + + private static ValueSet.DesignationComponent ToValueSetDesignation(this CodeSystem.DesignationComponent csDesignation) + { + return new ValueSet.DesignationComponent + { + Language = csDesignation.Language, + Use = csDesignation.Use, + Value = csDesignation.Value + }; + } + + } +} + +#nullable restore \ No newline at end of file diff --git a/src/Hl7.Fhir.Specification.STU3.Tests/Source/TerminologyTests.cs b/src/Hl7.Fhir.Specification.STU3.Tests/Source/TerminologyTests.cs index f08ebe2bce..695692392a 100644 --- a/src/Hl7.Fhir.Specification.STU3.Tests/Source/TerminologyTests.cs +++ b/src/Hl7.Fhir.Specification.STU3.Tests/Source/TerminologyTests.cs @@ -12,6 +12,76 @@ namespace Hl7.Fhir.Specification.Tests { + /// + /// Extension methods to help constructing a in code. + /// + public static class ValueSetConfigurators + { + /// + /// Adds a new to the includes of a and returns + /// it to enable fluent construction of valuesets. + /// + public static ValueSet Includes(this ValueSet vs, Action a) + { + var csc = new ValueSet.ConceptSetComponent(); + a(csc); + if (vs.Compose is null) vs.Compose = new(); + vs.Compose.Include.Add(csc); + return vs; + } + + /// + /// Adds a new to the excludes of a and returns + /// it to enable fluent construction of valuesets. + /// + public static ValueSet Excludes(this ValueSet vs, Action a) + { + var csc = new ValueSet.ConceptSetComponent(); + a(csc); + if (vs.Compose is null) vs.Compose = new(); + vs.Compose.Exclude.Add(csc); + return vs; + } + + /// + /// Sets the system of a to enable fluent construction of an include or exclude. + /// + public static ValueSet.ConceptSetComponent System(this ValueSet.ConceptSetComponent component, string system) + { + component.System = system; + return component; + } + + /// + /// Adds to the concepts of a to enable fluent construction of an include or exclude. + /// + public static ValueSet.ConceptSetComponent Concepts(this ValueSet.ConceptSetComponent component, IEnumerable refcomponents) + { + component.Concept.AddRange(refcomponents); + return component; + } + + /// component.Concepts(refcomponents.AsEnumerable()); + + /// component.Concepts(new ValueSet.ConceptReferenceComponent() { Code = code }); + + /// + /// Adds to the valuesets of a to enable fluent construction of an include or exclude. + /// + public static ValueSet.ConceptSetComponent ValueSets(this ValueSet.ConceptSetComponent component, IEnumerable canonicals) + { + component.ValueSetElement.AddRange(canonicals.Select(c => new FhirUri(c))); + return component; + } + + /// + /// Adds to the valuesets of a to enable fluent construction of an include or exclude. + /// + public static ValueSet.ConceptSetComponent ValueSets(this ValueSet.ConceptSetComponent component, params string[] canonicals) => component.ValueSets(canonicals.AsEnumerable()); + } + public class TerminologyTests { private readonly IAsyncResourceResolver _resolver = new CachedResolver(ZipSource.CreateValidationSource()); @@ -21,6 +91,38 @@ public class TerminologyTests // Use here a FhirPackageSource without the expansion package. private readonly IAsyncResourceResolver _resolverWithoutExpansions = new CachedResolver(ZipSource.CreateValidationSource()); + [Fact] + public async T.Task CanCombineValueSets() + { + var vs1 = buildVs("http://valuesetA", "1", "2", "3").Includes(i => i.System("http://systemX").Concept("A")); + var vs2 = buildVs("http://valuesetB", "2", "3", "4"); + var vsCombined = new ValueSet() { Url = "http://combined", Status = PublicationStatus.Active }.Includes(i => i.ValueSets("http://valuesetA", "http://valuesetB")); + + var resolver = new InMemoryResourceResolver(vs1, vs2); + var expander = new ValueSetExpander(new() { ValueSetSource = resolver }); + await expander.ExpandAsync(vsCombined); + + vsCombined.Expansion.Contains.Select(c => c.Code).Should().BeEquivalentTo(new[] { "2", "3" }, because: "specifying multiple valuesets should return an intersection."); + vsCombined.Expansion.Contains.Should().AllSatisfy(c => c.System.Should().Be("http://system")); + + var vsFiltered = new ValueSet() { Url = "http://filtered", Status = PublicationStatus.Active }.Includes(i => i.ValueSets("http://valuesetA").System("http://systemX")); + await expander.ExpandAsync(vsFiltered); + vsFiltered.Expansion.Contains.Select(c => c.Code).Should().BeEquivalentTo(new[] { "A" }, because: "filtering on systemX only returns codes from systemX"); + + static ValueSet buildVs(string canonical, params string[] codes) + { + + var concepts = codes.Select(c => new ValueSet.ConceptReferenceComponent() { Code = c }).ToList(); + + return new ValueSet() + { + Url = canonical, + Version = "2022-08-01", + Status = PublicationStatus.Active, + }.Includes(i => i.System("http://system").Concepts(concepts)); + } + } + [Fact] public async T.Task ExpansionOfWholeSystem() { @@ -112,7 +214,7 @@ public async T.Task TestExpandingVsWithUnknownSystem() }; var job = async () => await expander.ExpandAsync(vs); - await job.Should().ThrowAsync().WithMessage("The ValueSet expander cannot find system 'http://www.unknown.org/', so the expansion cannot be completed."); + await job.Should().ThrowAsync().WithMessage("The ValueSet expander cannot find codesystem 'http://www.unknown.org/', so the expansion cannot be completed."); } [Fact]