From 1bb29aa4fbec3c44ba353891268ceb350185c542 Mon Sep 17 00:00:00 2001 From: Yannan <73408381+YannanGao-gs@users.noreply.github.com> Date: Tue, 23 Jul 2024 09:03:22 -0400 Subject: [PATCH] support use function pointer in curated template query (#2955) --- .../PackageableElementSixthPassBuilder.java | 1 + .../toPureGraph/extension/Processor.java | 88 ++++++++ .../pom.xml | 8 + .../DataSpaceCompilerExtension.java | 136 ++++++++++--- .../TestDataSpaceCompilationFromGrammar.java | 139 ++++++++++++- .../analytics/DataSpaceAnalyticsHelper.java | 29 ++- ...eAnalyticsArtifactGenerationExtension.java | 11 +- ...emplateExecutablesWithFunctionPointer.pure | 192 ++++++++++++++++++ ...thTemplateExecutablesWithInlineQuery.pure} | 0 .../from/antlr4/DataSpaceParserGrammar.g4 | 2 +- .../from/DataSpaceParseTreeWalker.java | 52 +++-- .../to/DataSpaceGrammarComposerExtension.java | 2 +- .../DataSpaceInlineTemplateExecutable.java | 34 ++++ .../DataSpaceTemplateExecutable.java | 22 +- .../DataSpaceTemplateExecutablePointer.java | 33 +++ 15 files changed, 698 insertions(+), 51 deletions(-) create mode 100644 legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/resources/models/dataSpaceWithTemplateExecutablesWithFunctionPointer.pure rename legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/resources/models/{dataSpaceWithTemplateExecutables.pure => dataSpaceWithTemplateExecutablesWithInlineQuery.pure} (100%) create mode 100644 legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceInlineTemplateExecutable.java create mode 100644 legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceTemplateExecutablePointer.java diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/PackageableElementSixthPassBuilder.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/PackageableElementSixthPassBuilder.java index f4278c9e733..d2c30528fe7 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/PackageableElementSixthPassBuilder.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/PackageableElementSixthPassBuilder.java @@ -40,6 +40,7 @@ public PackageableElementSixthPassBuilder(CompileContext context) @Override public PackageableElement visit(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement element) { + this.context.getExtraProcessorOrThrow(element).processSixthPass(element, this.context); return null; } diff --git a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/extension/Processor.java b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/extension/Processor.java index 831835a5d97..c5b99a5e4a2 100644 --- a/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/extension/Processor.java +++ b/legend-engine-core/legend-engine-core-base/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/extension/Processor.java @@ -56,6 +56,11 @@ public final void processFifthPass(PackageableElement element, CompileContext co processElementFifthPass(castElement(element), context); } + public final void processSixthPass(PackageableElement element, CompileContext context) + { + processElementSixthPass(castElement(element), context); + } + @Override public final boolean equals(Object other) { @@ -96,6 +101,12 @@ protected void processElementFifthPass(T element, CompileContext context) // nothing by default } + protected void processElementSixthPass(T element, CompileContext context) + { + // nothing by default + } + + private T castElement(PackageableElement element) { return getElementClass().cast(element); @@ -258,4 +269,81 @@ protected void processElementFifthPass(T element, CompileContext context) } }; } + + public static Processor newProcessor(Class elementClass, + Collection> prerequisiteClasses, + BiFunction firstPass, + BiConsumer secondPass, + BiConsumer thirdPass, + BiConsumer fourthPass, + BiConsumer fifthPass, + BiConsumer sixthPass) + { + Collection> resolvedPrerequisiteClasses = (prerequisiteClasses == null) ? Collections.emptyList() : prerequisiteClasses; + return new Processor() + { + @Override + public Class getElementClass() + { + return elementClass; + } + + @Override + public Collection> getPrerequisiteClasses() + { + return resolvedPrerequisiteClasses; + } + + @Override + protected org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement processElementFirstPass(T element, CompileContext context) + { + return firstPass.apply(element, context); + } + + @Override + protected void processElementSecondPass(T element, CompileContext context) + { + if (secondPass != null) + { + secondPass.accept(element, context); + } + } + + @Override + protected void processElementThirdPass(T element, CompileContext context) + { + if (thirdPass != null) + { + thirdPass.accept(element, context); + } + } + + @Override + protected void processElementFourthPass(T element, CompileContext context) + { + if (fourthPass != null) + { + fourthPass.accept(element, context); + } + } + + @Override + protected void processElementFifthPass(T element, CompileContext context) + { + if (fifthPass != null) + { + fifthPass.accept(element, context); + } + } + + @Override + protected void processElementSixthPass(T element, CompileContext context) + { + if (sixthPass != null) + { + sixthPass.accept(element, context); + } + } + }; + } } diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/pom.xml b/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/pom.xml index e4b849123b3..9c86404a48a 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/pom.xml +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/pom.xml @@ -71,6 +71,10 @@ org.finos.legend.engine legend-engine-protocol-pure + + org.finos.legend.engine + legend-engine-pure-platform-java + org.finos.legend.engine legend-engine-xt-data-space-protocol @@ -79,6 +83,10 @@ org.finos.legend.engine legend-engine-language-pure-compiler + + org.finos.legend.engine + legend-engine-pure-platform-dsl-store-java + org.finos.legend.engine legend-engine-xt-diagram-compiler diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataSpaceCompilerExtension.java b/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataSpaceCompilerExtension.java index d84875bccf2..c41aff72178 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataSpaceCompilerExtension.java +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataSpaceCompilerExtension.java @@ -14,6 +14,7 @@ package org.finos.legend.engine.language.pure.compiler.toPureGraph; +import org.eclipse.collections.api.RichIterable; import org.eclipse.collections.api.block.function.Function; import org.eclipse.collections.api.block.function.Function2; import org.eclipse.collections.api.block.function.Function3; @@ -50,33 +51,24 @@ import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.MappingIncludeDataSpace; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpacePackageableElementExecutable; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpaceTemplateExecutable; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpaceTemplateExecutablePointer; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpaceInlineTemplateExecutable; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpaceExecutionContext; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.diagram.Diagram; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.runtime.PackageableRuntime; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.StoreProviderPointer; import org.finos.legend.engine.shared.core.operational.Assert; import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; -import org.finos.legend.pure.generated.Root_meta_pure_data_DataElementReference; -import org.finos.legend.pure.generated.Root_meta_pure_data_EmbeddedData; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpace; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpaceDiagram_Impl; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpaceExecutionContext; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpaceExecutionContext_Impl; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpaceSupportCombinedInfo_Impl; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpaceSupportEmail_Impl; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpaceSupportInfo; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpace_Impl; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_extension_TaggedValue_Impl; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_type_generics_GenericType_Impl; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_valuespecification_InstanceValue_Impl; -import org.finos.legend.pure.generated.Root_meta_pure_runtime_PackageableRuntime; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpacePackageableElementExecutable_Impl; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpaceTemplateExecutable_Impl; +import org.finos.legend.pure.generated.*; import org.finos.legend.pure.m3.coreinstance.meta.pure.mapping.Mapping; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionDefinition; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.InstanceValue; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.ValueSpecification; import org.finos.legend.pure.m3.coreinstance.meta.pure.store.Store; +import org.finos.legend.pure.m3.navigation.function.FunctionDescriptor; +import org.finos.legend.pure.m3.navigation.function.InvalidFunctionDescriptorException; import java.util.Collections; import java.util.HashSet; @@ -195,7 +187,7 @@ public Iterable> getExtraProcessors() MutableSet excludePaths = ListIterate.select(dataSpace.elements, el -> el.exclude != null && el.exclude).collect(el -> el.path).toSet(); includes.forEach(include -> HelperDataSpaceBuilder.collectElements(include, elements, excludePaths, context)); - metamodel._elements(elements.toSortedList(Comparator.comparing(el -> HelperModelBuilder.getElementFullPath(el, context.pureModel.getExecutionSupport())))); + metamodel._elements(elements.toSortedList(Comparator.comparing(el -> getElementFullPath(el, context.pureModel.getExecutionSupport())))); } // executables @@ -217,11 +209,28 @@ else if (executable instanceof DataSpaceTemplateExecutable) { throw new EngineException("Data space template executable's executionContextKey, " + ((DataSpaceTemplateExecutable) executable).executionContextKey + ", is not valid. Please specify one from " + dataSpace.executionContexts.stream().map(c -> c.name).collect(Collectors.toList()).toString(), dataSpace.sourceInformation, EngineErrorType.COMPILATION); } + + FunctionDefinition templateExecutableQuery = null; + if (executable instanceof DataSpaceInlineTemplateExecutable) + { + templateExecutableQuery = HelperValueSpecificationBuilder.buildLambda(((DataSpaceInlineTemplateExecutable) executable).query, context); + } + else if (executable instanceof DataSpaceTemplateExecutablePointer) + { + try + { + templateExecutableQuery = (FunctionDefinition) context.resolvePackageableElement(FunctionDescriptor.functionDescriptorToId((((DataSpaceTemplateExecutablePointer) executable).query).path), (((DataSpaceTemplateExecutablePointer) executable).query).sourceInformation); + } + catch (InvalidFunctionDescriptorException e) + { + throw new RuntimeException(e); + } + } return new Root_meta_pure_metamodel_dataSpace_DataSpaceTemplateExecutable_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::dataSpace::DataSpaceTemplateExecutable")) ._id(((DataSpaceTemplateExecutable) executable).id) ._title(executable.title) ._description(executable.description) - ._query(HelperValueSpecificationBuilder.buildLambda(((DataSpaceTemplateExecutable) executable).query, context)) + ._query(templateExecutableQuery) ._executionContextKey(((DataSpaceTemplateExecutable) executable).executionContextKey); } else @@ -282,6 +291,79 @@ else if (dataSpace.supportInfo instanceof DataSpaceSupportCombinedInfo) } metamodel._supportInfo(supportInfo); } + }, + (dataSpace, context) -> + { + // Move checking mapping/runtime utilized in the function within the curated template query does not align with the mapping/runtime applied in the execution context + // to the sixth pass because function body and parameter are processed in the fifthPassBuilder (see https://github.com/finos/legend-engine/pull/2918) + // if we do the checking in the fifth pass, function body and parameter will be an empty list + if (dataSpace.executables != null) + { + dataSpace.executables.forEach(executable -> + { + if (executable instanceof DataSpaceTemplateExecutablePointer) + { + try + { + FunctionDefinition templateExecutableQuery = (FunctionDefinition) context.resolvePackageableElement(FunctionDescriptor.functionDescriptorToId((((DataSpaceTemplateExecutablePointer) executable).query).path), (((DataSpaceTemplateExecutablePointer) executable).query).sourceInformation); + if (templateExecutableQuery instanceof Root_meta_pure_metamodel_function_ConcreteFunctionDefinition_Impl) + { + Optional fromFunc = templateExecutableQuery._expressionSequence().toList().stream() + .filter(func -> func instanceof Root_meta_pure_metamodel_valuespecification_SimpleFunctionExpression_Impl && ((Root_meta_pure_metamodel_valuespecification_SimpleFunctionExpression_Impl) func)._functionName.equals("from")).findAny(); + if (fromFunc.isPresent()) + { + Root_meta_pure_metamodel_valuespecification_SimpleFunctionExpression_Impl fromFuncExpression = (Root_meta_pure_metamodel_valuespecification_SimpleFunctionExpression_Impl) fromFunc.get(); + + // only check mapping and runtime if using ->from() in the most basic way e.g. ->from(model::Mapping, model::Runtime) + ValueSpecification mappingInstance = fromFuncExpression._parametersValues().toList().get(1); + if (mappingInstance instanceof Root_meta_pure_metamodel_valuespecification_InstanceValue_Impl) + { + String executionContextKey = ((DataSpaceTemplateExecutable) executable).executionContextKey != null ? ((DataSpaceTemplateExecutable) executable).executionContextKey : dataSpace.defaultExecutionContext; + DataSpaceExecutionContext executionContext = dataSpace.executionContexts.stream().filter(ec -> ec.name.equals(executionContextKey)).collect(Collectors.toList()).get(0); + // check if mapping matches to what is used in execution key + Object mappingImpl = ((Root_meta_pure_metamodel_valuespecification_InstanceValue_Impl) mappingInstance)._values().toList().get(0); + if (mappingImpl instanceof Root_meta_pure_mapping_Mapping_Impl) + { + String mappingPath = platform_pure_essential_meta_graph_elementToPath.Root_meta_pure_functions_meta_elementToPath_PackageableElement_1__String_1__String_1_((Root_meta_pure_mapping_Mapping_Impl) mappingImpl, "::", context.pureModel.getExecutionSupport()); + if (!mappingPath.equals(executionContext.mapping.path)) + { + throw new EngineException("The mapping utilized in the function within the curated template query does not align with the mapping applied in the execution context `" + executionContext.name + "`."); + } + } + // check if runtime matches to what is used in execution key + RichIterable runtimes = org.finos.legend.pure.generated.core_pure_corefunctions_metaExtension.Root_meta_pure_functions_meta_extractRuntimesFromFunctionDefinition_FunctionDefinition_1__Runtime_MANY_(templateExecutableQuery, context.pureModel.getExecutionSupport()); + Root_meta_core_runtime_Runtime runtimeImpl; + if (runtimes.isEmpty()) + { + runtimeImpl = null; + } + else if (runtimes.size() == 1) + { + runtimeImpl = runtimes.getOnly(); + } + else + { + throw new UnsupportedOperationException("More than one runtime present in from() function"); + } + if (runtimeImpl != null) + { + String runtimePath = context.pureModel.getRuntimePath(runtimeImpl); + if (!runtimePath.equals(executionContext.defaultRuntime.path)) + { + throw new EngineException("The runtime utilized in the function within the curated template query does not align with the runtime applied in the execution context `" + executionContext.name + "`."); + } + } + } + } + } + } + catch (InvalidFunctionDescriptorException e) + { + throw new RuntimeException(e); + } + } + }); + } } )); } @@ -296,7 +378,7 @@ public Map getExtraIncludedMappingHandlers() @Override - public List> getExtraValueSpecificationBuilderForFuncExpr() + public List> getExtraValueSpecificationBuilderForFuncExpr() { return org.eclipse.collections.impl.factory.Lists.mutable.with((packageableElement, context, processingContext) -> { @@ -328,15 +410,15 @@ private static Store resolveStore(StoreProviderPointer storeProviderPointer, Com Root_meta_pure_metamodel_dataSpace_DataSpace dataspace = DataSpaceCompilerExtension.dataSpacesIndex.get(packageAddress); ImmutableList stores = HelperMappingBuilder.getStoresFromMappingIgnoringIncludedMappings(dataspace._defaultExecutionContext()._mapping(),context); - String dataspacePath = HelperModelBuilder.getElementFullPath(dataspace, context.pureModel.getExecutionSupport()); - String mappingPath = HelperModelBuilder.getElementFullPath(dataspace._defaultExecutionContext()._mapping(), context.pureModel.getExecutionSupport()); + String dataSpacePath = getElementFullPath(dataspace, context.pureModel.getExecutionSupport()); + String mappingPath = getElementFullPath(dataspace._defaultExecutionContext()._mapping(), context.pureModel.getExecutionSupport()); if (stores.isEmpty()) { - throw new EngineException("Default mapping (" + mappingPath + ") in dataspace (" + dataspacePath + ") is not mapped to a store type supported by the ExtraSetImplementationSourceScanners.", storeProviderPointer.sourceInformation, EngineErrorType.COMPILATION); + throw new EngineException("Default mapping (" + mappingPath + ") in dataspace (" + dataSpacePath + ") is not mapped to a store type supported by the ExtraSetImplementationSourceScanners.", storeProviderPointer.sourceInformation, EngineErrorType.COMPILATION); } else if (stores.size() > 1) { - throw new EngineException("Default mapping (" + mappingPath + ") in dataspace (" + dataspacePath + throw new EngineException("Default mapping (" + mappingPath + ") in dataspace (" + dataSpacePath + ") cannot be resolved to a single store. Stores found : [" + stores.collect(s -> getElementFullPath(s, context.pureModel.getExecutionSupport())).makeString(",") + "]. Please notify dataspace owners that their dataspace cannot be used as a store interface.", @@ -364,7 +446,7 @@ private Root_meta_pure_data_EmbeddedData compileDataspaceDataElementReference(Em return ((Root_meta_pure_data_DataElementReference) Optional .ofNullable(DataSpaceCompilerExtension.dataSpacesIndex.get(data.dataElement.path)._defaultExecutionContext()._testData()) .orElseThrow(() -> new EngineException("Dataspace " + data.dataElement.path + " does not have test data in its default execution context.", data.sourceInformation, EngineErrorType.COMPILATION)) - )._dataElement()._data(); + )._dataElement()._data(); } throw new EngineException("Dataspace " + data.dataElement.path + " cannot be found.", data.sourceInformation, EngineErrorType.COMPILATION); } @@ -375,7 +457,7 @@ private Root_meta_pure_data_EmbeddedData compileDataspaceDataElementReference(Em @Override public Iterable>> getExtraDataElementReferencePMCDTraversers() { - return org.eclipse.collections.api.factory.Lists.immutable.with(DataSpaceCompilerExtension::getDataFromDataReferencePMCD); + return Lists.immutable.with(DataSpaceCompilerExtension::getDataFromDataReferencePMCD); } private static List getDataFromDataReferencePMCD(DataElementReference dataElementReference, PureModelContextData pureModelContextData) @@ -390,7 +472,7 @@ private static List getDataFromDataReferencePMCD(DataElementRefere public List>> getExtraFunctionHandlerDispatchBuilderInfoCollectors() { return Collections.singletonList((handlers) -> - org.eclipse.collections.api.factory.Lists.mutable.with( + Lists.mutable.with( new FunctionHandlerDispatchBuilderInfo("meta::pure::mapping::from_T_m__DataSpaceExecutionContext_1__T_m_", (List ps) -> ps.size() == 2 && handlers.isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "DataSpaceExecutionContext".equals(ps.get(1)._genericType()._rawType()._name()))), new FunctionHandlerDispatchBuilderInfo("meta::pure::metamodel::dataSpace::get_DataSpace_1__String_1__DataSpaceExecutionContext_1_", (List ps) -> ps.size() == 2 && handlers.isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "DataSpace".equals(ps.get(0)._genericType()._rawType()._name())) && handlers.isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name()))) @@ -401,7 +483,7 @@ public List>> getExt public List>> getExtraFunctionExpressionBuilderRegistrationInfoCollectors() { return Collections.singletonList((handlers) -> - org.eclipse.collections.api.factory.Lists.mutable.with( + Lists.mutable.with( new FunctionExpressionBuilderRegistrationInfo(org.eclipse.collections.impl.factory.Lists.mutable.with(0), handlers.m(handlers.h("meta::pure::mapping::from_T_m__DataSpaceExecutionContext_1__T_m_", false, ps -> handlers.res(ps.get(0)._genericType(), ps.get(0)._multiplicity()), ps -> ps.size() == 2 && handlers.typeOne(ps.get(1), "DataSpaceExecutionContext"))) ), diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/test/TestDataSpaceCompilationFromGrammar.java b/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/test/TestDataSpaceCompilationFromGrammar.java index fd4e846a02c..0e4e4c3c92f 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/test/TestDataSpaceCompilationFromGrammar.java +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/test/TestDataSpaceCompilationFromGrammar.java @@ -787,7 +787,7 @@ public void testDataSpaceWithElements() } @Test - public void testDataSpaceWithTemplateExecutable() + public void testDataSpaceWithTemplateExecutableWithInlineQuery() { test("Class model::element {}\n" + "Class model::sub::element {}\n" + @@ -988,6 +988,143 @@ public void testDataSpaceWithTemplateExecutable() "}\n", "COMPILATION error at [20:1-46:1]: Data space template executable's executionContextKey, random 1, is not valid. Please specify one from [Context 1]"); } + @Test + public void testDataSpaceWithTemplateExecutableWithFunctionPointer() + { + String model = "Class model::element {}\n" + + "Class model::sub::element\n" + + "{" + + " name: String[1];\n" + + "}\n" + + "function model::templateFunc(): meta::pure::tds::TabularDataSet[1]\n" + + "{\n" + + " model::sub::element.all()->project([x | $x.name],['Name'])->from(model::dummyMapping, model::dummyRuntime)\n" + + "}\n" + + "function model::templateFunc1(): meta::pure::tds::TabularDataSet[1]\n" + + "{\n" + + " model::sub::element.all()->project([x | $x.name],['Name'])->from(model::dummyMapping1, model::dummyRuntime)\n" + + "}\n" + + "function model::templateFunc2(): meta::pure::tds::TabularDataSet[1]\n" + + "{\n" + + " model::sub::element.all()->project([x | $x.name],['Name'])->from(model::dummyMapping, model::dummyRuntime1)\n" + + "}\n" + + "###Mapping\n" + + "Mapping model::dummyMapping\n" + + "(\n" + + ")\n" + + "\n" + + "Mapping model::dummyMapping1\n" + + "(\n" + + ")\n" + + "\n" + + "\n" + + "###Runtime\n" + + "Runtime model::dummyRuntime\n" + + "{\n" + + " mappings:\n" + + " [\n" + + " model::dummyMapping\n" + + " ];\n" + + "}\n" + + "Runtime model::dummyRuntime1\n" + + "{\n" + + " mappings:\n" + + " [\n" + + " model::dummyMapping\n" + + " ];\n" + + "}\n" + + "\n"; + + test(model + + "###DataSpace\n" + + "DataSpace model::dataSpace" + + "{\n" + + " executionContexts:\n" + + " [\n" + + " {\n" + + " name: 'Context 1';\n" + + " description: 'some information about the context';\n" + + " mapping: model::dummyMapping;\n" + + " defaultRuntime: model::dummyRuntime;\n" + + " }\n" + + " ];\n" + + " defaultExecutionContext: 'Context 1';\n" + + " elements: [model::element, model, model::sub];\n" + + " executables:\n" + + " [\n" + + " {\n" + + " id: 1;\n" + + " title: 'Template 1';\n" + + " query: model::templateFunc():TabularDataSet[1];\n" + + " executionContextKey: 'Context 1';\n" + + " },\n" + + " {\n" + + " id: 2;\n" + + " title: 'Template 2';\n" + + " query: model::templateFunc():TabularDataSet[1];\n" + + " },\n" + + " {\n" + + " id: 3;\n" + + " title: 'Template 3';\n" + + " description: 'an example of a template query';\n" + + " query: model::templateFunc():TabularDataSet[1];\n" + + " }\n" + + " ];\n" + + "}\n"); + + test(model + + "###DataSpace\n" + + "DataSpace model::dataSpace" + + "{\n" + + " executionContexts:\n" + + " [\n" + + " {\n" + + " name: 'Context 1';\n" + + " description: 'some information about the context';\n" + + " mapping: model::dummyMapping;\n" + + " defaultRuntime: model::dummyRuntime;\n" + + " }\n" + + " ];\n" + + " defaultExecutionContext: 'Context 1';\n" + + " elements: [model::element, model, model::sub];\n" + + " executables:\n" + + " [\n" + + " {\n" + + " id: 1;\n" + + " title: 'Template 1';\n" + + " query: model::templateFunc1():TabularDataSet[1];\n" + + " executionContextKey: 'Context 1';\n" + + " }\n" + + " ];\n" + + "}\n", " at [44:1-65:1]: Error in 'model::dataSpace': The mapping utilized in the function within the curated template query does not align with the mapping applied in the execution context `Context 1`."); + + test(model + + "###DataSpace\n" + + "DataSpace model::dataSpace" + + "{\n" + + " executionContexts:\n" + + " [\n" + + " {\n" + + " name: 'Context 1';\n" + + " description: 'some information about the context';\n" + + " mapping: model::dummyMapping;\n" + + " defaultRuntime: model::dummyRuntime;\n" + + " }\n" + + " ];\n" + + " defaultExecutionContext: 'Context 1';\n" + + " elements: [model::element, model, model::sub];\n" + + " executables:\n" + + " [\n" + + " {\n" + + " id: 1;\n" + + " title: 'Template 1';\n" + + " query: model::templateFunc2():TabularDataSet[1];\n" + + " executionContextKey: 'Context 1';\n" + + " }\n" + + " ];\n" + + "}\n", " at [44:1-65:1]: Error in 'model::dataSpace': The runtime utilized in the function within the curated template query does not align with the runtime applied in the execution context `Context 1`."); + } + @Test public void testDataSpaceWithUnsupportedElement() { diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/main/java/org/finos/legend/engine/generation/analytics/DataSpaceAnalyticsHelper.java b/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/main/java/org/finos/legend/engine/generation/analytics/DataSpaceAnalyticsHelper.java index 1dc379f7723..3340c56187a 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/main/java/org/finos/legend/engine/generation/analytics/DataSpaceAnalyticsHelper.java +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/main/java/org/finos/legend/engine/generation/analytics/DataSpaceAnalyticsHelper.java @@ -36,11 +36,14 @@ import org.finos.legend.engine.protocol.analytics.model.MappingModelCoverageAnalysisResult; import org.finos.legend.engine.protocol.pure.PureClientVersions; import org.finos.legend.engine.protocol.pure.v1.PureProtocolObjectMapperFactory; +import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementPointer; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.ResultType; import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.result.TDSResultType; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpace; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpaceInlineTemplateExecutable; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpaceTemplateExecutable; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpaceTemplateExecutablePointer; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Function; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.externalFormat.Binding; @@ -49,6 +52,7 @@ import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.service.PureMultiExecution; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.service.PureSingleExecution; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.service.Service; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.executionContext.BaseExecutionContext; import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; import org.finos.legend.engine.shared.core.api.grammar.RenderStyle; @@ -57,10 +61,13 @@ import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.extension.Profile; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.ConcreteFunctionDefinition; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionDefinition; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Class; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Enum; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Enumeration; +import org.finos.legend.pure.m3.navigation.function.FunctionDescriptor; +import org.finos.legend.pure.m3.navigation.function.InvalidFunctionDescriptorException; import java.util.ArrayList; import java.util.List; @@ -402,14 +409,32 @@ else if (elementDoc instanceof Root_meta_pure_metamodel_dataSpace_analytics_Data // get V1 lambda DataSpaceTemplateExecutable executableV1 = (DataSpaceTemplateExecutable) dataSpaceProtocol.executables.stream().filter(e -> e instanceof DataSpaceTemplateExecutable && ((DataSpaceTemplateExecutable) e).id.equals(((Root_meta_pure_metamodel_dataSpace_DataSpaceTemplateExecutable) executable)._id())).findFirst().get(); - templateExecutableInfo.query = executableV1.query.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().withIndentation(getTabSize(1)).build()); + FunctionDefinition lambda = null; + if (executableV1 instanceof DataSpaceInlineTemplateExecutable) + { + lambda = (FunctionDefinition) ((Root_meta_pure_metamodel_dataSpace_DataSpaceTemplateExecutable) executable)._query(); + templateExecutableInfo.query = (((DataSpaceInlineTemplateExecutable) executableV1).query).accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance().withIndentation(getTabSize(1)).build()); + } + else if (executableV1 instanceof DataSpaceTemplateExecutablePointer) + { + try + { + PackageableElementPointer functionPointer = ((DataSpaceTemplateExecutablePointer) executableV1).query; + lambda = pureModel.getConcreteFunctionDefinition(FunctionDescriptor.functionDescriptorToId(functionPointer.path), functionPointer.sourceInformation); + templateExecutableInfo.query = functionPointer.path; + } + catch (InvalidFunctionDescriptorException e) + { + throw new RuntimeException(e); + } + } org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpaceExecutionContext executionContext = ((Root_meta_pure_metamodel_dataSpace_DataSpaceTemplateExecutable) executable)._executionContextKey() == null ? dataSpace._defaultExecutionContext() : dataSpace._executionContexts().toList().stream().filter(c -> c._name().equals(((Root_meta_pure_metamodel_dataSpace_DataSpaceTemplateExecutable) executable)._executionContextKey())).findFirst().get(); templateExecutableInfo.executionContextKey = ((Root_meta_pure_metamodel_dataSpace_DataSpaceTemplateExecutable) executable)._executionContextKey() == null ? dataSpace._defaultExecutionContext()._name() : ((Root_meta_pure_metamodel_dataSpace_DataSpaceTemplateExecutable) executable)._executionContextKey(); executableAnalysisResult.info = templateExecutableInfo; executableAnalysisResult.result = buildExecutableResult(PlanGenerator.generateExecutionPlanDebug( - ((Root_meta_pure_metamodel_dataSpace_DataSpaceTemplateExecutable) executable)._query(), + lambda, executionContext._mapping(), executionContext._defaultRuntime()._runtimeValue(), HelperValueSpecificationBuilder.processExecutionContext(new BaseExecutionContext(), pureModel.getContext()), diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/java/org/finos/legend/engine/generation/TestDataSpaceAnalyticsArtifactGenerationExtension.java b/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/java/org/finos/legend/engine/generation/TestDataSpaceAnalyticsArtifactGenerationExtension.java index 1c3e987df6d..e945449a803 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/java/org/finos/legend/engine/generation/TestDataSpaceAnalyticsArtifactGenerationExtension.java +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/java/org/finos/legend/engine/generation/TestDataSpaceAnalyticsArtifactGenerationExtension.java @@ -27,7 +27,6 @@ import org.finos.legend.engine.shared.core.deployment.DeploymentMode; import org.finos.legend.engine.shared.core.deployment.DeploymentStateAndVersions; import org.finos.legend.engine.shared.core.identity.Identity; -import org.finos.legend.engine.shared.core.identity.factory.*; import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpace; import org.junit.Assert; import org.junit.Test; @@ -120,9 +119,15 @@ public void testAnalyticsForDataSpaceWithExecutables() throws Exception } @Test - public void testAnalyticsForDataSpaceTemplateWithExecutables() throws Exception + public void testAnalyticsForDataSpaceTemplateWithExecutablesWithInlineQuery() throws Exception { - testDataSpaceAnalyticsArtifactGenerationExtension("models/dataSpaceWithTemplateExecutables.pure", "domain::COVIDDataspace", "{\"defaultExecutionContext\":\"dummyContext\",\"description\":\"this is description of this COVIDDataspace\",\"diagrams\":[],\"elementDocs\":[],\"elements\":[],\"executables\":[{\"description\":\"Some more exec description\",\"info\":{\"_type\":\"templateExecutableInfo\",\"executionContextKey\":\"dummyContext\",\"id\":\"1\",\"query\":\"|domain::COVIDData.all()->project([x: domain::COVIDData[1]|$x.cases], ['Cases'])\"},\"result\":{\"_type\":\"tds\",\"columns\":[{\"name\":\"Cases\",\"relationalType\":\"INTEGER\",\"type\":\"Float\"}]},\"title\":\"this is template\"},{\"info\":{\"_type\":\"templateExecutableInfo\",\"executionContextKey\":\"dummyContext\",\"id\":\"2\",\"query\":\"|domain::COVIDData.all()->project([x: domain::COVIDData[1]|$x.fips], ['Fips'])\"},\"result\":{\"_type\":\"tds\",\"columns\":[{\"name\":\"Fips\",\"relationalType\":\"VARCHAR(200)\",\"type\":\"String\"}]},\"title\":\"this is template2\"}],\"executionContexts\":[{\"compatibleRuntimes\":[\"runtime::H2Runtime\"],\"datasets\":[{\"_type\":\"relationalDatabaseTable\",\"database\":\"CovidDataStore\",\"name\":\"default.DEMOGRAPHICS\",\"schema\":\"default\",\"table\":\"DEMOGRAPHICS\",\"type\":\"H2\"},{\"_type\":\"relationalDatabaseTable\",\"database\":\"CovidDataStore\",\"name\":\"default.COVID_DATA\",\"schema\":\"default\",\"table\":\"COVID_DATA\",\"type\":\"H2\"}],\"defaultRuntime\":\"runtime::H2Runtime\",\"mapping\":\"mapping::CovidDataMapping\",\"mappingModelCoverageAnalysisResult\":{\"mappedEntities\":[{\"path\":\"domain::COVIDData\",\"properties\":[{\"_type\":\"MappedProperty\",\"name\":\"caseType\"},{\"_type\":\"MappedProperty\",\"name\":\"cases\"},{\"_type\":\"MappedProperty\",\"name\":\"date\"},{\"_type\":\"entity\",\"entityPath\":\"domain::Demographics\",\"name\":\"demographics\"},{\"_type\":\"MappedProperty\",\"name\":\"fips\"},{\"_type\":\"MappedProperty\",\"name\":\"id\"},{\"_type\":\"MappedProperty\",\"name\":\"lastReportedFlag\"}]},{\"path\":\"domain::Demographics\",\"properties\":[{\"_type\":\"MappedProperty\",\"name\":\"fips\"},{\"_type\":\"MappedProperty\",\"name\":\"state\"}]}]},\"name\":\"dummyContext\"}],\"model\":{\"_type\":\"data\",\"elements\":[]},\"name\":\"COVIDDataspace\",\"package\":\"domain\",\"path\":\"domain::COVIDDataspace\",\"stereotypes\":[],\"taggedValues\":[],\"title\":\"COVID Sample Data\"}"); + testDataSpaceAnalyticsArtifactGenerationExtension("models/dataSpaceWithTemplateExecutablesWithInlineQuery.pure", "domain::COVIDDataspace", "{\"defaultExecutionContext\":\"dummyContext\",\"description\":\"this is description of this COVIDDataspace\",\"diagrams\":[],\"elementDocs\":[],\"elements\":[],\"executables\":[{\"description\":\"Some more exec description\",\"info\":{\"_type\":\"templateExecutableInfo\",\"executionContextKey\":\"dummyContext\",\"id\":\"1\",\"query\":\"|domain::COVIDData.all()->project([x: domain::COVIDData[1]|$x.cases], ['Cases'])\"},\"result\":{\"_type\":\"tds\",\"columns\":[{\"name\":\"Cases\",\"relationalType\":\"INTEGER\",\"type\":\"Float\"}]},\"title\":\"this is template\"},{\"info\":{\"_type\":\"templateExecutableInfo\",\"executionContextKey\":\"dummyContext\",\"id\":\"2\",\"query\":\"|domain::COVIDData.all()->project([x: domain::COVIDData[1]|$x.fips], ['Fips'])\"},\"result\":{\"_type\":\"tds\",\"columns\":[{\"name\":\"Fips\",\"relationalType\":\"VARCHAR(200)\",\"type\":\"String\"}]},\"title\":\"this is template2\"}],\"executionContexts\":[{\"compatibleRuntimes\":[\"runtime::H2Runtime\"],\"datasets\":[{\"_type\":\"relationalDatabaseTable\",\"database\":\"CovidDataStore\",\"name\":\"default.DEMOGRAPHICS\",\"schema\":\"default\",\"table\":\"DEMOGRAPHICS\",\"type\":\"H2\"},{\"_type\":\"relationalDatabaseTable\",\"database\":\"CovidDataStore\",\"name\":\"default.COVID_DATA\",\"schema\":\"default\",\"table\":\"COVID_DATA\",\"type\":\"H2\"}],\"defaultRuntime\":\"runtime::H2Runtime\",\"mapping\":\"mapping::CovidDataMapping\",\"mappingModelCoverageAnalysisResult\":{\"mappedEntities\":[{\"path\":\"domain::COVIDData\",\"properties\":[{\"_type\":\"MappedProperty\",\"name\":\"caseType\"},{\"_type\":\"MappedProperty\",\"name\":\"cases\"},{\"_type\":\"MappedProperty\",\"name\":\"date\"},{\"_type\":\"entity\",\"entityPath\":\"domain::Demographics\",\"name\":\"demographics\"},{\"_type\":\"MappedProperty\",\"name\":\"fips\"},{\"_type\":\"MappedProperty\",\"name\":\"id\"},{\"_type\":\"MappedProperty\",\"name\":\"lastReportedFlag\"}]},{\"path\":\"domain::Demographics\",\"properties\":[{\"_type\":\"MappedProperty\",\"name\":\"fips\"},{\"_type\":\"MappedProperty\",\"name\":\"state\"}]}]},\"name\":\"dummyContext\"}],\"model\":{\"_type\":\"data\",\"elements\":[]},\"name\":\"COVIDDataspace\",\"package\":\"domain\",\"path\":\"domain::COVIDDataspace\",\"stereotypes\":[],\"taggedValues\":[],\"title\":\"COVID Sample Data\"}"); + } + + @Test + public void testAnalyticsForDataSpaceTemplateWithExecutablesWithFunctionPointer() throws Exception + { + testDataSpaceAnalyticsArtifactGenerationExtension("models/dataSpaceWithTemplateExecutablesWithFunctionPointer.pure", "domain::COVIDDataspace", "{\"defaultExecutionContext\":\"dummyContext\",\"description\":\"this is description of this COVIDDataspace\",\"diagrams\":[],\"elementDocs\":[],\"elements\":[],\"executables\":[{\"description\":\"Some more exec description\",\"info\":{\"_type\":\"templateExecutableInfo\",\"executionContextKey\":\"dummyContext\",\"id\":\"1\",\"query\":\"domain::COVIDData_QueryFunction():TabularDataSet[1]\"},\"result\":{\"_type\":\"tds\",\"columns\":[{\"name\":\"Cases\",\"relationalType\":\"INTEGER\",\"type\":\"Float\"}]},\"title\":\"this is template\"},{\"info\":{\"_type\":\"templateExecutableInfo\",\"executionContextKey\":\"dummyContext\",\"id\":\"2\",\"query\":\"domain::COVIDData_QueryFunction():TabularDataSet[1]\"},\"result\":{\"_type\":\"tds\",\"columns\":[{\"name\":\"Cases\",\"relationalType\":\"INTEGER\",\"type\":\"Float\"}]},\"title\":\"this is template2\"}],\"executionContexts\":[{\"compatibleRuntimes\":[\"runtime::H2Runtime\"],\"datasets\":[{\"_type\":\"relationalDatabaseTable\",\"database\":\"CovidDataStore\",\"name\":\"default.DEMOGRAPHICS\",\"schema\":\"default\",\"table\":\"DEMOGRAPHICS\",\"type\":\"H2\"},{\"_type\":\"relationalDatabaseTable\",\"database\":\"CovidDataStore\",\"name\":\"default.COVID_DATA\",\"schema\":\"default\",\"table\":\"COVID_DATA\",\"type\":\"H2\"}],\"defaultRuntime\":\"runtime::H2Runtime\",\"mapping\":\"mapping::CovidDataMapping\",\"mappingModelCoverageAnalysisResult\":{\"mappedEntities\":[{\"path\":\"domain::COVIDData\",\"properties\":[{\"_type\":\"MappedProperty\",\"name\":\"caseType\"},{\"_type\":\"MappedProperty\",\"name\":\"cases\"},{\"_type\":\"MappedProperty\",\"name\":\"date\"},{\"_type\":\"entity\",\"entityPath\":\"domain::Demographics\",\"name\":\"demographics\"},{\"_type\":\"MappedProperty\",\"name\":\"fips\"},{\"_type\":\"MappedProperty\",\"name\":\"id\"},{\"_type\":\"MappedProperty\",\"name\":\"lastReportedFlag\"}]},{\"path\":\"domain::Demographics\",\"properties\":[{\"_type\":\"MappedProperty\",\"name\":\"fips\"},{\"_type\":\"MappedProperty\",\"name\":\"state\"}]}]},\"name\":\"dummyContext\"}],\"model\":{\"_type\":\"data\",\"elements\":[]},\"name\":\"COVIDDataspace\",\"package\":\"domain\",\"path\":\"domain::COVIDDataspace\",\"stereotypes\":[],\"taggedValues\":[],\"title\":\"COVID Sample Data\"}"); } @Test diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/resources/models/dataSpaceWithTemplateExecutablesWithFunctionPointer.pure b/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/resources/models/dataSpaceWithTemplateExecutablesWithFunctionPointer.pure new file mode 100644 index 00000000000..73f7b9aa6a4 --- /dev/null +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/resources/models/dataSpaceWithTemplateExecutablesWithFunctionPointer.pure @@ -0,0 +1,192 @@ +###DataSpace +DataSpace domain::COVIDDataspace +{ + executionContexts: + [ + { + name: 'dummyContext'; + mapping: mapping::CovidDataMapping; + defaultRuntime: runtime::H2Runtime; + } + ]; + defaultExecutionContext: 'dummyContext'; + title: 'COVID Sample Data'; + description: 'this is description of this COVIDDataspace'; + executables: + [ + { + id: 1; + title: 'this is template'; + description: 'Some more exec description'; + query: domain::COVIDData_QueryFunction():TabularDataSet[1]; + executionContextKey: 'dummyContext'; + }, + { + id: 2; + title: 'this is template2'; + query: domain::COVIDData_QueryFunction():TabularDataSet[1]; + executionContextKey: 'dummyContext'; + } + ]; +} + +###Service +Service service::CovidDataMulti +{ + pattern: '/9566f101-2108-408f-863f-6d7e154dc17a'; + owners: + [ + 'anonymous', + 'akphi' + ]; + documentation: ''; + autoActivateUpdates: true; + execution: Multi + { + query: |domain::COVIDData.all()->project([x|$x.cases], ['Cases']); + key: 'env'; + executions['PROD']: + { + mapping: mapping::CovidDataMapping; + runtime: runtime::H2Runtime; + } + executions['DEV']: + { + mapping: mapping::CovidDataMapping; + runtime: runtime::H2Runtime; + } + } +} + +Service service::CovidDataSingle +{ + pattern: '/9566f101-2108-408f-863f-6d7e154dc17b'; + owners: + [ + 'anonymous', + 'akphi' + ]; + documentation: ''; + autoActivateUpdates: true; + execution: Single + { + query: |domain::COVIDData.all()->project([x|$x.cases], ['Cases']); + mapping: mapping::CovidDataMapping; + runtime: runtime::H2Runtime; + } +} + + +###Relational +Database store::CovidDataStore +( + Table DEMOGRAPHICS + ( + FIPS VARCHAR(200), + STATE VARCHAR(200) + ) + Table COVID_DATA + ( + ID INTEGER PRIMARY KEY, + FIPS VARCHAR(200), + DATE DATE, + CASE_TYPE VARCHAR(200), + CASES INTEGER, + LAST_REPORTED_FLAG BIT + ) + + Join CovidDataDemographicsJoin(DEMOGRAPHICS.FIPS = COVID_DATA.FIPS) +) + + +###Pure +Enum {doc.doc = 'List of known viruses'} domain::Virus +{ + {doc.doc = 'Severe acute respiratory syndrome (SARS) is a viral respiratory disease of zoonotic origin caused by the severe acute respiratory syndrome coronavirus'} SARS, + {doc.doc = 'Coronavirus disease 2019 (COVID-19) is a contagious disease caused by a virus, the severe acute respiratory syndrome coronavirus 2 (SARS-CoV-2)'} COVID19 +} + +Class {doc.doc = 'COVID-19 data demographics consisting of geolocation information'} domain::Demographics +{ + {doc.doc = 'The Federal Information Processing Standard (FIPS) code (FIPS 6-4) uniquely identifies counties and county equivalents in the United States'} fips: String[0..1]; + {doc.doc = 'United States in 2-letter codes format'} state: String[0..1]; +} + +Class {doc.doc = 'COVID-19 data report consisting of case statistics details and basic information on demographics'} domain::COVIDData +{ + id: Integer[1]; + {doc.doc = 'The Federal Information Processing Standard (FIPS) code (FIPS 6-4) uniquely identifies counties and county equivalents in the United States'} fips: String[0..1]; + date: StrictDate[0..1]; + caseType: String[0..1]; + cases: Float[0..1]; + {doc.doc = 'A flag indicating if the similar case data has been reported previously'} lastReportedFlag: Boolean[0..1]; + demographics: domain::Demographics[0..1]; +} + +function domain::COVIDData_QueryFunction(): meta::pure::tds::TabularDataSet[1] +{ + domain::COVIDData.all()->project([x|$x.cases], ['Cases'])->from( + mapping::CovidDataMapping, + runtime::H2Runtime + ) +} + +###Mapping +Mapping mapping::CovidDataMapping +( + domain::Demographics: Relational + { + ~primaryKey + ( + [store::CovidDataStore]DEMOGRAPHICS.FIPS + ) + ~mainTable [store::CovidDataStore]DEMOGRAPHICS + fips: [store::CovidDataStore]DEMOGRAPHICS.FIPS, + state: [store::CovidDataStore]DEMOGRAPHICS.STATE + } + domain::COVIDData: Relational + { + ~primaryKey + ( + [store::CovidDataStore]COVID_DATA.ID + ) + ~mainTable [store::CovidDataStore]COVID_DATA + id: [store::CovidDataStore]COVID_DATA.ID, + fips: [store::CovidDataStore]COVID_DATA.FIPS, + date: [store::CovidDataStore]COVID_DATA.DATE, + caseType: [store::CovidDataStore]COVID_DATA.CASE_TYPE, + cases: [store::CovidDataStore]COVID_DATA.CASES, + lastReportedFlag: [store::CovidDataStore]COVID_DATA.LAST_REPORTED_FLAG, + demographics[domain_Demographics]: [store::CovidDataStore]@CovidDataDemographicsJoin + } +) + +###Connection +RelationalDatabaseConnection runtime::connection::H2Connection +{ + store: store::CovidDataStore; + type: H2; + specification: LocalH2 + { + testDataSetupSqls: [ + 'DROP TABLE IF EXISTS COVID_DATA;\nDROP TABLE IF EXISTS DEMOGRAPHICS;\n\nCREATE TABLE DEMOGRAPHICS(\n FIPS VARCHAR(200) PRIMARY KEY,\n STATE VARCHAR(200)\n);\n\nCREATE TABLE COVID_DATA(\n ID INT PRIMARY KEY,\n FIPS VARCHAR(200),\n DATE DATE,\n CASE_TYPE VARCHAR(200),\n CASES INT,\n LAST_REPORTED_FLAG BIT,\n FOREIGN KEY (FIPS) REFERENCES DEMOGRAPHICS(FIPS)\n);\n\nINSERT INTO DEMOGRAPHICS VALUES(\'1\', \'NY\');\nINSERT INTO DEMOGRAPHICS VALUES(\'2\', \'NJ\');\nINSERT INTO DEMOGRAPHICS VALUES(\'3\', \'CA\');\n\nINSERT INTO COVID_DATA VALUES(1, \'1\', \'2021-04-01\', \'Confirmed\', 405, 0);\nINSERT INTO COVID_DATA VALUES(2, \'2\', \'2021-04-01\', \'Active\', 290, 1);\n' + ]; + }; + auth: DefaultH2; +} + +###Runtime +Runtime runtime::H2Runtime +{ + mappings: + [ + mapping::CovidDataMapping + ]; + connections: + [ + store::CovidDataStore: + [ + connection_1: runtime::connection::H2Connection + ] + ]; +} diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/resources/models/dataSpaceWithTemplateExecutables.pure b/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/resources/models/dataSpaceWithTemplateExecutablesWithInlineQuery.pure similarity index 100% rename from legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/resources/models/dataSpaceWithTemplateExecutables.pure rename to legend-engine-xts-data-space/legend-engine-xt-data-space-generation/src/test/resources/models/dataSpaceWithTemplateExecutablesWithInlineQuery.pure diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataSpaceParserGrammar.g4 b/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataSpaceParserGrammar.g4 index 3f6ea796585..e34d5ca4a17 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataSpaceParserGrammar.g4 +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataSpaceParserGrammar.g4 @@ -154,7 +154,7 @@ executableDescription: DATA_SPACE__DESCRIPTION COLON STRING SEMI_CO ; executablePath: DATA_SPACE_EXECUTABLE COLON qualifiedName SEMI_COLON ; -executableTemplateQuery: DATA_SPACE__TEMPLATE_QUERY COLON combinedExpression SEMI_COLON +executableTemplateQuery: DATA_SPACE__TEMPLATE_QUERY COLON (combinedExpression | functionIdentifier) SEMI_COLON ; executableTemplateQueryId: DATA_SPACE__TEMPLATE_QUERY__ID COLON (VALID_STRING | INTEGER | DECIMAL) SEMI_COLON ; diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/from/DataSpaceParseTreeWalker.java b/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/from/DataSpaceParseTreeWalker.java index f7e3ea62787..82977c4639b 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/from/DataSpaceParseTreeWalker.java +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/from/DataSpaceParseTreeWalker.java @@ -21,6 +21,7 @@ import org.finos.legend.engine.language.pure.grammar.from.domain.DomainParser; import org.finos.legend.engine.language.pure.grammar.from.antlr4.DataSpaceParserGrammar; import org.finos.legend.engine.language.pure.grammar.from.data.embedded.HelperEmbeddedDataGrammarParser; +import org.finos.legend.engine.protocol.pure.v1.model.SourceInformation; import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType; import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementPointer; import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementType; @@ -37,6 +38,8 @@ import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpaceSupportInfo; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpaceTemplateExecutable; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpacePackageableElementExecutable; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpaceTemplateExecutablePointer; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpaceInlineTemplateExecutable; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.StereotypePtr; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.TagPtr; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.TaggedValue; @@ -242,29 +245,50 @@ private DataSpaceExecutable visitDataSpacePackageableElementExecutable(DataSpace private DataSpaceExecutable visitDataSpaceTemplateExecutable(DataSpaceParserGrammar.ExecutableContext ctx) { - DataSpaceTemplateExecutable executable = new DataSpaceTemplateExecutable(); - executable.sourceInformation = this.walkerSourceInformation.getSourceInformation(ctx); + DataSpaceTemplateExecutable executable; + SourceInformation sourceInformation = this.walkerSourceInformation.getSourceInformation(ctx); // ID - DataSpaceParserGrammar.ExecutableTemplateQueryIdContext executableTemplateQueryIdContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.executableTemplateQueryId(), "id", executable.sourceInformation); - executable.id = executableTemplateQueryIdContext.VALID_STRING() != null ? executableTemplateQueryIdContext.VALID_STRING().getText() : executableTemplateQueryIdContext.DECIMAL() != null ? executableTemplateQueryIdContext.DECIMAL().getText() : executableTemplateQueryIdContext.INTEGER().getText(); + DataSpaceParserGrammar.ExecutableTemplateQueryIdContext executableTemplateQueryIdContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.executableTemplateQueryId(), "id", sourceInformation); + String id = executableTemplateQueryIdContext.VALID_STRING() != null ? executableTemplateQueryIdContext.VALID_STRING().getText() : executableTemplateQueryIdContext.DECIMAL() != null ? executableTemplateQueryIdContext.DECIMAL().getText() : executableTemplateQueryIdContext.INTEGER().getText(); // Title - DataSpaceParserGrammar.ExecutableTitleContext executableTitleContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.executableTitle(), "title", executable.sourceInformation); - executable.title = PureGrammarParserUtility.fromGrammarString(executableTitleContext.STRING().getText(), true); + DataSpaceParserGrammar.ExecutableTitleContext executableTitleContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.executableTitle(), "title", sourceInformation); + String title = PureGrammarParserUtility.fromGrammarString(executableTitleContext.STRING().getText(), true); // Description (optional) - DataSpaceParserGrammar.ExecutableDescriptionContext descriptionContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.executableDescription(), "description", executable.sourceInformation); - executable.description = descriptionContext != null ? PureGrammarParserUtility.fromGrammarString(descriptionContext.STRING().getText(), true) : null; - - // query - DataSpaceParserGrammar.ExecutableTemplateQueryContext queryContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.executableTemplateQuery(), "query", executable.sourceInformation); - executable.query = visitLambda(queryContext.combinedExpression()); + DataSpaceParserGrammar.ExecutableDescriptionContext descriptionContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.executableDescription(), "description", sourceInformation); + String description = descriptionContext != null ? PureGrammarParserUtility.fromGrammarString(descriptionContext.STRING().getText(), true) : null; // executionContextKey - DataSpaceParserGrammar.ExecutableExecutionContextKeyContext executionContextKeyContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.executableExecutionContextKey(), "executionContextKey", executable.sourceInformation); - executable.executionContextKey = executionContextKeyContext != null ? PureGrammarParserUtility.fromGrammarString(executionContextKeyContext.STRING().getText(), true) : null; + DataSpaceParserGrammar.ExecutableExecutionContextKeyContext executionContextKeyContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.executableExecutionContextKey(), "executionContextKey", sourceInformation); + String executionContextKey = executionContextKeyContext != null ? PureGrammarParserUtility.fromGrammarString(executionContextKeyContext.STRING().getText(), true) : null; + // query could either be type of PackageableElementPointer or Lambda + DataSpaceParserGrammar.ExecutableTemplateQueryContext queryContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.executableTemplateQuery(), "query", sourceInformation); + if (queryContext.functionIdentifier() != null) + { + DataSpaceParserGrammar.ExecutableTemplateQueryContext functionContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.executableTemplateQuery(), "query", sourceInformation); + executable = new DataSpaceTemplateExecutablePointer(id, executionContextKey, new PackageableElementPointer( + PackageableElementType.FUNCTION, + functionContext.functionIdentifier().getText(), + walkerSourceInformation.getSourceInformation(functionContext.functionIdentifier()) + )); + executable.description = description; + executable.title = title; + executable.sourceInformation = sourceInformation; + } + else if (queryContext.combinedExpression() != null) + { + executable = new DataSpaceInlineTemplateExecutable(id, executionContextKey, visitLambda(queryContext.combinedExpression())); + executable.description = description; + executable.title = title; + executable.sourceInformation = sourceInformation; + } + else + { + throw new UnsupportedOperationException("Can't parse unsupported query in curated dataspace template query"); + } return executable; } diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/to/DataSpaceGrammarComposerExtension.java b/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/to/DataSpaceGrammarComposerExtension.java index a40f1c260a8..96c8189d878 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/to/DataSpaceGrammarComposerExtension.java +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/src/main/java/org/finos/legend/engine/language/pure/grammar/to/DataSpaceGrammarComposerExtension.java @@ -162,7 +162,7 @@ private static String renderDataspaceTemplateExecutable(DataSpaceTemplateExecuta (getTabString(3) + "id: " + executable.id + ";\n") + (getTabString(3) + "title: " + convertString(executable.title, true) + ";\n") + (executable.description != null ? (getTabString(3) + "description: " + convertString(executable.description, true) + ";\n") : "") + - getTabString(3) + "query: " + executable.query.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance(context).withIndentation(getTabSize(3)).build()) + ";\n" + + getTabString(3) + "query: " + (executable instanceof DataSpaceInlineTemplateExecutable ? (((DataSpaceInlineTemplateExecutable) executable).query).accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance(context).withIndentation(getTabSize(3)).build()) : (((DataSpaceTemplateExecutablePointer) executable).query).path) + ";\n" + (executable.executionContextKey != null ? getTabString(3) + "executionContextKey: " + convertString(executable.executionContextKey, true) + ";\n" : "") + getTabString(2) + "}"; } diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceInlineTemplateExecutable.java b/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceInlineTemplateExecutable.java new file mode 100644 index 00000000000..fe16ec8cff8 --- /dev/null +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceInlineTemplateExecutable.java @@ -0,0 +1,34 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace; + +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; + +@Deprecated +public class DataSpaceInlineTemplateExecutable extends DataSpaceTemplateExecutable +{ + public Lambda query; + + public DataSpaceInlineTemplateExecutable() + { + + } + + public DataSpaceInlineTemplateExecutable(String id, String executionContextKey, Lambda query) + { + super(id, executionContextKey); + this.query = query; + } +} diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceTemplateExecutable.java b/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceTemplateExecutable.java index 0c9c60f91ef..9125f4b9965 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceTemplateExecutable.java +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceTemplateExecutable.java @@ -14,11 +14,29 @@ package org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace; -import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type", defaultImpl = DataSpaceTemplateExecutablePointer.class) +@JsonSubTypes({ + @JsonSubTypes.Type(value = DataSpaceTemplateExecutablePointer.class, name = "dataSpaceTemplateExecutablePointer"), + @JsonSubTypes.Type(value = DataSpaceInlineTemplateExecutable.class, name = "dataSpaceTemplateExecutable"), +}) +@JsonIgnoreProperties(ignoreUnknown = true) public class DataSpaceTemplateExecutable extends DataSpaceExecutable { - public Lambda query; public String id; public String executionContextKey; + + public DataSpaceTemplateExecutable() + { + + } + + public DataSpaceTemplateExecutable(String id, String executionContextKey) + { + this.id = id; + this.executionContextKey = executionContextKey; + } } diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceTemplateExecutablePointer.java b/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceTemplateExecutablePointer.java new file mode 100644 index 00000000000..249ca301d82 --- /dev/null +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/dataSpace/DataSpaceTemplateExecutablePointer.java @@ -0,0 +1,33 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace; + +import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementPointer; + +public class DataSpaceTemplateExecutablePointer extends DataSpaceTemplateExecutable +{ + public PackageableElementPointer query; + + public DataSpaceTemplateExecutablePointer() + { + + } + + public DataSpaceTemplateExecutablePointer(String id, String executionContextKey, PackageableElementPointer query) + { + super(id,executionContextKey); + this.query = query; + } +} \ No newline at end of file