Fix type matching for input and return types (finos#3370)

goldmansachs · Jan 25, 2025 · 8a3c7de · 8a3c7de
1 parent 9a22234
commit 8a3c7de
Show file tree

Hide file tree

Showing 3 changed files with 128 additions and 13 deletions.
diff --git a/...ne-xt-hostedService-pure/src/main/resources/core_hostedservice/generation/generation.pure b/...ne-xt-hostedService-pure/src/main/resources/core_hostedservice/generation/generation.pure
@@ -43,6 +43,10 @@ function meta::external::function::activator::hostedService::validator::validate
   let params  = $h.function->functionType().parameters->evaluateAndDeactivate();
   let oneParams  = $params->filter(p|$p->deactivate().multiplicity == PureOne);
   let nonOneParams  = $params->filter(p|!$p->deactivate().multiplicity == PureOne);
+
+  //all [1] params names should not contain underscores if they're in pattern
+  $oneParams->map(p| assert(if($pattern->contains('{'+$p.name+'}'), | !$p.name->contains('_'), |true), 'Service pattern violation. Parameter "'+$p.name+'"" contains underscores (_).'));
+
   //all [1] params in pattern TODO
   //$oneParams->map(p| assert($pattern->contains('{'+$p.name+'}'), 'Parameter "'+$p.name+'"" with multiplicty [1] must be in service pattern.'));
   //all non [1] params not in pattern

diff --git a/...gine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure b/...gine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure
@@ -72,33 +72,49 @@ function  meta::external::function::activator::snowflakeApp::generation::generat
 {  
   let params = $f->functionType().parameters->evaluateAndDeactivate();   
   assertEmpty($params->filter(p | $p.multiplicity != PureOne), 'not implemented, only multiplicity PureOne for snowflake app params is supported: ' + $params->filter(p | $p.multiplicity != PureOne)->map(p|$p.name)->joinStrings(', '));
-  let typeMap = typeMap();
+  let typeMap = inputTypeMap();
 
   '(' + $params->map(p|if($includeParamName,|'"' + $p.name + '" ',|'') + if($p.genericType.rawType->toOne()->instanceOf(Enumeration), | 'VARCHAR', | $typeMap->get($p.genericType.rawType->toOne())->toOne('Relational type missing for type: ' + $p.genericType.rawType->toOne()->toString()))
   )->joinStrings(',') + ')';    
 }
 
 
-function <<access.private>>  meta::external::function::activator::snowflakeApp::generation::typeMap():Map<PrimitiveType, String>[1]
+function <<access.private>>  meta::external::function::activator::snowflakeApp::generation::inputTypeMap():Map<PrimitiveType, String>[1]
 {
   [
-  pair(Integer, 'INTEGER'),
-  pair(Float, 'FLOAT'),
-  pair(Number, 'NUMBER'),
-  pair(String, 'VARCHAR'),
-  pair(Date, 'DATE'),
-  pair(DateTime, 'TIMESTAMP'),
-  pair(StrictDate, 'DATE'),
-  pair(Boolean, 'BOOLEAN'),
-  pair(Decimal, 'DECIMAL')
+    pair(Integer, 'INTEGER'),
+    pair(Float, 'FLOAT'),
+    pair(Number, 'NUMBER'),
+    pair(String, 'VARCHAR'),
+    pair(Date, 'DATE'),
+    pair(DateTime, 'TIMESTAMP'),
+    pair(StrictDate, 'DATE'),
+    pair(Boolean, 'BOOLEAN'),
+    pair(Decimal, 'DECIMAL')
+  ]->newMap()
+}
+
+function <<access.private>>  meta::external::function::activator::snowflakeApp::generation::returnTypeMap():Map<PrimitiveType, Function<{TDSColumn[1] ->String[1]}>>[1]
+{
+  [
+    pair(Integer, {col:TDSColumn[1]|'INTEGER'}),
+    pair(Float, {col:TDSColumn[1]| if($col.sourceDataType->isNotEmpty(),| $col->meta::external::function::activator::snowflakeApp::generation::matchDBType(),|'DECIMAL');}),
+    pair(Number, {col:TDSColumn[1]|'NUMBER'}),
+    pair(String, {col:TDSColumn[1]|'VARCHAR'}),
+    pair(Date, {col:TDSColumn[1]|'DATE'}),
+    pair(DateTime, {col:TDSColumn[1]|'TIMESTAMP'}),
+    pair(StrictDate, {col:TDSColumn[1]|'DATE'}),
+    pair(Boolean, {col:TDSColumn[1]|'BOOLEAN'}),
+    pair(Decimal, {col:TDSColumn[1]| fail();if($col.sourceDataType->isNotEmpty(),| $col->meta::external::function::activator::snowflakeApp::generation::matchDBType(),|'DECIMAL');})
   ]->newMap()
 }
 
 function <<access.private>>  meta::external::function::activator::snowflakeApp::generation::pureTypeToSnowflakeTypeMatcher(c:meta::pure::tds::TDSColumn[1]):String[1]
 {
-  let tdsTypeToRelationalTypeMap = typeMap();
+  let tdsTypeToRelationalTypeMap = returnTypeMap();
   if($c.type->isNotEmpty(),
-    | $tdsTypeToRelationalTypeMap->get($c.type->toOne())->toOne('Relational type missing for type: '+ $c.type->toOne()->toString()),
+    | 
+     $tdsTypeToRelationalTypeMap->get($c.type->toOne())->toOne('Relational type missing for type: '+ $c.type->toOne()->toString())->eval($c);,
     | $c.sourceDataType->toOne('Source Data Type Missing for column: '+ +$c.name)
       ->match(
         [
@@ -107,6 +123,17 @@ function <<access.private>>  meta::external::function::activator::snowflakeApp::
           all:meta::relational::metamodel::datatype::DataType[1]| $all->meta::relational::metamodel::datatype::dataTypeToSqlText()
         ])
   );
+
+}
+
+function <<access.private>>  meta::external::function::activator::snowflakeApp::generation::matchDBType(c:meta::pure::tds::TDSColumn[1]):String[1]
+{
+  $c.sourceDataType->toOne()->match(
+        [
+          v:meta::relational::metamodel::datatype::Varchar[1] | 'VARCHAR',
+          b:meta::relational::metamodel::datatype::Bit[1]|'BOOLEAN',
+          all:meta::relational::metamodel::datatype::DataType[1]| $all->meta::relational::metamodel::datatype::dataTypeToSqlText()
+        ]);
 }
 
 function meta::external::function::activator::snowflakeApp::generation::computeLineage(s: meta::external::function::activator::snowflakeApp::SnowflakeApp[1], extensions:meta::pure::extension::Extension[*]): String[1]

diff --git a/...wflakeApp-pure/src/main/resources/core_snowflakeapp/tests/testSnowflakeAppGeneration.pure b/...wflakeApp-pure/src/main/resources/core_snowflakeapp/tests/testSnowflakeAppGeneration.pure
@@ -71,6 +71,18 @@ function <<test.Test>> meta::external::function::activator::snowflakeApp::tests:
   assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionEnumParam_Gender_1__TabularDataSet_1_, $expected);
 }
 
+function <<test.Test>> meta::external::function::activator::snowflakeApp::tests::testInputDataTypes():Boolean[1]
+{
+  let expected = 'CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.APP1("str" VARCHAR,"int" INTEGER,"bool" BOOLEAN,"dt" TIMESTAMP) RETURNS TABLE ("BIG INT" INTEGER,"BOOLEAN" BOOLEAN,"DATE" DATE,"DATE TIME" TIMESTAMP,"DECIMAL" DECIMAL(20, 8),"INTEGER" INTEGER,"STRING" VARCHAR) LANGUAGE SQL AS $$ select "root".bigInt as "Big Int", case when "root".boolean is null then null else case when "root".boolean = 1 then true else false end end as "Boolean", "root"."DATE" as "Date", "root".dateTime as "Date Time", "root".decimal as "Decimal", "root".integer as "Integer", "root".string as "String" from dataTypeTable as "root" $$;';
+  assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::testReturnType_String_1__Integer_1__Boolean_1__DateTime_1__TabularDataSet_1_, $expected);
+}
+
+function <<test.Test>> meta::external::function::activator::snowflakeApp::tests::testReturnDataTypes():Boolean[1]
+{
+  let expected = 'CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.APP1() RETURNS TABLE ("BIG INT" INTEGER,"BOOLEAN" BOOLEAN,"DATE" DATE,"DATE TIME" TIMESTAMP,"DECIMAL" DECIMAL(20, 8),"INTEGER" INTEGER,"STRING" VARCHAR) LANGUAGE SQL AS $$ select "root".bigInt as "Big Int", case when "root".boolean is null then null else case when "root".boolean = 1 then true else false end end as "Boolean", "root"."DATE" as "Date", "root".dateTime as "Date Time", "root".decimal as "Decimal", "root".integer as "Integer", "root".string as "String" from dataTypeTable as "root" $$;';
+  assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::testDataType__TabularDataSet_1_, $expected);
+}
+
 //function <<test.ToFix>> meta::external::function::activator::snowflakeApp::tests::testUdtfOnUdtf():Any[*]
 //{
 // let result  = meta::legend::compileLegendGrammar(readFile('/core_relational_snowflake/relational/tests/tabularFunctionModel.txt')->toOne());
@@ -154,3 +166,75 @@ function meta::external::function::activator::snowflakeApp::tests::assertSnowfla
   assertEquals($expected, $generatedQuery);
 }
 
+function meta::external::function::activator::snowflakeApp::tests::testDataType():TabularDataSet[1]
+{
+  meta::external::function::activator::snowflakeApp::tests::DataTypeClass.all()->project(
+    [col(x|$x.bigInt, 'Big Int') ,
+    col(x|$x.boolean,'Boolean'),  
+    col(x|$x.date,'Date') ,
+    col(x|$x.dateTime, 'Date Time'), 
+    col(x|$x.decimal, 'Decimal') ,
+    col(x|$x.integer, 'Integer') , 
+    col(x|$x.string, 'String') ]
+  )
+  ->from(meta::external::function::activator::snowflakeApp::tests::DatatypeDBMapping, testRuntimeWithTimeZone(dbInc));
+}
+
+function meta::external::function::activator::snowflakeApp::tests::testReturnType(str:String[1], int:Integer[1], bool:Boolean[1], dt:DateTime[1]):TabularDataSet[1]
+{
+  meta::external::function::activator::snowflakeApp::tests::DataTypeClass.all()->project(
+    [col(x|$x.bigInt, 'Big Int') ,
+    col(x|$x.boolean,'Boolean'),  
+    col(x|$x.date,'Date') ,
+    col(x|$x.dateTime, 'Date Time'), 
+    col(x|$x.decimal, 'Decimal') ,
+    col(x|$x.integer, 'Integer') , 
+    col(x|$x.string, 'String') ]
+
+)->from(meta::external::function::activator::snowflakeApp::tests::DatatypeDBMapping, testRuntimeWithTimeZone(dbInc));
+}
+
+Class  meta::external::function::activator::snowflakeApp::tests::DataTypeClass
+{
+  string: String[0..1];
+  dateTime: DateTime[0..1];
+  decimal: Float[0..1];
+  boolean: Boolean[0..1];
+  bigInt: Integer[0..1];
+  integer: Integer[0..1];
+  date: StrictDate[0..1];
+  semiStructured: String[0..1];
+}
+
+###Mapping
+Mapping meta::external::function::activator::snowflakeApp::tests::DatatypeDBMapping
+(
+  *meta::external::function::activator::snowflakeApp::tests::DataTypeClass: Relational
+  {
+
+    ~mainTable [meta::external::function::activator::snowflakeApp::tests::DatatypeDB]dataTypeTable
+    string: [meta::external::function::activator::snowflakeApp::tests::DatatypeDB]dataTypeTable.string,
+    dateTime: [meta::external::function::activator::snowflakeApp::tests::DatatypeDB]dataTypeTable.dateTime,
+    decimal: [meta::external::function::activator::snowflakeApp::tests::DatatypeDB]dataTypeTable.decimal,
+    boolean: case(isNull([meta::external::function::activator::snowflakeApp::tests::DatatypeDB]dataTypeTable.boolean), sqlNull(), case(equal([meta::external::function::activator::snowflakeApp::tests::DatatypeDB]dataTypeTable.boolean, 1), sqlTrue(), sqlFalse())),
+    bigInt: [meta::external::function::activator::snowflakeApp::tests::DatatypeDB]dataTypeTable.bigInt,
+    integer: [meta::external::function::activator::snowflakeApp::tests::DatatypeDB]dataTypeTable.integer,
+    date: [meta::external::function::activator::snowflakeApp::tests::DatatypeDB]dataTypeTable.date
+    // semiStructured: [meta::external::function::activator::snowflakeApp::tests::DatatypeDB]dataTypeTable.semiStructured
+  }
+)
+
+###Relational
+Database meta::external::function::activator::snowflakeApp::tests::DatatypeDB
+(
+   Table dataTypeTable
+  (
+    string VARCHAR(250) PRIMARY KEY,
+    dateTime TIMESTAMP,
+    decimal DECIMAL(20, 8),
+    boolean BIT,
+    bigInt BIGINT,
+    integer INTEGER,
+    date DATE
+      )
+)