Clean up User-defined type related parser flag (#2697)

* Remove `AllowParseAsTypeLiteral, AllowTypeLiteral` parser flag. * AsType, IsType and ParseJSON overloads are part of Builtin Functions * Add constant for Type Keyword. * Add type arg functions to Built-in * Refactor test
microsoft · Oct 22, 2024 · f473c48 · f473c48
1 parent 2434fd3
commit f473c48
Show file tree

Hide file tree

Showing 23 changed files with 113 additions and 144 deletions.
diff --git a/releasenotes/releasenotes-1.3.0-rc.md b/releasenotes/releasenotes-1.3.0-rc.md
@@ -25,6 +25,11 @@
 `Patch(DS, table_of_rows_with_updates)`\
 `Patch(Record, Updates1, Updates2,…)`
 
+  - ParseJSON, IsType, AsType (https://github.com/microsoft/Power-Fx/pull/2569): ParseJSON, IsType, AsType functions now supports in-lined and user-defined types as argument.\
+  `ParseJSON(Text, Type)`\
+  `IsType(UntypedObject, Type)`\
+  `AsType(UntypedObject, Type)` 
+
 ## Other:  
   - Untyped object
     - Read a field from an untyped object by index (https://github.com/microsoft/Power-Fx/pull/2555):  

diff --git a/src/libraries/Microsoft.PowerFx.Core/Localization/ExpressionLocalizationHelper.cs b/src/libraries/Microsoft.PowerFx.Core/Localization/ExpressionLocalizationHelper.cs
@@ -22,7 +22,7 @@ internal static string ConvertExpression(string expressionText, RecordType param
 
             var formula = new Formula(expressionText, toDisplay ? CultureInfo.InvariantCulture : options?.Culture ?? CultureInfo.InvariantCulture);
 
-            formula.EnsureParsed(options.GetParserFlags());
+            formula.EnsureParsed(options.GetParserFlags(), flags);
 
             var binding = TexlBinding.Run(
                 binderGlue,

diff --git a/src/libraries/Microsoft.PowerFx.Core/Parser/ParserOptions.cs b/src/libraries/Microsoft.PowerFx.Core/Parser/ParserOptions.cs
@@ -47,11 +47,6 @@ public class ParserOptions
         /// </summary>
         public int MaxExpressionLength { get; set; }
 
-        /// <summary>
-        /// Flag for parse type literals.
-        /// </summary>
-        internal bool AllowParseAsTypeLiteral { get; set; }
-
         /// <summary>
         /// Allow parsing of attributes on user definitions
         /// This is an early prototype, and so is internal.
@@ -88,7 +83,6 @@ internal ParseResult Parse(string script, Features features)
                         (NumberIsFloat ? TexlParser.Flags.NumberIsFloat : 0) |
                         (DisableReservedKeywords ? TexlParser.Flags.DisableReservedKeywords : 0) |
                         (TextFirst ? TexlParser.Flags.TextFirst : 0) |
-                        (AllowParseAsTypeLiteral ? TexlParser.Flags.AllowTypeLiteral : 0) |
                         (features.PowerFxV1CompatibilityRules ? TexlParser.Flags.PFxV1 : 0);
 
             var result = TexlParser.ParseScript(script, features, Culture, flags);

diff --git a/src/libraries/Microsoft.PowerFx.Core/Parser/TexlParser.cs b/src/libraries/Microsoft.PowerFx.Core/Parser/TexlParser.cs
@@ -34,9 +34,6 @@ public enum Flags
             // When specified, allows reserved keywords to be used as identifiers.
             DisableReservedKeywords = 1 << 3,
 
-            // When specified, allows type literals to be parsed.
-            AllowTypeLiteral = 1 << 4,
-
             // Text first.  Implemented entirely in the Lexer.
             TextFirst = 1 << 5,
 
@@ -90,16 +87,16 @@ private TexlParser(IReadOnlyList<Token> tokens, Flags flags, Features features =
         /// </summary>
         /// <param name="script">Script to be parsed.</param>
         /// <param name="parserOptions">Options for parsing an expression.</param>
+        /// <param name="features">Power Fx feature flags.</param>
         /// <returns><see cref="ParseUserDefinitionResult"/>.</returns>
-        public static ParseUserDefinitionResult ParseUserDefinitionScript(string script, ParserOptions parserOptions)
+        public static ParseUserDefinitionResult ParseUserDefinitionScript(string script, ParserOptions parserOptions, Features features = null)
         {
             Contracts.AssertValue(parserOptions);
             var flags = Flags.NamedFormulas |
                 (parserOptions.NumberIsFloat ? Flags.NumberIsFloat : 0) |
-                (parserOptions.AllowParseAsTypeLiteral ? Flags.AllowTypeLiteral : 0) |
                 (parserOptions.AllowAttributes ? Flags.AllowAttributes : 0);
             var formulaTokens = TokenizeScript(script, parserOptions.Culture, flags);
-            var parser = new TexlParser(formulaTokens, flags);
+            var parser = new TexlParser(formulaTokens, flags, features);
 
             return parser.ParseUDFsAndNamedFormulas(script, parserOptions);
         }
@@ -300,7 +297,7 @@ private ParseUserDefinitionResult ParseUDFsAndNamedFormulas(string script, Parse
                     continue;
                 }
 
-                if (_curs.TidCur == TokKind.ColonEqual && _flagsMode.Peek().HasFlag(Flags.AllowTypeLiteral))
+                if (_curs.TidCur == TokKind.ColonEqual && _features.IsUserDefinedTypesEnabled)
                 {
                     var declaration = script.Substring(declarationStart, _curs.TokCur.Span.Min - declarationStart);
                     _curs.TokMove();
@@ -1268,7 +1265,7 @@ private TexlNode ParseOperand()
 
                     if (AfterSpaceTokenId() == TokKind.ParenOpen)
                     {
-                        if (ident.Token.As<IdentToken>().Name.Value == "Type" && _flagsMode.Peek().HasFlag(Flags.AllowTypeLiteral))
+                        if (ident.Token.As<IdentToken>().Name.Value == LanguageConstants.TypeLiteralInvariantName && _features.IsUserDefinedTypesEnabled)
                         {
                             var typeLiteralNode = ParseTypeLiteral();
 
@@ -2048,11 +2045,13 @@ internal static string GetTokString(TokKind kind)
         /// </summary>
         /// <param name="text">Expression text to format.</param>
         /// <param name="flags">Optional flags to customize the behavior of underlying lexer and parser. By default, expression chaining is enabled.</param>
+        /// <param name="features">Power Fx features.</param>
         /// <returns>Formatted expression text.</returns>
-        public static string Format(string text, Flags flags = Flags.EnableExpressionChaining)
+        public static string Format(string text, Flags flags = Flags.EnableExpressionChaining, Features features = null)
         {
             var result = ParseScript(
                 text,
+                features ?? Features.None,
                 flags: flags);
 
             // Can't pretty print a script with errors.
@@ -2064,11 +2063,12 @@ public static string Format(string text, Flags flags = Flags.EnableExpressionCha
             return PrettyPrintVisitor.Format(result.Root, result.Before, result.After, text);
         }
 
-        public static string FormatUserDefinitions(string text, ParserOptions options)
+        public static string FormatUserDefinitions(string text, ParserOptions options, Features features = null)
         {
             var result = ParseUserDefinitionScript(
                 text,
-                options);
+                options,
+                features ?? Features.None);
 
             // Can't pretty print a script with errors.
             if (result.HasErrors)

diff --git a/src/libraries/Microsoft.PowerFx.Core/Public/DefinitionsCheckResult.cs b/src/libraries/Microsoft.PowerFx.Core/Public/DefinitionsCheckResult.cs
@@ -40,6 +40,8 @@ public class DefinitionsCheckResult : IOperationStatus
 
         private ParseUserDefinitionResult _parse;
 
+        private readonly Features _features;
+
         // Local symboltable to store new symbols in a given script and use in binding.
         private readonly SymbolTable _localSymbolTable;
 
@@ -49,9 +51,15 @@ public class DefinitionsCheckResult : IOperationStatus
         // All errors accumulated. 
         private readonly List<ExpressionError> _errors = new List<ExpressionError>();
 
-        public DefinitionsCheckResult()
+        public DefinitionsCheckResult() 
+            : this(Features.PowerFxV1) 
+        { 
+        }
+
+        public DefinitionsCheckResult(Features features = null)
         {
             _localSymbolTable = new SymbolTable { DebugName = "LocalUserDefinitions" };
+            _features = features ?? Features.PowerFxV1;
         }
 
         internal DefinitionsCheckResult SetBindingInfo(ReadOnlySymbolTable symbols)
@@ -93,7 +101,7 @@ internal ParseUserDefinitionResult ApplyParse()
 
             if (_parse == null)
             {
-                _parse = UserDefinitions.Parse(_definitions, _parserOptions);
+                _parse = UserDefinitions.Parse(_definitions, _parserOptions, _features);
 
                 if (_parse.HasErrors)
                 {

diff --git a/src/libraries/Microsoft.PowerFx.Core/Syntax/Formula.cs b/src/libraries/Microsoft.PowerFx.Core/Syntax/Formula.cs
@@ -70,13 +70,13 @@ private void AssertValid()
         // True if the formula has already been parsed.
         public bool IsParsed => ParseTree != null;
 
-        public bool EnsureParsed(TexlParser.Flags flags)
+        public bool EnsureParsed(TexlParser.Flags flags, Features features = null)
         {
             AssertValid();
 
             if (ParseTree == null)
             {
-                var result = TexlParser.ParseScript(Script, loc: Loc, flags: flags);
+                var result = TexlParser.ParseScript(Script, features ?? Features.None, culture: Loc, flags: flags);
                 ApplyParse(result);
             }
 

diff --git a/src/libraries/Microsoft.PowerFx.Core/Syntax/TexlPretty.cs b/src/libraries/Microsoft.PowerFx.Core/Syntax/TexlPretty.cs
@@ -412,7 +412,7 @@ public override LazyList<string> Visit(TypeLiteralNode node, Precedence parentPr
 
             result = result
                 .With(
-                    "Type",
+                    LanguageConstants.TypeLiteralInvariantName,
                     TexlLexer.PunctuatorParenOpen)
                 .With(node.TypeRoot.Accept(this, Precedence.Atomic))
                 .With(TexlLexer.PunctuatorParenClose);
@@ -466,9 +466,7 @@ private string SpacedOper(string op)
 
     internal sealed class PrettyPrintVisitor : TexlFunctionalVisitor<LazyList<string>, PrettyPrintVisitor.Context>
     {
-        private readonly string _script;
-
-        public const string TypeInvariantFunctionName = "Type";
+        private readonly string _script;
 
         private static readonly Dictionary<BinaryOp, Precedence> BinaryPrecedence =
             new Dictionary<BinaryOp, Precedence>()
@@ -542,7 +540,7 @@ public static string FormatUserDefinitions(ParseUserDefinitionResult result, str
                     case UserDefinitionType.DefinedType:
                         var type = result.DefinedTypes.First(type => type.Ident == name);
 
-                        definitions.Add(declaration + $" = {TypeInvariantFunctionName}(" + string.Concat(visitor.CommentsOf(before).With(type.Type.Accept(visitor, new Context(0)).With(visitor.CommentsOf(after)))) + ")");
+                        definitions.Add(declaration + $" = {LanguageConstants.TypeLiteralInvariantName}(" + string.Concat(visitor.CommentsOf(before).With(type.Type.Accept(visitor, new Context(0)).With(visitor.CommentsOf(after)))) + ")");
                         break;
                     default:
                         continue;

diff --git a/src/libraries/Microsoft.PowerFx.Core/Syntax/UserDefinitions.cs b/src/libraries/Microsoft.PowerFx.Core/Syntax/UserDefinitions.cs
@@ -44,14 +44,15 @@ private UserDefinitions(string script, ParserOptions parserOptions, Features fea
         /// </summary>
         /// <param name="script">Script with named formulas, user-defined functions and user-defined types.</param>
         /// <param name="parserOptions">Options for parsing an expression.</param>
+        /// <param name="features">Power Fx feature flags.</param>
         /// <returns><see cref="ParseUserDefinitionResult"/>.</returns>
-        public static ParseUserDefinitionResult Parse(string script, ParserOptions parserOptions)
+        public static ParseUserDefinitionResult Parse(string script, ParserOptions parserOptions, Features features = null)
         {
-            var parseResult = TexlParser.ParseUserDefinitionScript(script, parserOptions);
+            var parseResult = TexlParser.ParseUserDefinitionScript(script, parserOptions, features);
 
             if (parserOptions.AllowAttributes)
             {
-                var userDefinitions = new UserDefinitions(script, parserOptions);
+                var userDefinitions = new UserDefinitions(script, parserOptions, features);
                 parseResult = userDefinitions.ProcessPartialAttributes(parseResult);
             }
 

diff --git a/src/libraries/Microsoft.PowerFx.Core/Texl/BuiltinFunctionsCore.cs b/src/libraries/Microsoft.PowerFx.Core/Texl/BuiltinFunctionsCore.cs
@@ -40,7 +40,8 @@ internal class BuiltinFunctionsCore
         public static readonly TexlFunction And = _library.Add(new VariadicLogicalFunction(isAnd: true));
         public static readonly TexlFunction Asin = _library.Add(new AsinFunction());
         public static readonly TexlFunction AsinT = _library.Add(new AsinTableFunction());
-        public static readonly TexlFunction AsType = _library.Add(new AsTypeFunction());
+        public static readonly TexlFunction AsType = _library.Add(new AsTypeFunction());
+        public static readonly TexlFunction AsType_UO = _library.Add(new AsTypeFunction_UO());
         public static readonly TexlFunction Atan = _library.Add(new AtanFunction());
         public static readonly TexlFunction Atan2 = _library.Add(new Atan2Function());
         public static readonly TexlFunction AtanT = _library.Add(new AtanTableFunction());
@@ -132,7 +133,8 @@ internal class BuiltinFunctionsCore
         public static readonly TexlFunction IsError = _library.Add(new IsErrorFunction());
         public static readonly TexlFunction IsNumeric = _library.Add(new IsNumericFunction());
         public static readonly TexlFunction ISOWeekNum = _library.Add(new ISOWeekNumFunction());
-        public static readonly TexlFunction IsToday = _library.Add(new IsTodayFunction());
+        public static readonly TexlFunction IsToday = _library.Add(new IsTodayFunction());
+        public static readonly TexlFunction IsType_UO = _library.Add(new IsTypeFunction_UO());
         public static readonly TexlFunction Language = _library.Add(new LanguageFunction());
         public static readonly TexlFunction Last = _library.Add(new FirstLastFunction(isFirst: false));
         public static readonly TexlFunction Last_UO = _library.Add(new FirstLastFunction_UO(isFirst: false));
@@ -231,6 +233,7 @@ internal class BuiltinFunctionsCore
         public static readonly TexlFunction TrimT = _library.Add(new TrimTFunction());
         public static readonly TexlFunction Trunc = _library.Add(new TruncFunction());
         public static readonly TexlFunction TruncT = _library.Add(new TruncTableFunction());
+        public static readonly TexlFunction TypedParseJSON = _library.Add(new TypedParseJSONFunction());
         public static readonly TexlFunction UniChar = _library.Add(new UniCharFunction());
         public static readonly TexlFunction UniCharT = _library.Add(new UniCharTFunction());
         public static readonly TexlFunction Upper = _library.Add(new LowerUpperFunction(isLower: false));
@@ -260,10 +263,7 @@ internal class BuiltinFunctionsCore
         public static readonly TexlFunction UTCToday = _featureGateFunctions.Add(new UTCTodayFunction());
         public static readonly TexlFunction BooleanL = _featureGateFunctions.Add(new BooleanLFunction());
         public static readonly TexlFunction BooleanL_T = _featureGateFunctions.Add(new BooleanLFunction_T());
-        public static readonly TexlFunction Summarize = _featureGateFunctions.Add(new SummarizeFunction());
-        public static readonly TexlFunction AsType_UO = _featureGateFunctions.Add(new AsTypeFunction_UO());
-        public static readonly TexlFunction IsType_UO = _featureGateFunctions.Add(new IsTypeFunction_UO());
-        public static readonly TexlFunction TypedParseJSON = _featureGateFunctions.Add(new TypedParseJSONFunction());
+        public static readonly TexlFunction Summarize = _featureGateFunctions.Add(new SummarizeFunction());
 
         // Slow API, only use for backward compatibility
 #pragma warning disable CS0618 // Type or member is obsolete        

diff --git a/src/libraries/Microsoft.PowerFx.Core/Utils/LanguageConstants.cs b/src/libraries/Microsoft.PowerFx.Core/Utils/LanguageConstants.cs
@@ -75,5 +75,10 @@ internal class LanguageConstants
         /// The string value representing the JSON format.
         /// </summary>
         internal const string JSONFormatEnumString = "JSONFormat";
+
+        /// <summary>
+        /// The string value representing the Type literal keyword.
+        /// </summary>
+        public const string TypeLiteralInvariantName = "Type";
     }
 }
diff --git a/src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngine.cs b/src/libraries/Microsoft.PowerFx.Interpreter/RecalcEngine.cs
@@ -397,13 +397,12 @@ public void AddUserDefinitions(string script, CultureInfo parseCulture = null, A
             var options = new ParserOptions()
             {
                 AllowsSideEffects = false,
-                AllowParseAsTypeLiteral = true,
                 Culture = parseCulture ?? CultureInfo.InvariantCulture
             };
 
             var sb = new StringBuilder();
 
-            var checkResult = new DefinitionsCheckResult()
+            var checkResult = new DefinitionsCheckResult(this.Config.Features)
                                     .SetText(script, options);
 
             var parseResult = checkResult.ApplyParse();

diff --git a/src/libraries/Microsoft.PowerFx.Repl/Modules/ModuleLoadContext.cs b/src/libraries/Microsoft.PowerFx.Repl/Modules/ModuleLoadContext.cs
@@ -142,11 +142,10 @@ private bool ResolveBody(ModulePoco poco, SymbolTable moduleExports, ReadOnlySym
             bool allowSideEffects = true;
             var options = new ParserOptions
             {
-                AllowParseAsTypeLiteral = true,
                 AllowsSideEffects = true
             };
 
-            var parseResult = UserDefinitions.Parse(str, options);
+            var parseResult = UserDefinitions.Parse(str, options, Features.PowerFxV1);
 
             var fragmentLocation = poco.Formulas.Location;
 

diff --git a/src/libraries/Microsoft.PowerFx.Repl/Repl.cs b/src/libraries/Microsoft.PowerFx.Repl/Repl.cs
@@ -503,7 +503,7 @@ await this.Output.WriteLineAsync($"Error: Can't set '{name}' to a Void value.",
             var errors = check.ApplyErrors();
             if (!check.IsSuccess)
             {
-                var definitionsCheckResult = new DefinitionsCheckResult();
+                var definitionsCheckResult = new DefinitionsCheckResult(this.Engine.Config.Features);
 
                 definitionsCheckResult.SetText(expression, this.ParserOptions)
                     .ApplyParseErrors();

diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/AsType_UO.txt b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/AsType_UO.txt
@@ -1,5 +1,3 @@
-#SETUP: AllowTypeLiteral, TimeZoneInfo("Pacific Standard Time")
-
 // Primitives
 >> AsType(ParseJSON("987654321"), Number)
 987654321
@@ -25,27 +23,18 @@ true
 >> AsType(ParseJSON("""1984-01-01"""), Date)
 Date(1984,1,1)
 
->> AsType(ParseJSON("""1900-12-31T23:59:59.999Z"""), DateTime)
-DateTime(1900,12,31,15,59,59,999)
-
->> AsType(ParseJSON("""1900-12-31T23:59:59.999"""), DateTime)
-DateTime(1900,12,31,23,59,59,999)
+>> AsType(ParseJSON("""1900-12-31T23:59:59.999"""), Date)
+Date(1900,12,31)
 
->> AsType(ParseJSON("""1900-12-31T23:59:59.999+00:00"""), DateTime)
-DateTime(1900,12,31,15,59,59,999)
+>> AsType(ParseJSON("""1900-12-31T00:00:00.000Z"""), Date)
+Date(1900,12,31)
 
->> AsType(ParseJSON("""1900-12-31T23:59:59.999-08:00"""), DateTime)
+>> AsType(ParseJSON("""1900-12-31T23:59:59.999"""), DateTime)
 DateTime(1900,12,31,23,59,59,999)
 
 >> AsType(ParseJSON("""1900-12-31"""), DateTime)
 DateTime(1900,12,31,0,0,0,0)
 
->> AsType(ParseJSON("""1900-12-31T23:59:59.999"""), Date)
-Date(1900,12,31)
-
->> AsType(ParseJSON("""1900-12-31T00:00:00.000Z"""), Date)
-Date(1900,12,31)
-
 >> AsType(ParseJSON("""11:59:59.999"""), Time)
 Time(11,59,59,999)
 
@@ -64,12 +53,6 @@ DateTime(1900,12,31,0,0,0,0)
 >> AsType(ParseJSON("""1900-12-31T00:00:00.000-08:00"""), DateTimeTZInd)
 DateTime(1900,12,31,8,0,0,0)
 
->> DateTimeValue(AsType(ParseJSON("""1900-12-31T00:00:00.000Z"""), UntypedObject))
-DateTime(1900,12,30,16,0,0,0)
-
->> DateValue(AsType(ParseJSON("""1900-12-31T00:00:00.000Z"""), UntypedObject))
-Date(1900,12,30)
-
 >> Value(AsType(ParseJSON("42"), UntypedObject))
 42
 
@@ -111,9 +94,6 @@ Error({Kind:ErrorKind.InvalidArgument})
 >> AsType(ParseJSON("""42"""), Number)
 Error({Kind:ErrorKind.InvalidArgument})
 
->> AsType(ParseJSON("""1900-12-31T24:59:59.1002Z"""), DateTime)
-Error({Kind:ErrorKind.InvalidArgument})
-
 >> AsType(ParseJSON("""24:59:59.12345678"""), Time)
 Error({Kind:ErrorKind.InvalidArgument})
 

diff --git a/...ts/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/AsType_UO_TimeZone_Seattle.txt b/...ts/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/AsType_UO_TimeZone_Seattle.txt
@@ -0,0 +1,19 @@
+#SETUP: TimeZoneInfo("Pacific Standard Time")
+
+>> AsType(ParseJSON("""1900-12-31T23:59:59.999Z"""), DateTime)
+DateTime(1900,12,31,15,59,59,999)
+
+>> AsType(ParseJSON("""1900-12-31T23:59:59.999+00:00"""), DateTime)
+DateTime(1900,12,31,15,59,59,999)
+
+>> AsType(ParseJSON("""1900-12-31T23:59:59.999-08:00"""), DateTime)
+DateTime(1900,12,31,23,59,59,999)
+
+>> DateTimeValue(AsType(ParseJSON("""1900-12-31T00:00:00.000Z"""), UntypedObject))
+DateTime(1900,12,30,16,0,0,0)
+
+>> DateValue(AsType(ParseJSON("""1900-12-31T00:00:00.000Z"""), UntypedObject))
+Date(1900,12,30)
+
+>> AsType(ParseJSON("""1900-12-31T24:59:59.1002Z"""), DateTime)
+Error({Kind:ErrorKind.InvalidArgument})
diff --git a/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/IsType_UO.txt b/src/tests/Microsoft.PowerFx.Core.Tests.Shared/ExpressionTestCases/IsType_UO.txt
@@ -1,5 +1,3 @@
-#SETUP: AllowTypeLiteral, TimeZoneInfo("Pacific Standard Time")
-
 // Primitives
 >> IsType(ParseJSON("987654321"), Number)
 true