Skip to content

Commit cd485ee

Browse files
PuchaczovPuchaczov
committed
The inferred columns of a query will be passed to appropriate node so that the user can construct it's own type based on that columns dynamically (AI infersion). The next commit will add additional tests and fixes to make sure everything is working properly
1 parent 55fe80d commit cd485ee

13 files changed

+1204
-45
lines changed

‎Musoq.Converter/Build/TranformTree.cs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,15 @@ public override void Build(BuildItems items)
2020

2121
var queryTree = items.RawQueryTree;
2222

23-
var metadata = new BuildMetadataAndInferTypeVisitor(items.SchemaProvider, items.PositionalEnvironmentVariables);
23+
var extractColumnsVisitor = new ExtractRawColumnsVisitor();
24+
var extractRawColumnsTraverseVisitor = new ExtractRawColumnsTraverseVisitor(extractColumnsVisitor);
25+
26+
queryTree.Accept(extractRawColumnsTraverseVisitor);
27+
28+
var metadata = new BuildMetadataAndInferTypeVisitor(items.SchemaProvider, items.PositionalEnvironmentVariables, extractColumnsVisitor.Columns);
2429
var metadataTraverser = new BuildMetadataAndInferTypeTraverseVisitor(metadata);
2530

2631
queryTree.Accept(metadataTraverser);
27-
2832
queryTree = metadata.Root;
2933

3034
var rewriter = new RewriteQueryVisitor();

‎Musoq.Evaluator.Tests/DynamicSourceQueryTests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public void WithDynamicSource_SimpleQuery_ShouldPass()
4141
{
4242
const string query = "select Id, Name from #dynamic.all()";
4343
var sources =
44-
new List<dynamic>()
44+
new List<dynamic>
4545
{
4646
CreateExpandoObject(1, "Test1"),
4747
CreateExpandoObject(2, "Test2")

‎Musoq.Evaluator.Tests/Schema/Dynamic/DynamicQueryTestsBase.cs

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,12 @@ protected CompiledQuery CreateAndRunVirtualMachine(
2929

3030
protected CompiledQuery CreateAndRunVirtualMachine(
3131
string script,
32-
(string Name, IReadOnlyCollection<dynamic> Values, IReadOnlyDictionary<string, Type> Schema)[] sources
32+
IEnumerable<(string Name, IReadOnlyCollection<dynamic> Values, IReadOnlyDictionary<string, Type> Schema)> sources
3333
)
3434
{
35-
var schemas = new Dictionary<string, (IReadOnlyDictionary<string, Type> Schema, IEnumerable<dynamic> Values)>();
36-
foreach (var source in sources)
37-
{
38-
schemas.Add(source.Name, (source.Schema, source.Values));
39-
}
35+
var schemas = sources
36+
.ToDictionary<(string Name, IReadOnlyCollection<dynamic> Values, IReadOnlyDictionary<string, Type> Schema), string, (IReadOnlyDictionary<string, Type> Schema, IEnumerable<dynamic> Values)>(source => source.Name, source => (source.Schema, source.Values));
37+
4038
return InstanceCreator.CompileForExecution(
4139
script,
4240
Guid.NewGuid().ToString(),

‎Musoq.Evaluator.Tests/SingleSchemaEvaluatorTests.cs

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ namespace Musoq.Evaluator.Tests
1111
public class SingleSchemaEvaluatorTests : BasicEntityTestBase
1212
{
1313
[TestMethod]
14-
public void SimpleVNextTest()
14+
public void WhenSingleValueTableIsCoupledWithSchema_ShouldHaveAppropriateTypesTest()
1515
{
1616
var query =
1717
"table DummyTable {" +
@@ -47,6 +47,55 @@ public void SimpleVNextTest()
4747
Assert.AreEqual("XXX", table[2].Values[0]);
4848
Assert.AreEqual("dadsqqAA", table[3].Values[0]);
4949
}
50+
51+
[TestMethod]
52+
public void WhenTwoValuesTableIsCoupledWithSchema_ShouldHaveAppropriateTypesTest()
53+
{
54+
var query =
55+
"table DummyTable {" +
56+
" Country 'System.String'," +
57+
" Population 'System.Decimal'" +
58+
"};" +
59+
"couple #A.Entities with table DummyTable as SourceOfDummyRows;" +
60+
"select Country, Population from SourceOfDummyRows();";
61+
62+
var sources = new Dictionary<string, IEnumerable<BasicEntity>>
63+
{
64+
{
65+
"#A",
66+
new[]
67+
{
68+
new BasicEntity("ABCAACBA", 10),
69+
new BasicEntity("AAeqwgQEW", 20),
70+
new BasicEntity("XXX", 30),
71+
new BasicEntity("dadsqqAA", 40)
72+
}
73+
}
74+
};
75+
76+
var vm = CreateAndRunVirtualMachine(query, sources);
77+
var table = vm.Run();
78+
79+
Assert.AreEqual(2, table.Columns.Count());
80+
Assert.AreEqual("Country", table.Columns.ElementAt(0).ColumnName);
81+
Assert.AreEqual(typeof(string), table.Columns.ElementAt(0).ColumnType);
82+
83+
Assert.AreEqual("Population", table.Columns.ElementAt(1).ColumnName);
84+
Assert.AreEqual(typeof(decimal), table.Columns.ElementAt(1).ColumnType);
85+
86+
Assert.AreEqual(4, table.Count);
87+
Assert.AreEqual("ABCAACBA", table[0].Values[0]);
88+
Assert.AreEqual(10m, table[0].Values[1]);
89+
90+
Assert.AreEqual("AAeqwgQEW", table[1].Values[0]);
91+
Assert.AreEqual(20m, table[1].Values[1]);
92+
93+
Assert.AreEqual("XXX", table[2].Values[0]);
94+
Assert.AreEqual(30m, table[2].Values[1]);
95+
96+
Assert.AreEqual("dadsqqAA", table[3].Values[0]);
97+
Assert.AreEqual(40m, table[3].Values[1]);
98+
}
5099

51100
[TestMethod]
52101
public void LikeOperatorTest()

‎Musoq.Evaluator/AliasGenerator.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ namespace Musoq.Evaluator;
88

99
public static class AliasGenerator
1010
{
11-
public static string CreateAliasIfEmpty(string alias, IReadOnlyList<string> usedAliases, string seed = "defaultSeed")
11+
public static string
12+
CreateAliasIfEmpty(string alias, IReadOnlyList<string> usedAliases, string seed = "defaultSeed")
1213
{
1314
if (!string.IsNullOrEmpty(alias))
1415
return alias;

‎Musoq.Evaluator/Visitors/BuildMetadataAndInferTypeTraverseVisitor.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public class BuildMetadataAndInferTypeTraverseVisitor : IQueryPartAwareExpressio
1313
{
1414
private readonly Stack<Scope> _scopes = new();
1515
private readonly IAwareExpressionVisitor _visitor;
16+
1617
private IdentifierNode _theMostInnerIdentifier;
1718

1819
public BuildMetadataAndInferTypeTraverseVisitor(IAwareExpressionVisitor visitor)
@@ -279,6 +280,8 @@ public void Visit(JoinFromNode node)
279280
firstTableSymbol = firstTableSymbol.MakeNullableIfPossible();
280281
Scope.ScopeSymbolTable.UpdateSymbol(Scope[join.Source.Id], firstTableSymbol);
281282
break;
283+
default:
284+
throw new ArgumentOutOfRangeException();
282285
}
283286

284287
var id = $"{Scope[join.Source.Id]}{Scope[join.With.Id]}";
@@ -309,6 +312,8 @@ public void Visit(JoinFromNode node)
309312
previousTableSymbol = previousTableSymbol.MakeNullableIfPossible();
310313
Scope.ScopeSymbolTable.UpdateSymbol(id, previousTableSymbol);
311314
break;
315+
default:
316+
throw new ArgumentOutOfRangeException();
312317
}
313318

314319
id = $"{id}{Scope[join.With.Id]}";

‎Musoq.Evaluator/Visitors/BuildMetadataAndInferTypeVisitor.cs

Lines changed: 49 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ public class BuildMetadataAndInferTypeVisitor : IAwareExpressionVisitor
5959

6060
private readonly List<FieldNode> _groupByFields = new();
6161
private readonly List<Type> _nullSuspiciousTypes;
62+
private readonly IReadOnlyDictionary<string, string[]> _columns;
6263

6364
private int _setKey;
6465
private int _schemaFromKey;
@@ -71,13 +72,17 @@ public class BuildMetadataAndInferTypeVisitor : IAwareExpressionVisitor
7172

7273
private Stack<string> Methods { get; } = new();
7374

74-
public BuildMetadataAndInferTypeVisitor(ISchemaProvider provider,
75-
IReadOnlyDictionary<uint, IReadOnlyDictionary<string, string>> positionalEnvironmentVariables)
75+
public BuildMetadataAndInferTypeVisitor(
76+
ISchemaProvider provider,
77+
IReadOnlyDictionary<uint, IReadOnlyDictionary<string, string>> positionalEnvironmentVariables,
78+
IReadOnlyDictionary<string, string[]> columns)
7679
{
7780
_provider = provider;
7881
_positionalEnvironmentVariables = positionalEnvironmentVariables;
82+
_columns = columns;
7983
_positionalEnvironmentVariablesKey = 0;
8084
_nullSuspiciousTypes = new List<Type>();
85+
8186
}
8287

8388
protected Stack<Node> Nodes { get; } = new();
@@ -447,7 +452,7 @@ public void Visit(IdentifierNode node)
447452
if (column == null)
448453
PrepareAndThrowUnknownColumnExceptionMessage(node.Name, tableSymbol.GetColumns());
449454

450-
Visit(new AccessColumnNode(node.Name, string.Empty, column.ColumnType, TextSpan.Empty));
455+
Visit(new AccessColumnNode(node.Name, string.Empty, column?.ColumnType, TextSpan.Empty));
451456
return;
452457
}
453458

@@ -743,24 +748,28 @@ public void Visit(SchemaFromNode node)
743748
{
744749
var schema = _provider.GetSchema(node.Schema);
745750

751+
_queryAlias = AliasGenerator.CreateAliasIfEmpty(node.Alias, _generatedAliases, _schemaFromKey.ToString());
752+
_generatedAliases.Add(_queryAlias);
753+
746754
var table = _currentScope.Name != "Desc"
747-
? schema.GetTableByName(node.Method, new RuntimeContext(
748-
CancellationToken.None,
749-
Array.Empty<ISchemaColumn>(),
750-
_positionalEnvironmentVariables.ContainsKey(_positionalEnvironmentVariablesKey)
751-
? _positionalEnvironmentVariables[_positionalEnvironmentVariablesKey]
752-
: new Dictionary<string, string>(),
753-
(node, Array.Empty<ISchemaColumn>(), AllTrueWhereNode)), _schemaFromArgs.ToArray())
755+
? schema.GetTableByName(
756+
node.Method,
757+
new RuntimeContext(
758+
CancellationToken.None,
759+
_columns[_queryAlias + _schemaFromKey].Select((f, i) => new SchemaColumn(f, i, typeof(object))).ToArray(),
760+
_positionalEnvironmentVariables.ContainsKey(_positionalEnvironmentVariablesKey)
761+
? _positionalEnvironmentVariables[_positionalEnvironmentVariablesKey]
762+
: new Dictionary<string, string>(),
763+
(node, Array.Empty<ISchemaColumn>(), AllTrueWhereNode)
764+
),
765+
_schemaFromArgs.ToArray())
754766
: new DynamicTable(Array.Empty<ISchemaColumn>());
755767

756768
_positionalEnvironmentVariablesKey += 1;
757769
_schemaFromArgs.Clear();
758770

759771
AddAssembly(schema.GetType().Assembly);
760772

761-
_queryAlias = AliasGenerator.CreateAliasIfEmpty(node.Alias, _generatedAliases, _schemaFromKey.ToString());
762-
_generatedAliases.Add(_queryAlias);
763-
764773
var tableSymbol = new TableSymbol(_queryAlias, schema, table, !string.IsNullOrEmpty(node.Alias));
765774
_currentScope.ScopeSymbolTable.AddSymbol(_queryAlias, tableSymbol);
766775
_currentScope[node.Id] = _queryAlias;
@@ -774,8 +783,7 @@ public void Visit(SchemaFromNode node)
774783
if (!_usedColumns.ContainsKey(aliasedSchemaFromNode))
775784
_usedColumns.Add(aliasedSchemaFromNode, new List<ISchemaColumn>());
776785

777-
if (!_usedWhereNodes.ContainsKey(aliasedSchemaFromNode))
778-
_usedWhereNodes.Add(aliasedSchemaFromNode, AllTrueWhereNode);
786+
_usedWhereNodes.TryAdd(aliasedSchemaFromNode, AllTrueWhereNode);
779787

780788
Nodes.Push(aliasedSchemaFromNode);
781789
}
@@ -798,21 +806,35 @@ public void Visit(AliasedFromNode node)
798806
_queryAlias = AliasGenerator.CreateAliasIfEmpty(node.Alias, _generatedAliases, _schemaFromKey.ToString());
799807
_generatedAliases.Add(_queryAlias);
800808

801-
var tableSymbol = new TableSymbol(_queryAlias, schema, table, !string.IsNullOrEmpty(node.Alias));
802-
_currentScope.ScopeSymbolTable.AddSymbol(_queryAlias, tableSymbol);
803-
_currentScope[node.Id] = _queryAlias;
804-
805809
var aliasedSchemaFromNode = new Parser.SchemaFromNode(schemaInfo.Schema, schemaInfo.Method, node.Args,
806810
_queryAlias, node.InSourcePosition);
807811

812+
var tableSymbol = new TableSymbol(
813+
_queryAlias,
814+
schema,
815+
schema.GetTableByName(
816+
schemaInfo.Method,
817+
new RuntimeContext(
818+
CancellationToken.None,
819+
table.Columns,
820+
_positionalEnvironmentVariables.ContainsKey(_positionalEnvironmentVariablesKey)
821+
? _positionalEnvironmentVariables[_positionalEnvironmentVariablesKey]
822+
: new Dictionary<string, string>(),
823+
(aliasedSchemaFromNode, Array.Empty<ISchemaColumn>(), AllTrueWhereNode)
824+
)
825+
) ?? table,
826+
!string.IsNullOrEmpty(node.Alias)
827+
);
828+
_currentScope.ScopeSymbolTable.AddSymbol(_queryAlias, tableSymbol);
829+
_currentScope[node.Id] = _queryAlias;
830+
808831
if (!_inferredColumns.ContainsKey(aliasedSchemaFromNode))
809832
_inferredColumns.Add(aliasedSchemaFromNode, table.Columns);
810833

811834
if (!_usedColumns.ContainsKey(aliasedSchemaFromNode))
812835
_usedColumns.Add(aliasedSchemaFromNode, new List<ISchemaColumn>());
813836

814-
if (!_usedWhereNodes.ContainsKey(aliasedSchemaFromNode))
815-
_usedWhereNodes.Add(aliasedSchemaFromNode, AllTrueWhereNode);
837+
_usedWhereNodes.TryAdd(aliasedSchemaFromNode, AllTrueWhereNode);
816838

817839
Nodes.Push(aliasedSchemaFromNode);
818840
}
@@ -840,7 +862,11 @@ public void Visit(InMemoryTableFromNode node)
840862
else
841863
{
842864
var scope = _currentScope;
865+
843866
while (scope != null && scope.Name != "CTE") scope = scope.Parent;
867+
868+
if (scope is null)
869+
throw new NotSupportedException($"Table {node.VariableName} is not defined.");
844870

845871
tableSymbol = scope.ScopeSymbolTable.GetSymbol<TableSymbol>(node.VariableName);
846872
}
@@ -1132,7 +1158,6 @@ private FieldNode[] CreateFields(FieldNode[] oldFields)
11321158

11331159
for (var i = reorderedList.Length - 1; i >= 0; i--) reorderedList[i] = Nodes.Pop() as FieldNode;
11341160

1135-
11361161
for (int i = 0, j = reorderedList.Length, p = 0; i < j; ++i)
11371162
{
11381163
var field = reorderedList[i];
@@ -1443,11 +1468,6 @@ public void SetTheMostInnerIdentifierOfDotNode(IdentifierNode node)
14431468
_theMostInnerIdentifier = node;
14441469
}
14451470

1446-
public void PushNode(Node node)
1447-
{
1448-
Nodes.Push(node);
1449-
}
1450-
14511471
private Type FindGreatestCommonSubtype()
14521472
{
14531473
var types = _nullSuspiciousTypes.Where(type => type != NullNode.NullType.Instance).Select(StripNullable)
@@ -1484,7 +1504,7 @@ private static void PrepareAndThrowUnknownColumnExceptionMessage(string identifi
14841504
var candidatesColumns = columns.Where(
14851505
col =>
14861506
library.Soundex(col.ColumnName) == library.Soundex(identifier) ||
1487-
library.LevenshteinDistance(col.ColumnName, identifier).Value < 3).ToArray();
1507+
library.LevenshteinDistance(col.ColumnName, identifier) < 3).ToArray();
14881508

14891509
for (var i = 0; i < candidatesColumns.Length - 1; i++)
14901510
{
@@ -1512,7 +1532,7 @@ private static void PrepareAndThrowUnknownPropertyExceptionMessage(string identi
15121532
var candidatesProperties = properties.Where(
15131533
prop =>
15141534
library.Soundex(prop.Name) == library.Soundex(identifier) ||
1515-
library.LevenshteinDistance(prop.Name, identifier).Value < 3).ToArray();
1535+
library.LevenshteinDistance(prop.Name, identifier) < 3).ToArray();
15161536

15171537
for (var i = 0; i < candidatesProperties.Length - 1; i++)
15181538
{

0 commit comments

Comments
 (0)