From aa819deba77eae9b998265e64fa62b580787a760 Mon Sep 17 00:00:00 2001 From: Luc Genetier Date: Mon, 27 Jan 2025 18:27:07 +0100 Subject: [PATCH] Fix issue when using Unicode characters in CDP datasets --- .../Tabular/CdpTableResolver.cs | 2 +- .../Tabular/Services/CdpDataSource.cs | 2 +- .../Tabular/Services/CdpServiceBase.cs | 7 ++- .../Tabular/Services/CdpTable.cs | 2 +- .../PowerPlatformTabularTests.cs | 51 ++++++++++++++++--- 5 files changed, 52 insertions(+), 12 deletions(-) diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs index f6ea4849b5..efaa7bd4c1 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs @@ -52,7 +52,7 @@ public async Task ResolveTableAsync(string tableName, Cancellatio } } - string dataset = _doubleEncoding ? CdpServiceBase.DoubleEncode(_tabularTable.DatasetName) : _tabularTable.DatasetName; + string dataset = _doubleEncoding ? CdpServiceBase.DoubleEncode(_tabularTable.DatasetName) : CdpServiceBase.SingleEncode(_tabularTable.DatasetName); string uri = (_uriPrefix ?? string.Empty) + (UseV2(_uriPrefix) ? "/v2" : string.Empty) + $"/$metadata.json/datasets/{dataset}/tables/{CdpServiceBase.DoubleEncode(tableName)}?api-version=2015-09-01"; string text = await CdpServiceBase.GetObject(_httpClient, $"Get table metadata", uri, null, cancellationToken, Logger).ConfigureAwait(false); diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpDataSource.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpDataSource.cs index 6d849e105a..6aeaf2e958 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpDataSource.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpDataSource.cs @@ -43,7 +43,7 @@ public virtual async Task> GetTablesAsync(HttpClient httpC string uri = (_uriPrefix ?? string.Empty) + (CdpTableResolver.UseV2(uriPrefix) ? "/v2" : string.Empty) - + $"/datasets/{(DatasetMetadata.IsDoubleEncoding ? DoubleEncode(DatasetName) : DatasetName)}" + + $"/datasets/{(DatasetMetadata.IsDoubleEncoding ? DoubleEncode(DatasetName) : SingleEncode(DatasetName))}" + (uriPrefix.Contains("/sharepointonline/") ? "/alltables" : "/tables"); GetTables tables = await GetObject(httpClient, "Get tables", uri, null, cancellationToken, logger).ConfigureAwait(false); diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpServiceBase.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpServiceBase.cs index 4b6c24c733..33fb10e859 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpServiceBase.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpServiceBase.cs @@ -69,7 +69,12 @@ protected internal static async Task GetObject(HttpClient httpClient, st internal static string DoubleEncode(string param) { - return Uri.EscapeDataString(Uri.EscapeDataString(param)); + return SingleEncode(SingleEncode(param)); + } + + internal static string SingleEncode(string param) + { + return Uri.EscapeDataString(param); } } } diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs index f039efa1b5..610eec7270 100644 --- a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs +++ b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs @@ -113,7 +113,7 @@ protected override async Task>> GetItems Uri uri = new Uri( (_uriPrefix ?? string.Empty) + (CdpTableResolver.UseV2(_uriPrefix) ? "/v2" : string.Empty) + - $"/datasets/{(DatasetMetadata.IsDoubleEncoding ? DoubleEncode(DatasetName) : DatasetName)}/tables/{Uri.EscapeDataString(TableName)}/items?api-version=2015-09-01" + queryParams, UriKind.Relative); + $"/datasets/{(DatasetMetadata.IsDoubleEncoding ? DoubleEncode(DatasetName) : SingleEncode(DatasetName))}/tables/{Uri.EscapeDataString(TableName)}/items?api-version=2015-09-01" + queryParams, UriKind.Relative); string text = await GetObject(_httpClient, $"List items ({nameof(GetItemsInternalAsync)})", uri.ToString(), null, cancellationToken, executionLogger).ConfigureAwait(false); return !string.IsNullOrWhiteSpace(text) ? GetResult(text) : Array.Empty>(); diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs index d88628b7bd..2e0bea4647 100644 --- a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs +++ b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs @@ -240,7 +240,7 @@ public async Task SQL_CdpTabular_GetTables2() // For SQL we don't have relationships bool b = sqlTable.RecordType.TryGetFieldExternalTableName("ProductModelID", out string externalTableName, out string foreignKey); Assert.False(b); - + testConnector.SetResponseFromFiles(@"Responses\SQL GetSchema ProductModel.json"); b = sqlTable.RecordType.TryGetFieldType("ProductModelID", out FormulaType productModelID); @@ -251,6 +251,41 @@ public async Task SQL_CdpTabular_GetTables2() Assert.Equal("ProductID", string.Join("|", GetPrimaryKeyNames(sqlTable.RecordType))); } + [Fact] + public async Task SQL_CdpTabular_GetTables3() + { + using var testConnector = new LoggingTestServer(null /* no swagger */, _output); + var config = new PowerFxConfig(Features.PowerFxV1); + var engine = new RecalcEngine(config); + + ConsoleLogger logger = new ConsoleLogger(_output); + using var httpClient = new HttpClient(testConnector); + string connectionId = "29941b77eb0a40fe925cd7a03cb85b40"; + string jwt = "eyJ0eX..."; + using var client = new PowerPlatformConnectorClient("49970107-0806-e5a7-be5e-7c60e2750f01.12.common.firstrelease.azure-apihub.net", "49970107-0806-e5a7-be5e-7c60e2750f01", connectionId, () => jwt, httpClient) { SessionId = "8e67ebdc-d402-455a-b33a-304820832383" }; + + testConnector.SetResponseFromFile(@"Responses\SQL GetDatasetsMetadata.json"); + DatasetMetadata dm = await CdpDataSource.GetDatasetsMetadataAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger); + + Assert.NotNull(dm); + Assert.Null(dm.Blob); + + // Use of Unicode characters in DataSource + CdpDataSource cds = new CdpDataSource("pfxdev-sql.database.windows.net,Aßþ"); + + testConnector.SetResponseFromFiles(@"Responses\SQL GetDatasetsMetadata.json", @"Responses\SQL GetTables SampleDB.json"); + IEnumerable tables = await cds.GetTablesAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger); + + Assert.NotNull(tables); + + CdpTable connectorTable = tables.First(t => t.DisplayName == "Customer"); + Assert.False(connectorTable.IsInitialized); + + testConnector.SetResponseFromFile(@"Responses\SQL Server Load Customers DB.json"); + await connectorTable.InitAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger); + Assert.True(connectorTable.IsInitialized); + } + [Fact] public async Task SQL_CdpTabular_JoinCapabilityTest() { @@ -264,15 +299,15 @@ public async Task SQL_CdpTabular_JoinCapabilityTest() string jwt = "eyJ0eXAiOiJKSuA..."; using var client = new PowerPlatformConnectorClient("dac64a92-df6a-ee6e-a6a2-be41a923e371.15.common.tip1002.azure-apihub.net", "dac64a92-df6a-ee6e-a6a2-be41a923e371", connectionId, () => jwt, httpClient) { SessionId = "8e67ebdc-d402-455a-b33a-304820832383" }; - string realTableName = "Product"; - + string realTableName = "Product"; + CdpDataSource cds = new CdpDataSource("default,default"); testConnector.SetResponseFromFiles(@"Responses\SQL GetDatasetsMetadata.json", @"Responses\SQL GetTables SampleDB.json"); IEnumerable tables = await cds.GetTablesAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger); - + CdpTable table = tables.First(t => t.DisplayName == realTableName); - + testConnector.SetResponseFromFiles(@"Responses\SQL GetSchema Products v2.json"); await table.InitAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger); Assert.True(table.IsInitialized); @@ -280,7 +315,7 @@ public async Task SQL_CdpTabular_JoinCapabilityTest() CdpTableValue sqlTable = table.GetTableValue(); Assert.True(sqlTable._tabularService.IsInitialized); Assert.True(sqlTable.IsDelegable); - + HashSet ads = sqlTable.Type._type.AssociatedDataSources; Assert.NotNull(ads); Assert.Single(ads); @@ -645,7 +680,7 @@ private static IEnumerable GetPrimaryKeyNames(RecordType rt) [Fact] public async Task SF_CdpTabular_GetTables() - { + { using var testConnector = new LoggingTestServer(null /* no swagger */, _output); var config = new PowerFxConfig(Features.PowerFxV1); var engine = new RecalcEngine(config); @@ -942,7 +977,7 @@ public async Task ZD_CdpTabular_GetTables() Assert.True(zdTable._tabularService.IsInitialized); Assert.True(zdTable.IsDelegable); - Assert.Equal( + Assert.Equal( "r![active:b, alias:s, created_at:d, custom_role_id:w, details:s, email:s, external_id:s, id:w, last_login_at:d, locale:s, locale_id:w, moderator:b, name:s, notes:s, only_private_comments:b, organization_id:w, " + "phone:s, photo:s, restricted_agent:b, role:s, shared:b, shared_agent:b, signature:s, suspended:b, tags:s, ticket_restriction:s, time_zone:s, updated_at:d, url:s, user_fields:s, verified:b]", ((CdpRecordType)zdTable.RecordType).ToStringWithDisplayNames());