Fix issue when using Unicode characters in CDP datasets

microsoft · Jan 27, 2025 · aa819de · aa819de
1 parent 49c1c99
commit aa819de
Show file tree

Hide file tree

Showing 5 changed files with 52 additions and 12 deletions.
diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/CdpTableResolver.cs
@@ -52,7 +52,7 @@ public async Task<ConnectorType> ResolveTableAsync(string tableName, Cancellatio
                 }
             }
 
-            string dataset = _doubleEncoding ? CdpServiceBase.DoubleEncode(_tabularTable.DatasetName) : _tabularTable.DatasetName;
+            string dataset = _doubleEncoding ? CdpServiceBase.DoubleEncode(_tabularTable.DatasetName) : CdpServiceBase.SingleEncode(_tabularTable.DatasetName);
             string uri = (_uriPrefix ?? string.Empty) + (UseV2(_uriPrefix) ? "/v2" : string.Empty) + $"/$metadata.json/datasets/{dataset}/tables/{CdpServiceBase.DoubleEncode(tableName)}?api-version=2015-09-01";
 
             string text = await CdpServiceBase.GetObject(_httpClient, $"Get table metadata", uri, null, cancellationToken, Logger).ConfigureAwait(false);

diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpDataSource.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpDataSource.cs
@@ -43,7 +43,7 @@ public virtual async Task<IEnumerable<CdpTable>> GetTablesAsync(HttpClient httpC
 
             string uri = (_uriPrefix ?? string.Empty)
                 + (CdpTableResolver.UseV2(uriPrefix) ? "/v2" : string.Empty)
-                + $"/datasets/{(DatasetMetadata.IsDoubleEncoding ? DoubleEncode(DatasetName) : DatasetName)}"
+                + $"/datasets/{(DatasetMetadata.IsDoubleEncoding ? DoubleEncode(DatasetName) : SingleEncode(DatasetName))}"
                 + (uriPrefix.Contains("/sharepointonline/") ? "/alltables" : "/tables");
 
             GetTables tables = await GetObject<GetTables>(httpClient, "Get tables", uri, null, cancellationToken, logger).ConfigureAwait(false);

diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpServiceBase.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpServiceBase.cs
@@ -69,7 +69,12 @@ protected internal static async Task<string> GetObject(HttpClient httpClient, st
 
         internal static string DoubleEncode(string param)
         {
-            return Uri.EscapeDataString(Uri.EscapeDataString(param));
+            return SingleEncode(SingleEncode(param));
+        }
+
+        internal static string SingleEncode(string param)
+        {
+            return Uri.EscapeDataString(param);
         }
     }
 }
diff --git a/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs b/src/libraries/Microsoft.PowerFx.Connectors/Tabular/Services/CdpTable.cs
@@ -113,7 +113,7 @@ protected override async Task<IReadOnlyCollection<DValue<RecordValue>>> GetItems
             Uri uri = new Uri(
                    (_uriPrefix ?? string.Empty) +
                    (CdpTableResolver.UseV2(_uriPrefix) ? "/v2" : string.Empty) +
-                   $"/datasets/{(DatasetMetadata.IsDoubleEncoding ? DoubleEncode(DatasetName) : DatasetName)}/tables/{Uri.EscapeDataString(TableName)}/items?api-version=2015-09-01" + queryParams, UriKind.Relative);
+                   $"/datasets/{(DatasetMetadata.IsDoubleEncoding ? DoubleEncode(DatasetName) : SingleEncode(DatasetName))}/tables/{Uri.EscapeDataString(TableName)}/items?api-version=2015-09-01" + queryParams, UriKind.Relative);
 
             string text = await GetObject(_httpClient, $"List items ({nameof(GetItemsInternalAsync)})", uri.ToString(), null, cancellationToken, executionLogger).ConfigureAwait(false);
             return !string.IsNullOrWhiteSpace(text) ? GetResult(text) : Array.Empty<DValue<RecordValue>>();

diff --git a/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs b/src/tests/Microsoft.PowerFx.Connectors.Tests.Shared/PowerPlatformTabularTests.cs
@@ -240,7 +240,7 @@ public async Task SQL_CdpTabular_GetTables2()
             // For SQL we don't have relationships
             bool b = sqlTable.RecordType.TryGetFieldExternalTableName("ProductModelID", out string externalTableName, out string foreignKey);
             Assert.False(b);
-            
+
             testConnector.SetResponseFromFiles(@"Responses\SQL GetSchema ProductModel.json");
             b = sqlTable.RecordType.TryGetFieldType("ProductModelID", out FormulaType productModelID);
 
@@ -251,6 +251,41 @@ public async Task SQL_CdpTabular_GetTables2()
             Assert.Equal("ProductID", string.Join("|", GetPrimaryKeyNames(sqlTable.RecordType)));
         }
 
+        [Fact]
+        public async Task SQL_CdpTabular_GetTables3()
+        {
+            using var testConnector = new LoggingTestServer(null /* no swagger */, _output);
+            var config = new PowerFxConfig(Features.PowerFxV1);
+            var engine = new RecalcEngine(config);
+
+            ConsoleLogger logger = new ConsoleLogger(_output);
+            using var httpClient = new HttpClient(testConnector);
+            string connectionId = "29941b77eb0a40fe925cd7a03cb85b40";
+            string jwt = "eyJ0eX...";
+            using var client = new PowerPlatformConnectorClient("49970107-0806-e5a7-be5e-7c60e2750f01.12.common.firstrelease.azure-apihub.net", "49970107-0806-e5a7-be5e-7c60e2750f01", connectionId, () => jwt, httpClient) { SessionId = "8e67ebdc-d402-455a-b33a-304820832383" };                      
+
+            testConnector.SetResponseFromFile(@"Responses\SQL GetDatasetsMetadata.json");
+            DatasetMetadata dm = await CdpDataSource.GetDatasetsMetadataAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
+
+            Assert.NotNull(dm);
+            Assert.Null(dm.Blob);
+
+            // Use of Unicode characters in DataSource
+            CdpDataSource cds = new CdpDataSource("pfxdev-sql.database.windows.net,Aßþ");
+
+            testConnector.SetResponseFromFiles(@"Responses\SQL GetDatasetsMetadata.json", @"Responses\SQL GetTables SampleDB.json");
+            IEnumerable<CdpTable> tables = await cds.GetTablesAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
+
+            Assert.NotNull(tables);             
+
+            CdpTable connectorTable = tables.First(t => t.DisplayName == "Customer");
+            Assert.False(connectorTable.IsInitialized);
+
+            testConnector.SetResponseFromFile(@"Responses\SQL Server Load Customers DB.json");
+            await connectorTable.InitAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
+            Assert.True(connectorTable.IsInitialized);
+        }
+
         [Fact]
         public async Task SQL_CdpTabular_JoinCapabilityTest()
         {
@@ -264,23 +299,23 @@ public async Task SQL_CdpTabular_JoinCapabilityTest()
             string jwt = "eyJ0eXAiOiJKSuA...";
             using var client = new PowerPlatformConnectorClient("dac64a92-df6a-ee6e-a6a2-be41a923e371.15.common.tip1002.azure-apihub.net", "dac64a92-df6a-ee6e-a6a2-be41a923e371", connectionId, () => jwt, httpClient) { SessionId = "8e67ebdc-d402-455a-b33a-304820832383" };
 
-            string realTableName = "Product";            
-           
+            string realTableName = "Product";
+
             CdpDataSource cds = new CdpDataSource("default,default");
 
             testConnector.SetResponseFromFiles(@"Responses\SQL GetDatasetsMetadata.json", @"Responses\SQL GetTables SampleDB.json");
             IEnumerable<CdpTable> tables = await cds.GetTablesAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
-            
+
             CdpTable table = tables.First(t => t.DisplayName == realTableName);
-                       
+
             testConnector.SetResponseFromFiles(@"Responses\SQL GetSchema Products v2.json");
             await table.InitAsync(client, $"/apim/sql/{connectionId}", CancellationToken.None, logger);
             Assert.True(table.IsInitialized);
 
             CdpTableValue sqlTable = table.GetTableValue();
             Assert.True(sqlTable._tabularService.IsInitialized);
             Assert.True(sqlTable.IsDelegable);
-            
+
             HashSet<IExternalTabularDataSource> ads = sqlTable.Type._type.AssociatedDataSources;
             Assert.NotNull(ads);
             Assert.Single(ads);
@@ -645,7 +680,7 @@ private static IEnumerable<string> GetPrimaryKeyNames(RecordType rt)
 
         [Fact]
         public async Task SF_CdpTabular_GetTables()
-        {                       
+        {
             using var testConnector = new LoggingTestServer(null /* no swagger */, _output);
             var config = new PowerFxConfig(Features.PowerFxV1);
             var engine = new RecalcEngine(config);
@@ -942,7 +977,7 @@ public async Task ZD_CdpTabular_GetTables()
             Assert.True(zdTable._tabularService.IsInitialized);
             Assert.True(zdTable.IsDelegable);
 
-            Assert.Equal( 
+            Assert.Equal(
                 "r![active:b, alias:s, created_at:d, custom_role_id:w, details:s, email:s, external_id:s, id:w, last_login_at:d, locale:s, locale_id:w, moderator:b, name:s, notes:s, only_private_comments:b, organization_id:w, " +
                 "phone:s, photo:s, restricted_agent:b, role:s, shared:b, shared_agent:b, signature:s, suspended:b, tags:s, ticket_restriction:s, time_zone:s, updated_at:d, url:s, user_fields:s, verified:b]", ((CdpRecordType)zdTable.RecordType).ToStringWithDisplayNames());