fix(databricks-driver): fix databricks bucket URL parsing. Now driver…

… supports both (legacy and modern urls) (#8968) * chore(testing-drivers): update databricks export to azure configuration * fix azure blobs filtering * add support for azure 'dfs.core.windows.net' alongside with 'blob.core.windows.net' * update test snapshots # Conflicts: # packages/cubejs-testing-drivers/test/__snapshots__/databricks-jdbc-export-bucket-azure-full.test.ts.snap
cube-js · Dec 7, 2024 · 26c9a27 · 26c9a27
1 parent b20a3ca
commit 26c9a27
Show file tree

Hide file tree

Showing 3 changed files with 1,688 additions and 283 deletions.
diff --git a/packages/cubejs-base-driver/src/BaseDriver.ts b/packages/cubejs-base-driver/src/BaseDriver.ts
@@ -751,7 +751,8 @@ export abstract class BaseDriver implements DriverInterface {
     bucketName: string,
     tableName: string
   ): Promise<string[]> {
-    const parts = bucketName.split('.blob.core.windows.net/');
+    const splitter = bucketName.includes('blob.core') ? '.blob.core.windows.net/' : '.dfs.core.windows.net/';
+    const parts = bucketName.split(splitter);
     const account = parts[0];
     const container = parts[1].split('/')[0];
     let credential: StorageSharedKeyCredential | DefaultAzureCredential;
@@ -805,7 +806,7 @@ export abstract class BaseDriver implements DriverInterface {
 
     const csvFiles: string[] = [];
     const containerClient = blobServiceClient.getContainerClient(container);
-    const blobsList = containerClient.listBlobsFlat({ prefix: `${tableName}/` });
+    const blobsList = containerClient.listBlobsFlat({ prefix: `${tableName}` });
     for await (const blob of blobsList) {
       if (blob.name && (blob.name.endsWith('.csv.gz') || blob.name.endsWith('.csv'))) {
         const starts = new Date();

diff --git a/packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json b/packages/cubejs-testing-drivers/fixtures/databricks-jdbc.json
@@ -15,7 +15,7 @@
       "cube": {
         "environment": {
           "CUBEJS_DB_EXPORT_BUCKET_TYPE": "azure",
-          "CUBEJS_DB_EXPORT_BUCKET": "wasbs://databricks-drivers-tests-preaggs@coreteamdevtest.blob.core.windows.net",
+          "CUBEJS_DB_EXPORT_BUCKET": "abfss://databricks-drivers-tests-preaggs@coreteamdevtest.dfs.core.windows.net",
           "CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY": "${DRIVERS_TESTS_CUBEJS_DB_EXPORT_BUCKET_AZURE_KEY}"
         }
       }