DuckDB SQL Tests (finos#2870)

goldmansachs · May 28, 2024 · 7d591ba · 7d591ba
1 parent eb98f4f
commit 7d591ba
Show file tree

Hide file tree

Showing 6 changed files with 310 additions and 1 deletion.
diff --git a/.github/workflows/database-duckdb-sql-generation-integration-test.yml b/.github/workflows/database-duckdb-sql-generation-integration-test.yml
@@ -0,0 +1,98 @@
+# Copyright 2024 Goldman Sachs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Database DuckDB SQL Generation Integration Tests
+
+env:
+  CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }}
+  CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+on:
+  schedule:
+    - cron: "0 0 * * *" # every day at 00:00 on default/base branch
+  workflow_dispatch:
+    inputs:
+      pullRequestId:
+        description: "Pull request id"
+        required: false
+
+jobs:
+  build:
+    # NOTE: we cannot run this action in fork because secrets are not accessible from forks
+    # See https://community.sonarsource.com/t/github-action-ci-build-fail-with-set-the-sonar-token-env-variable/38997
+    if: github.repository == 'finos/legend-engine'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+      - name: Set up JDK
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+          server-id: ossrh
+          server-username: CI_DEPLOY_USERNAME
+          server-password: CI_DEPLOY_PASSWORD
+      - name: Configure git
+        run: |
+          git config --global committer.email "infra@finos.org"
+          git config --global committer.name "FINOS Admin"
+          git config --global author.email "${GITHUB_ACTOR}@users.noreply.github.com"
+          git config --global author.name "${GITHUB_ACTOR}"
+      - name: Download patch
+        if: github.event.inputs.pullRequestId != ''
+        run: |
+          wget --output-document=run.diff http://patch-diff.githubusercontent.com/raw/finos/legend-engine/pull/${{ github.event.inputs.pullRequestId }}.diff
+          git apply run.diff
+      - name: Print out mvn version
+        run: mvn -v
+      - name: Download deps and plugins
+        run: mvn de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
+      - name: Build
+        env:
+          MAVEN_OPTS: -Xmx4g
+        run: mvn -pl legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution-tests -am clean install -DskipTests=true
+      - name: SQL Generation Integration Tests
+        env:
+          MAVEN_OPTS: -Xmx4g
+        run: |
+          set -o pipefail
+          echo "| Test Representing Db Feature | Support Status |" >> $GITHUB_STEP_SUMMARY
+          echo "| ---------------------------- | -------------- |" >> $GITHUB_STEP_SUMMARY
+          mvn -pl legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution-tests -Dtest=Test_Relational_DbSpecific_DuckDB_UsingPureClientTestSuite test | tee >(grep -o "Tests run:[^T]*" | head -1 | tr -d '\n' >> sql_test_summary.txt) | tee >(echo "Ignored tests deviating from standard: $(grep -c deviating-from-standard)" >> sql_test_summary.txt) | tee >(grep -o "[|] [*][*].*" >> $GITHUB_STEP_SUMMARY)
+      - name: Write Summary
+        if: always()
+        run: |
+          touch sql_test_summary.txt
+          cat sql_test_summary.txt >> $GITHUB_STEP_SUMMARY
+      - name: Upload Summary
+        if: always() && (github.event.inputs.pullRequestId == '')
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{github.workflow}}
+          path: sql_test_summary.txt
+          retention-days: 1
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results
+          path: legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution-tests/target/surefire-reports/*.xml
+      - name: Upload CI Event
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: event-file
+          path: ${{ github.event_path }}          
diff --git a/...xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution-tests/pom.xml b/...xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution-tests/pom.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.finos.legend.engine</groupId>
+        <artifactId>legend-engine-xt-relationalStore-duckdb</artifactId>
+        <version>4.46.2-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>legend-engine-xt-relationalStore-duckdb-execution-tests</artifactId>
+    <packaging>jar</packaging>
+    <name>Legend Engine - XT - Relational Store - DuckDB - Execution - Tests</name>
+
+    <properties>
+        <maven.compiler.source>11</maven.compiler.source>
+        <maven.compiler.target>11</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <build>
+        <plugins>
+            <!-- Block running the integration tests by default (They should be setup to run as Integration Tests).
+            However, if you think they are light enough to run as part of regular builds, pls comment out this section -->
+            <plugin>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <useSystemClassLoader>false</useSystemClassLoader>
+                    <excludes>
+                        <exclude>**/Test_Relational_DbSpecific_DuckDB_UsingPureClientTestSuite.java</exclude>
+                    </excludes>
+                    <argLine>${argLine} ${surefire.vm.params}</argLine>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <!-- Jackson -->
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <!-- Jackson -->
+
+        <!-- JUNIT -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+        </dependency>
+        <!-- JUNIT -->
+
+        <!-- TEST -->
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-test-http-server</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-duckdb-pure</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-javaPlatformBinding-pure</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-pure-runtime-java-extension-compiled-functions-pureExtensions</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-core</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-connection-http-api</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-executionPlan-connection-authentication-default</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-executionPlan-connection-tests</artifactId>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.pure</groupId>
+            <artifactId>legend-pure-runtime-java-engine-compiled</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-xt-relationalStore-duckdb-execution</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <!-- TEST -->
+    </dependencies>
+
+</project>
diff --git a/...eClient/stores/dbSpecific/Test_Relational_DbSpecific_DuckDB_UsingPureClientTestSuite.java b/...eClient/stores/dbSpecific/Test_Relational_DbSpecific_DuckDB_UsingPureClientTestSuite.java
@@ -0,0 +1,35 @@
+// Copyright 2024 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.engine.server.test.pureClient.stores.dbSpecific;
+
+import com.fasterxml.jackson.databind.jsontype.NamedType;
+import junit.framework.Test;
+import org.finos.legend.engine.authentication.LegendDefaultDatabaseAuthenticationFlowProviderConfiguration;
+import org.finos.legend.engine.server.test.shared.Relational_DbSpecific_UsingPureClientTestSuite;
+import org.finos.legend.pure.runtime.java.compiled.testHelper.IgnoreUnsupportedApiPureTestSuiteRunner;
+import org.junit.runner.RunWith;
+
+@RunWith(IgnoreUnsupportedApiPureTestSuiteRunner.class)
+public class Test_Relational_DbSpecific_DuckDB_UsingPureClientTestSuite extends Relational_DbSpecific_UsingPureClientTestSuite
+{
+    public static Test suite() throws Exception
+    {
+        return createSuite(
+                "meta::relational::tests::sqlQueryToString::duckDB",
+                "org/finos/legend/engine/server/test/userTestConfig_withDuckDBTestConnection.json",
+                new NamedType(LegendDefaultDatabaseAuthenticationFlowProviderConfiguration.class, "legendDefault")
+        );
+    }
+}
diff --git a/...esources/org/finos/legend/engine/server/test/userTestConfig_withDuckDBTestConnection.json b/...esources/org/finos/legend/engine/server/test/userTestConfig_withDuckDBTestConnection.json
@@ -0,0 +1,61 @@
+{
+  "logging": {
+    "level": "error",
+    "appenders": [
+      {
+        "type": "console",
+        "logFormat": "%msg\r\n"
+      }
+    ]
+  },
+  "pac4j": {
+    "clients": [
+      {
+        "org.pac4j.core.client.direct.AnonymousClient": {
+        }
+      }
+    ]
+  },
+  "server": {
+    "type": "simple",
+    "applicationContextPath": "/",
+    "adminContextPath": "/admin",
+    "connector": {
+      "maxRequestHeaderSize": "32KiB",
+      "type": "http",
+      "port": 6300
+    },
+    "requestLog": {
+      "appenders": [
+      ]
+    }
+  },
+  "relationalexecution": {
+    "tempPath": "/tmp/",
+    "temporarytestdb": {
+      "port": 9099
+    },
+    "flowProviderClass": "org.finos.legend.engine.authentication.LegendDefaultDatabaseAuthenticationFlowProvider",
+    "flowProviderConfiguration": {
+      "_type": "legendDefault"
+    }
+  },
+  "testConnectionsToEnable" : ["DuckDB"],
+  "staticTestConnections" : {
+    "DuckDB" : {
+      "_type": "RelationalDatabaseConnection",
+      "datasourceSpecification": {
+        "_type": "duckDB",
+        "path": "~/test"
+      },
+      "authenticationStrategy": {
+        "_type": "test"
+      },
+      "type": "DuckDB",
+      "element" : "",
+      "databaseType": "DuckDB",
+      "postProcessors": [],
+      "postProcessorWithParameter": []
+    }
+  }
+}
diff --git a/...end-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/pom.xml b/...end-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/pom.xml
@@ -14,7 +14,7 @@
 
     <modules>
         <module>legend-engine-xt-relationalStore-duckdb-execution</module>
-<!--        <module>legend-engine-xt-relationalStore-duckdb-execution-tests</module>-->
+        <module>legend-engine-xt-relationalStore-duckdb-execution-tests</module>
         <module>legend-engine-xt-relationalStore-duckdb-grammar</module>
         <module>legend-engine-xt-relationalStore-duckdb-protocol</module>
         <module>legend-engine-xt-relationalStore-duckdb-pure</module>

diff --git a/pom.xml b/pom.xml
@@ -1173,6 +1173,11 @@
                 <artifactId>legend-engine-xt-relationalStore-duckdb-execution</artifactId>
                 <version>${project.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.finos.legend.engine</groupId>
+                <artifactId>legend-engine-xt-relationalStore-duckdb-execution-tests</artifactId>
+                <version>${project.version}</version>
+            </dependency>
 
             <dependency>
                 <groupId>org.finos.legend.engine</groupId>