Merge branch 'finos:master' into review-conn-eviction-fix

.github/workflows/database-duckdb-sql-generation-integration-test.yml

@@ -0,0 +1,98 @@
+# Copyright 2024 Goldman Sachs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Database DuckDB SQL Generation Integration Tests
+
+env:
+  CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }}
+  CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+
+on:
+  schedule:
+    - cron: "0 0 * * *" # every day at 00:00 on default/base branch
+  workflow_dispatch:
+    inputs:
+      pullRequestId:
+        description: "Pull request id"
+        required: false
+
+jobs:
+  build:
+    # NOTE: we cannot run this action in fork because secrets are not accessible from forks
+    # See https://community.sonarsource.com/t/github-action-ci-build-fail-with-set-the-sonar-token-env-variable/38997
+    if: github.repository == 'finos/legend-engine'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+      - name: Set up JDK
+        uses: actions/setup-java@v1
+        with:
+          java-version: 11
+          server-id: ossrh
+          server-username: CI_DEPLOY_USERNAME
+          server-password: CI_DEPLOY_PASSWORD
+      - name: Configure git
+        run: |
+          git config --global committer.email "infra@finos.org"
+          git config --global committer.name "FINOS Admin"
+          git config --global author.email "${GITHUB_ACTOR}@users.noreply.github.com"
+          git config --global author.name "${GITHUB_ACTOR}"
+      - name: Download patch
+        if: github.event.inputs.pullRequestId != ''
+        run: |
+          wget --output-document=run.diff http://patch-diff.githubusercontent.com/raw/finos/legend-engine/pull/${{ github.event.inputs.pullRequestId }}.diff
+          git apply run.diff
+      - name: Print out mvn version
+        run: mvn -v
+      - name: Download deps and plugins
+        run: mvn de.qaware.maven:go-offline-maven-plugin:resolve-dependencies
+      - name: Build
+        env:
+          MAVEN_OPTS: -Xmx4g
+        run: mvn -pl legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution-tests -am clean install -DskipTests=true
+      - name: SQL Generation Integration Tests
+        env:
+          MAVEN_OPTS: -Xmx4g
+        run: |
+          set -o pipefail
+          echo "| Test Representing Db Feature | Support Status |" >> $GITHUB_STEP_SUMMARY
+          echo "| ---------------------------- | -------------- |" >> $GITHUB_STEP_SUMMARY
+          mvn -pl legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution-tests -Dtest=Test_Relational_DbSpecific_DuckDB_UsingPureClientTestSuite test | tee >(grep -o "Tests run:[^T]*" | head -1 | tr -d '\n' >> sql_test_summary.txt) | tee >(echo "Ignored tests deviating from standard: $(grep -c deviating-from-standard)" >> sql_test_summary.txt) | tee >(grep -o "[|] [*][*].*" >> $GITHUB_STEP_SUMMARY)
+      - name: Write Summary
+        if: always()
+        run: |
+          touch sql_test_summary.txt
+          cat sql_test_summary.txt >> $GITHUB_STEP_SUMMARY
+      - name: Upload Summary
+        if: always() && (github.event.inputs.pullRequestId == '')
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{github.workflow}}
+          path: sql_test_summary.txt
+          retention-days: 1
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results
+          path: legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-duckdb/legend-engine-xt-relationalStore-duckdb-execution-tests/target/surefire-reports/*.xml
+      - name: Upload CI Event
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: event-file
+          path: ${{ github.event_path }}          

.github/workflows/database-h2-sql-generation-integration-test.yml

@@ -63,15 +63,15 @@ jobs:
       - name: Build
         env:
           MAVEN_OPTS: -Xmx4g
-        run: mvn -pl legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-server -am clean install -DskipTests=true
+        run: mvn -pl legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-http-server -am clean install -DskipTests=true
       - name: SQL Generation Integration Tests
         env:
           MAVEN_OPTS: -Xmx4g
         run: |
           set -o pipefail
           echo "| Test Representing Db Feature | Support Status |" >> $GITHUB_STEP_SUMMARY
           echo "| ---------------------------- | -------------- |" >> $GITHUB_STEP_SUMMARY
-          mvn -pl legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-server -Dtest=Test_Relational_DbSpecific_H2_UsingPureClientTestSuite test | tee >(grep -o "Tests run:[^T]*" | head -1 | tr -d '\n' >> sql_test_summary.txt) | tee >(echo "Ignored tests deviating from standard: $(grep -c deviating-from-standard)" >> sql_test_summary.txt) | tee >(grep -o "[|] [*][*].*" >> $GITHUB_STEP_SUMMARY)
+          mvn -pl legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-http-server -Dtest=Test_Relational_DbSpecific_H2_UsingPureClientTestSuite test | tee >(grep -o "Tests run:[^T]*" | head -1 | tr -d '\n' >> sql_test_summary.txt) | tee >(echo "Ignored tests deviating from standard: $(grep -c deviating-from-standard)" >> sql_test_summary.txt) | tee >(grep -o "[|] [*][*].*" >> $GITHUB_STEP_SUMMARY)
       - name: Write Summary
         if: always()
         run: |
@@ -89,7 +89,7 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: test-results
-          path: legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-server/target/surefire-reports/*.xml
+          path: legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-http-server/target/surefire-reports/*.xml
       - name: Upload CI Event
         if: always()
         uses: actions/upload-artifact@v3

docs/store/extensions/Relational/new-connector-tutorial.md

@@ -167,7 +167,7 @@ If your database supports it, we can alternatively launch a test instance at run
      
     package org.finos.legend.engine.dynamicTestConnections;
      
-    import org.finos.legend.engine.plan.execution.stores.relational.connection.tests.api.DynamicTestConnection;
+    import org.finos.legend.engine.plan.execution.stores.relational.connection.tests.api.TestConnectionIntegration;
     import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseType;
     import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection;
     import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.authentication.UserNamePasswordAuthenticationStrategy;
@@ -267,7 +267,7 @@ If your database supports it, we can alternatively launch a test instance at run
     </dependency>
     ~~~
 
-    Also, we need to define the file "org.finos.legend.engine.plan.execution.stores.relational.connection.tests.api.DynamicTestConnection" in src/main/resources/META-INF/services with this content
+    Also, we need to define the file "org.finos.legend.engine.plan.execution.stores.relational.connection.tests.api.TestConnectionIntegration" in src/main/resources/META-INF/services with this content
 
     ~~~
     org.finos.legend.engine.dynamicTestConnections.SqlServerUsingTestContainer

legend-engine-application-query/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.finos.legend.engine</groupId>
         <artifactId>legend-engine</artifactId>
-        <version>4.46.1-SNAPSHOT</version>
+        <version>4.47.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>legend-engine-application-query</artifactId>
@@ -45,6 +45,10 @@
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-xt-identity-pac4j</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-shared-vault-core</artifactId>
+        </dependency>
         <!-- ENGINE -->
 
         <!-- JACKSON -->

legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/api/ApplicationQuery.java

@@ -25,7 +25,6 @@
 import org.finos.legend.engine.application.query.model.QueryEvent;
 import org.finos.legend.engine.application.query.model.QuerySearchSpecification;
 import org.finos.legend.engine.shared.core.identity.Identity;
-import org.finos.legend.engine.shared.core.identity.factory.IdentityFactoryProvider;
 import org.finos.legend.engine.shared.core.kerberos.ProfileManagerHelper;
 import org.finos.legend.engine.shared.core.operational.errorManagement.ExceptionTool;
 import org.finos.legend.engine.shared.core.operational.logs.LoggingEventType;
@@ -150,7 +149,7 @@ public Response getQueryCount()
     public Response createQuery(Query query, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> profileManager)
     {
         MutableList<CommonProfile> profiles = ProfileManagerHelper.extractProfiles(profileManager);
-        Identity identity = IdentityFactoryProvider.getInstance().makeIdentity(profiles);
+        Identity identity = Identity.makeIdentity(profiles);
         try (Scope scope = GlobalTracer.get().buildSpan("Query: Create Query").startActive(true))
         {
             return Response.ok().entity(this.queryStoreManager.createQuery(query, getCurrentUser(profileManager))).build();
@@ -172,7 +171,7 @@ public Response createQuery(Query query, @ApiParam(hidden = true) @Pac4JProfileM
     public Response updateQuery(@PathParam("queryId") String queryId, Query query, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> profileManager)
     {
         MutableList<CommonProfile> profiles = ProfileManagerHelper.extractProfiles(profileManager);
-        Identity identity = IdentityFactoryProvider.getInstance().makeIdentity(profiles);
+        Identity identity = Identity.makeIdentity(profiles);
         try (Scope scope = GlobalTracer.get().buildSpan("Query: Update Query").startActive(true))
         {
             return Response.ok().entity(this.queryStoreManager.updateQuery(queryId, query, getCurrentUser(profileManager))).build();
@@ -194,7 +193,7 @@ public Response updateQuery(@PathParam("queryId") String queryId, Query query, @
     public Response patchQuery(@PathParam("queryId") String queryId, Query query, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> profileManager)
     {
         MutableList<CommonProfile> profiles = ProfileManagerHelper.extractProfiles(profileManager);
-        Identity identity = IdentityFactoryProvider.getInstance().makeIdentity(profiles);
+        Identity identity = Identity.makeIdentity(profiles);
         try (Scope scope = GlobalTracer.get().buildSpan("Patch Query - update selected query fields").startActive(true))
         {
             return Response.ok().entity(this.queryStoreManager.patchQuery(queryId, query, getCurrentUser(profileManager))).build();
@@ -216,7 +215,7 @@ public Response patchQuery(@PathParam("queryId") String queryId, Query query, @A
     public Response deleteQuery(@PathParam("queryId") String queryId, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> profileManager)
     {
         MutableList<CommonProfile> profiles = ProfileManagerHelper.extractProfiles(profileManager);
-        Identity identity = IdentityFactoryProvider.getInstance().makeIdentity(profiles);
+        Identity identity = Identity.makeIdentity(profiles);
         try (Scope scope = GlobalTracer.get().buildSpan("Query: Delete Query").startActive(true))
         {
             this.queryStoreManager.deleteQuery(queryId, getCurrentUser(profileManager));

legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/api/QueryStoreManager.java

@@ -23,6 +23,7 @@
 import com.mongodb.client.model.Aggregates;
 import com.mongodb.client.model.Field;
 import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.Updates;
 import com.mongodb.client.model.Projections;
 import com.mongodb.client.model.Sorts;
 import org.bson.Document;
@@ -298,7 +299,7 @@ else if (count == 0)
         return matchingQueries;
     }
 
-    public Query getQuery(String queryId)
+    public Query getQuery(String queryId) throws JsonProcessingException
     {
         List<Query> matchingQueries = LazyIterate.collect(this.getQueryCollection().find(Filters.eq("id", queryId)), this::documentToQuery).toList();
         if (matchingQueries.size() > 1)
@@ -309,7 +310,13 @@ else if (matchingQueries.size() == 0)
         {
             throw new ApplicationQueryException("Can't find query with ID '" + queryId + "'", Response.Status.NOT_FOUND);
         }
-        return matchingQueries.get(0);
+        Query query = matchingQueries.get(0);
+        query.lastOpenAt = Instant.now().toEpochMilli();
+        this.getQueryCollection().updateOne(
+                Filters.eq("id", queryId),
+                Updates.set("lastOpenAt", Instant.now().toEpochMilli())
+        );
+        return query;
     }
 
     public QueryStoreStats getQueryStoreStats() throws JsonProcessingException
@@ -339,6 +346,7 @@ public Query createQuery(Query query, String currentUser) throws JsonProcessingE
         }
         query.createdAt = Instant.now().toEpochMilli();
         query.lastUpdatedAt = query.createdAt;
+        query.lastOpenAt = query.createdAt;
         this.getQueryCollection().insertOne(queryToDocument(query));
         QueryEvent createdEvent = createEvent(query.id, QueryEvent.QueryEventType.CREATED);
         createdEvent.timestamp = query.createdAt;
@@ -374,6 +382,7 @@ else if (matchingQueries.size() == 0)
         query.owner = currentUser;
         query.createdAt = currentQuery.createdAt;
         query.lastUpdatedAt = Instant.now().toEpochMilli();
+        query.lastOpenAt = Instant.now().toEpochMilli();
         query.originalVersionId = currentQuery.originalVersionId;
         this.getQueryCollection().findOneAndReplace(Filters.eq("id", queryId), queryToDocument(query));
         QueryEvent updatedEvent = createEvent(query.id, QueryEvent.QueryEventType.UPDATED);
@@ -411,6 +420,7 @@ public Query patchQuery(String queryId, Query updatedQuery, String currentUser)
         }
         currentQuery.owner = currentUser;
         currentQuery.lastUpdatedAt = Instant.now().toEpochMilli();
+        currentQuery.lastOpenAt = Instant.now().toEpochMilli();
         this.getQueryCollection().findOneAndReplace(Filters.eq("id", queryId), queryToDocument(currentQuery));
         QueryEvent updatedEvent = createEvent(queryId, QueryEvent.QueryEventType.UPDATED);
         updatedEvent.timestamp = currentQuery.lastUpdatedAt;

legend-engine-application-query/src/main/java/org/finos/legend/engine/application/query/model/Query.java

@@ -39,6 +39,7 @@ public class Query
     public String content;
     public Long lastUpdatedAt;
     public Long createdAt;
+    public Long lastOpenAt;
 
     public List<TaggedValue> taggedValues;
     public List<StereotypePtr> stereotypes;

legend-engine-application-query/src/test/java/org/finos/legend/engine/application/query/api/TestQueryStoreManager.java

@@ -576,18 +576,24 @@ public void testGetQueriesWithParameterValues() throws Exception
 
         Query query1 = queryStoreManager.getQuery("1");
         Assert.assertEquals(0, query1.defaultParameterValues.size());
+        Assert.assertNotNull(query1.lastOpenAt);
+        Assert.assertNotEquals(query1.lastOpenAt, query1.createdAt);
 
         Query query2 = queryStoreManager.getQuery("2");
         Assert.assertEquals(1, query2.defaultParameterValues.size());
         Assert.assertEquals("booleanParam1", query2.defaultParameterValues.get(0).name);
         Assert.assertEquals("true", query2.defaultParameterValues.get(0).content);
+        Assert.assertNotNull(query2.lastOpenAt);
+        Assert.assertNotEquals(query2.lastOpenAt, query2.createdAt);
 
         Query query4 = queryStoreManager.getQuery("4");
         Assert.assertEquals(3, query4.defaultParameterValues.size());
         Assert.assertEquals("booleanParam1", query4.defaultParameterValues.get(0).name);
         Assert.assertEquals("stringParam2", query4.defaultParameterValues.get(1).name);
         Assert.assertEquals("myListParam3", query4.defaultParameterValues.get(2).name);
         Assert.assertEquals("['d','a']", query4.defaultParameterValues.get(2).content);
+        Assert.assertNotNull(query4.lastOpenAt);
+        Assert.assertNotEquals(query4.lastOpenAt, query4.createdAt);
     }
 
 
@@ -735,7 +741,7 @@ public void testCreateSimpleQuery() throws Exception
         Assert.assertEquals(0, createdQuery.taggedValues.size());
         Assert.assertNotNull(createdQuery.createdAt);
         Assert.assertEquals(createdQuery.createdAt, createdQuery.lastUpdatedAt);
-
+        Assert.assertEquals(createdQuery.lastOpenAt, createdQuery.lastUpdatedAt);
     }
 
     @Test

legend-engine-config/legend-engine-configuration/legend-engine-configuration-contract-extension-pure/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.finos.legend.engine</groupId>
         <artifactId>legend-engine-configuration</artifactId>
-        <version>4.46.1-SNAPSHOT</version>
+        <version>4.47.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>legend-engine-configuration-contract-extension-pure</artifactId>

legend-engine-config/legend-engine-configuration/legend-engine-configuration-plan-generation-serialization/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <groupId>org.finos.legend.engine</groupId>
         <artifactId>legend-engine-configuration</artifactId>
-        <version>4.46.1-SNAPSHOT</version>
+        <version>4.47.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>legend-engine-configuration-plan-generation-serialization</artifactId>

legend-engine-config/legend-engine-configuration/pom.xml

@@ -18,7 +18,7 @@
     <parent>
         <groupId>org.finos.legend.engine</groupId>
         <artifactId>legend-engine-config</artifactId>
-        <version>4.46.1-SNAPSHOT</version>
+        <version>4.47.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

legend-engine-config/legend-engine-extensions-collection-execution/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <artifactId>legend-engine-config</artifactId>
         <groupId>org.finos.legend.engine</groupId>
-        <version>4.46.1-SNAPSHOT</version>
+        <version>4.47.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

legend-engine-config/legend-engine-extensions-collection-generation/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <artifactId>legend-engine-config</artifactId>
         <groupId>org.finos.legend.engine</groupId>
-        <version>4.46.1-SNAPSHOT</version>
+        <version>4.47.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

legend-engine-config/legend-engine-repl/legend-engine-repl-client/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <artifactId>legend-engine-repl</artifactId>
         <groupId>org.finos.legend.engine</groupId>
-        <version>4.46.1-SNAPSHOT</version>
+        <version>4.47.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

legend-engine-config/legend-engine-repl/legend-engine-repl-client/src/main/java/org/finos/legend/engine/repl/core/Helpers.java

@@ -16,7 +16,6 @@
 
 import org.finos.legend.engine.repl.client.Client;
 import org.finos.legend.engine.shared.core.identity.Identity;
-import org.finos.legend.engine.shared.core.identity.factory.IdentityFactoryProvider;
 import org.finos.legend.engine.shared.core.kerberos.SubjectTools;
 
 public class Helpers
@@ -25,7 +24,7 @@ public static Identity resolveIdentityFromLocalSubject(Client client)
     {
         try
         {
-            return IdentityFactoryProvider.getInstance().makeIdentity(SubjectTools.getLocalSubject());
+            return Identity.makeIdentity(SubjectTools.getLocalSubject());
         }
         catch (Exception e)
         {

legend-engine-config/legend-engine-repl/legend-engine-repl-relational/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <artifactId>legend-engine-repl</artifactId>
         <groupId>org.finos.legend.engine</groupId>
-        <version>4.46.1-SNAPSHOT</version>
+        <version>4.47.4-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>