From 2bd24584fef860c687857fb0c128803c93422580 Mon Sep 17 00:00:00 2001 From: Max Postema <46395349+MaxPostema@users.noreply.github.com> Date: Wed, 9 Oct 2024 13:45:31 +0200 Subject: [PATCH 1/9] fix: add enable/disable user migration script (#4327) * fix: add enable/disable user migration script * formating --- .../src/main/java/org/molgenis/emx2/sql/Migrations.java | 6 +++++- .../src/main/java/org/molgenis/emx2/sql/SqlDatabase.java | 3 ++- .../main/resources/org/molgenis/emx2/sql/migration23.sql | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 backend/molgenis-emx2-sql/src/main/resources/org/molgenis/emx2/sql/migration23.sql diff --git a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/Migrations.java b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/Migrations.java index f8d61b7a50..5c28fbca44 100644 --- a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/Migrations.java +++ b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/Migrations.java @@ -22,7 +22,7 @@ public class Migrations { // version the current software needs to work - private static final int SOFTWARE_DATABASE_VERSION = 22; + private static final int SOFTWARE_DATABASE_VERSION = 23; public static final int THREE_MINUTES = 180; private static Logger logger = LoggerFactory.getLogger(Migrations.class); @@ -148,6 +148,10 @@ public static synchronized void initOrMigrate(SqlDatabase db) { } } + if (version < 23) { + executeMigrationFile(tdb, "migration23.sql", "add enable state to user metadata"); + } + // if success, update version to SOFTWARE_DATABASE_VERSION updateDatabaseVersion((SqlDatabase) tdb, SOFTWARE_DATABASE_VERSION); }); diff --git a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java index 5c375bba8b..172b37521b 100644 --- a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java +++ b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java @@ -493,7 +493,8 @@ public void removeUser(String user) { public void setEnabledUser(String user, Boolean enabled) { long start = System.currentTimeMillis(); if (user.equals("admin")) throw new MolgenisException("You cant enable or disable admin"); - if (user.equals("anonymous")) throw new MolgenisException("You cant enable or disable anonymous"); + if (user.equals("anonymous")) + throw new MolgenisException("You cant enable or disable anonymous"); if (!hasUser(user)) throw new MolgenisException( (enabled ? "Enabling" : "Disabling") diff --git a/backend/molgenis-emx2-sql/src/main/resources/org/molgenis/emx2/sql/migration23.sql b/backend/molgenis-emx2-sql/src/main/resources/org/molgenis/emx2/sql/migration23.sql new file mode 100644 index 0000000000..f17a21760f --- /dev/null +++ b/backend/molgenis-emx2-sql/src/main/resources/org/molgenis/emx2/sql/migration23.sql @@ -0,0 +1,2 @@ +ALTER TABLE "MOLGENIS"."users_metadata" + ADD COLUMN IF NOT EXISTS enabled BOOLEAN DEFAULT TRUE; \ No newline at end of file From 70caef15a8aa575f8bea19bb7167fbff2e3a39c2 Mon Sep 17 00:00:00 2001 From: Harm Brugge Date: Wed, 9 Oct 2024 13:54:55 +0200 Subject: [PATCH 2/9] feat(emx2) short time sorted auto-id using snowflake algorithm (#4258) * implemented SnowFlake IdGenerator * implemented SnowFlake IdGenerator * implemented SnowFlake IdGenerator * implemented SnowFlake IdGenerator * added snowflake id test * refactor to singleton * Use SnowflakeIdGenerator in SqlTypeUtils * Instance ids for SnowFlakeIdGenerator * implemented random instance id * added extraction methods for timestamp and instance id * Disabled ols tests * init snowflake id generator on sql database init * init snowflake id generator on sql database init * init snowflake id generator on sql database init * snowflake test checks if singleton is instantiated * snowflake test checks if singleton is instantiated * snowflake test checks if singleton is instantiated * Refactor random to class variable. * Refactor random to secure random --------- Co-authored-by: Morris Swertz --- .../org/molgenis/emx2/sql/SqlDatabase.java | 10 ++ .../org/molgenis/emx2/sql/SqlTypeUtils.java | 16 +- .../emx2/sql/TestDatabaseFactory.java | 5 +- .../emx2/sql/SnowflakeIdGeneratorTest.java | 97 +++++++++++ .../molgenis/emx2/sql/TestSqlTypeUtils.java | 10 ++ .../java/org/molgenis/emx2/Constants.java | 1 + .../utils/generator/SnowflakeIdGenerator.java | 155 ++++++++++++++++++ 7 files changed, 283 insertions(+), 11 deletions(-) create mode 100644 backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/SnowflakeIdGeneratorTest.java create mode 100644 backend/molgenis-emx2/src/main/java/org/molgenis/emx2/utils/generator/SnowflakeIdGenerator.java diff --git a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java index 172b37521b..6c0fdfabaf 100644 --- a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java +++ b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java @@ -9,6 +9,7 @@ import static org.molgenis.emx2.sql.SqlSchemaMetadataExecutor.executeCreateSchema; import com.zaxxer.hikari.HikariDataSource; +import java.security.SecureRandom; import java.util.*; import java.util.function.Supplier; import javax.sql.DataSource; @@ -20,6 +21,7 @@ import org.molgenis.emx2.*; import org.molgenis.emx2.utils.EnvironmentProperty; import org.molgenis.emx2.utils.RandomString; +import org.molgenis.emx2.utils.generator.SnowflakeIdGenerator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +35,7 @@ public class SqlDatabase extends HasSettings implements Database { public static final int TEN_SECONDS = 10; private static final Settings DEFAULT_JOOQ_SETTINGS = new Settings().withQueryTimeout(TEN_SECONDS); + private static final Random random = new SecureRandom(); // shared between all instances private static DataSource source; @@ -181,6 +184,13 @@ public void init() { // setup default stuff this.setSetting(Constants.IS_OIDC_ENABLED, String.valueOf(isOidcEnabled)); } + String instanceId = getSetting(Constants.MOLGENIS_INSTANCE_ID); + if (instanceId == null) { + instanceId = String.valueOf(random.nextLong(SnowflakeIdGenerator.MAX_ID)); + this.setSetting(Constants.MOLGENIS_INSTANCE_ID, instanceId); + } + if (!SnowflakeIdGenerator.hasInstance()) SnowflakeIdGenerator.init(instanceId); + if (getSetting(Constants.IS_PRIVACY_POLICY_ENABLED) == null) { this.setSetting(Constants.IS_PRIVACY_POLICY_ENABLED, String.valueOf(false)); } diff --git a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlTypeUtils.java b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlTypeUtils.java index a6df428810..9b28ffee65 100644 --- a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlTypeUtils.java +++ b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlTypeUtils.java @@ -9,13 +9,10 @@ import java.util.stream.Collectors; import org.molgenis.emx2.*; import org.molgenis.emx2.utils.TypeUtils; -import org.molgenis.emx2.utils.generator.IdGenerator; -import org.molgenis.emx2.utils.generator.IdGeneratorImpl; +import org.molgenis.emx2.utils.generator.SnowflakeIdGenerator; public class SqlTypeUtils extends TypeUtils { - public static final IdGenerator idGenerator = new IdGeneratorImpl(); - private SqlTypeUtils() { // to hide the public constructor } @@ -34,7 +31,7 @@ public static void applyValidationAndComputed(List columns, Row row) { row.setString( c.getName(), Constants.MG_USER_PREFIX + row.getString(Constants.MG_EDIT_ROLE)); } else if (AUTO_ID.equals(c.getColumnType())) { - applyAutoid(c, row); + applyAutoId(c, row); } else if (c.getDefaultValue() != null && !row.notNull(c.getName())) { if (c.getDefaultValue().startsWith("=")) { try { @@ -79,16 +76,15 @@ public static void applyValidationAndComputed(List columns, Row row) { } } - private static void applyAutoid(Column c, Row row) { + private static void applyAutoId(Column c, Row row) { if (row.isNull(c.getName(), c.getPrimitiveColumnType())) { + String id = SnowflakeIdGenerator.getInstance().generateId(); // do we use a template containing ${mg_autoid} for pre/postfixing ? if (c.getComputed() != null) { - row.set( - c.getName(), - c.getComputed().replace(Constants.COMPUTED_AUTOID_TOKEN, idGenerator.generateId())); + row.set(c.getName(), c.getComputed().replace(Constants.COMPUTED_AUTOID_TOKEN, id)); } // otherwise simply put the id - else row.set(c.getName(), idGenerator.generateId()); + else row.set(c.getName(), id); } } diff --git a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/TestDatabaseFactory.java b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/TestDatabaseFactory.java index 804df31e2e..3706c45185 100644 --- a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/TestDatabaseFactory.java +++ b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/TestDatabaseFactory.java @@ -1,13 +1,16 @@ package org.molgenis.emx2.sql; import org.molgenis.emx2.Database; +import org.molgenis.emx2.utils.generator.SnowflakeIdGenerator; public class TestDatabaseFactory { public static Database getTestDatabase() { Database db = new SqlDatabase(false); db.setActiveUser(db.getAdminUserName()); + + if (!SnowflakeIdGenerator.hasInstance()) SnowflakeIdGenerator.init("123"); + return db; - // don't share the database between tests because when setting active user that leads to errors } } diff --git a/backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/SnowflakeIdGeneratorTest.java b/backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/SnowflakeIdGeneratorTest.java new file mode 100644 index 0000000000..d372fd4115 --- /dev/null +++ b/backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/SnowflakeIdGeneratorTest.java @@ -0,0 +1,97 @@ +package org.molgenis.emx2.sql; + +import static org.junit.jupiter.api.Assertions.*; + +import java.time.Instant; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.molgenis.emx2.utils.generator.SnowflakeIdGenerator; + +public class SnowflakeIdGeneratorTest { + + private static SnowflakeIdGenerator generator; + + @BeforeAll + public static void before() { + if (!SnowflakeIdGenerator.hasInstance()) { + generator = SnowflakeIdGenerator.init("123"); + } else { + generator = SnowflakeIdGenerator.getInstance(); + } + } + + @Test + public void testIdGenerationIsUnique() { + Set generatedIds = new HashSet<>(); + + int totalIds = 100000; + for (int i = 0; i < totalIds; i++) { + String id = generator.generateId(); + assertFalse(generatedIds.contains(id), "Duplicate ID found: " + id); + generatedIds.add(id); + } + } + + @Test + public void testIdGenerationIsSorted() { + String[] generatedIds = new String[100000]; + + for (int i = 0; i < generatedIds.length; i++) { + generatedIds[i] = generator.generateId(); + } + + // Check if IDs are sorted + for (int i = 1; i < generatedIds.length; i++) { + assertTrue( + generatedIds[i - 1].compareTo(generatedIds[i]) <= 0, + "IDs are not sorted at index " + (i - 1) + " and " + i); + } + } + + @Test + public void testUniqueIdsOnFourThreads() throws InterruptedException { + Set uniqueIds = new HashSet<>(); + + ExecutorService executorService = Executors.newFixedThreadPool(4); + + for (int i = 0; i < 100000; i++) { + executorService.submit( + () -> { + String id = generator.generateId(); + synchronized (uniqueIds) { + uniqueIds.add(id); + } + }); + } + + // Shut down the executor and wait for tasks to finish + executorService.shutdown(); + executorService.awaitTermination(1, TimeUnit.MINUTES); + + // Verify that the number of unique IDs matches the total number of generated IDs + assertEquals(100000, uniqueIds.size(), "Not all generated IDs are unique."); + } + + @Test + public void testExtractionInstanceIdFromSnowflake() { + String snowflakeId = generator.generateId(); + + String instanceIdFromSnowflake = SnowflakeIdGenerator.extractInstanceId(snowflakeId); + assertEquals(generator.getInstanceId(), instanceIdFromSnowflake); + } + + @Test + public void testExtractTimestampFromSnowflake() { + long currentTime = Instant.now().toEpochMilli(); + String snowflakeId = generator.generateId(); + + long snowflakeTimestamp = SnowflakeIdGenerator.extractTimestamp(snowflakeId); + // Allow a mismatch of 1ms + assertTrue(Math.abs(currentTime - snowflakeTimestamp) <= 1); + } +} diff --git a/backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/TestSqlTypeUtils.java b/backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/TestSqlTypeUtils.java index 25a877a414..df04bfe32a 100644 --- a/backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/TestSqlTypeUtils.java +++ b/backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/TestSqlTypeUtils.java @@ -4,14 +4,24 @@ import static org.molgenis.emx2.TableMetadata.table; import static org.molgenis.emx2.sql.SqlTypeUtils.applyValidationAndComputed; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.molgenis.emx2.*; +import org.molgenis.emx2.utils.generator.SnowflakeIdGenerator; class TestSqlTypeUtils { + @BeforeAll + static void before() { + if (!SnowflakeIdGenerator.hasInstance()) { + SnowflakeIdGenerator.init("123"); + } + } + @Test void autoIdGetsGenerated() { TableMetadata tableMetadata = table("Test", new Column("myCol").setType(ColumnType.AUTO_ID)); + final Row row = new Row("myCol", null); applyValidationAndComputed(tableMetadata.getColumns(), row); assertNotNull(row.getString("myCol")); diff --git a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Constants.java b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Constants.java index 82c9b7b88a..74f5dc79a0 100644 --- a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Constants.java +++ b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Constants.java @@ -49,6 +49,7 @@ public class Constants { public static final String MOLGENIS_POSTGRES_PASS = "MOLGENIS_POSTGRES_PASS"; public static final String MOLGENIS_HTTP_PORT = "MOLGENIS_HTTP_PORT"; public static final String MOLGENIS_ADMIN_PW = "MOLGENIS_ADMIN_PW"; + public static final String MOLGENIS_INSTANCE_ID = "MOLGENIS_INSTANCE_ID"; public static final String IS_OIDC_ENABLED = "isOidcEnabled"; public static final String MOLGENIS_OIDC_CLIENT_ID = "MOLGENIS_OIDC_CLIENT_ID"; diff --git a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/utils/generator/SnowflakeIdGenerator.java b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/utils/generator/SnowflakeIdGenerator.java new file mode 100644 index 0000000000..f43a4d0f73 --- /dev/null +++ b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/utils/generator/SnowflakeIdGenerator.java @@ -0,0 +1,155 @@ +package org.molgenis.emx2.utils.generator; + +import java.math.BigInteger; +import java.time.Instant; +import org.molgenis.emx2.MolgenisException; + +public class SnowflakeIdGenerator implements IdGenerator { + + private static SnowflakeIdGenerator instance; + private final String instanceId; + + private static final String BASE62_CHARACTERS = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + private static final long EPOCH = 1727740800000L; // 1 October 2024 00:00:00 + + public static final int TOTAL_BITS = 64; + public static final int TIMESTAMP_BITS = 41; // 69 years of ids + public static final int SEQUENCE_BITS = 8; // 2^8 (256) ids per ms + public static final int ID_BITS = + TOTAL_BITS + - TIMESTAMP_BITS + - SEQUENCE_BITS; // 15 bits left for the instanceId ~= 32K unique hashes + + public static final long MAX_TIMESTAMP = maxValueForBits(TIMESTAMP_BITS); + public static final long MAX_SEQUENCE = maxValueForBits(SEQUENCE_BITS); + public static final long MAX_ID = maxValueForBits(ID_BITS); + + private long lastTimestamp = -1L; + private long sequence = 0L; + + private SnowflakeIdGenerator(String instanceId) { + this.instanceId = instanceId; + } + + public static synchronized SnowflakeIdGenerator getInstance() { + if (instance == null) { + throw new MolgenisException("SnowflakeIdGenerator not initialized"); + } + return instance; + } + + public static synchronized SnowflakeIdGenerator init(String instanceId) { + if (instance != null) { + throw new MolgenisException("SnowflakeIdGenerator is already initialized"); + } + instance = new SnowflakeIdGenerator(instanceId); + return instance; + } + + public static boolean hasInstance() { + return instance != null; + } + + public synchronized String generateId() { + long currentTimestamp = getCurrentTimestamp(); + + if (currentTimestamp == lastTimestamp) { + sequence = (sequence + 1) & MAX_SEQUENCE; + if (sequence == 0) { + currentTimestamp = waitForNextMillis(currentTimestamp); + } + } else { + sequence = 0; + } + lastTimestamp = currentTimestamp; + + long instanceIdBits = getInstanceIdBits(instanceId); + + BigInteger timePart = BigInteger.valueOf(currentTimestamp).shiftLeft(ID_BITS + SEQUENCE_BITS); + BigInteger instancePart = BigInteger.valueOf(instanceIdBits).shiftLeft(SEQUENCE_BITS); + BigInteger sequencePart = BigInteger.valueOf(sequence); + + BigInteger snowflakeId = timePart.or(instancePart).or(sequencePart); + + return base62Encode(snowflakeId); + } + + public static String extractInstanceId(String snowflakeId) { + BigInteger decodedSnowflakeId = base62Decode(snowflakeId); + BigInteger instancePart = decodedSnowflakeId.shiftRight(SEQUENCE_BITS); + BigInteger instanceIdBits = instancePart.and(BigInteger.valueOf(MAX_ID)); + + return instanceIdBits.toString(); + } + + public static long extractTimestamp(String snowflakeId) { + BigInteger decodedSnowflakeId = base62Decode(snowflakeId); + BigInteger timestampPart = decodedSnowflakeId.shiftRight(ID_BITS + SEQUENCE_BITS); + + return timestampPart.longValue() + EPOCH; + } + + private static long getInstanceIdBits(String instanceId) { + long instanceIdBits = Long.parseLong(instanceId); + if (instanceIdBits > MAX_ID) { + throw new MolgenisException("Instance id too large: " + instanceId); + } + return instanceIdBits; + } + + private static long maxValueForBits(int bits) { + return (1L << bits) - 1; + } + + private static String base62Encode(BigInteger snowflakeId) { + StringBuilder encoded = new StringBuilder(); + BigInteger base = BigInteger.valueOf(BASE62_CHARACTERS.length()); + + while (snowflakeId.compareTo(BigInteger.ZERO) > 0) { + int remainder = snowflakeId.mod(base).intValue(); + encoded.insert(0, BASE62_CHARACTERS.charAt(remainder)); + + snowflakeId = snowflakeId.divide(base); + } + + return encoded.toString(); + } + + private static BigInteger base62Decode(String encoded) { + BigInteger result = BigInteger.ZERO; + BigInteger base = BigInteger.valueOf(BASE62_CHARACTERS.length()); + + for (int i = 0; i < encoded.length(); i++) { + int charIndex = BASE62_CHARACTERS.indexOf(encoded.charAt(i)); + if (charIndex == -1) { + throw new IllegalArgumentException("Invalid character in Base62 encoded string"); + } + result = result.multiply(base).add(BigInteger.valueOf(charIndex)); + } + + return result; + } + + private long getCurrentTimestamp() { + long now = Instant.now().toEpochMilli(); + long currentTimestamp = now - EPOCH; + + if (currentTimestamp > MAX_TIMESTAMP) { + throw new MolgenisException("Snowflake id too old"); + } + return currentTimestamp; + } + + private long waitForNextMillis(long currentTimestamp) { + long newTimestamp = getCurrentTimestamp(); + while (newTimestamp <= currentTimestamp) { + newTimestamp = getCurrentTimestamp(); + } + return newTimestamp; + } + + public String getInstanceId() { + return this.instanceId; + } +} From 41fa6f40588ae909a4ab46802472881880d69ddb Mon Sep 17 00:00:00 2001 From: connoratrug <47183404+connoratrug@users.noreply.github.com> Date: Wed, 9 Oct 2024 16:57:07 +0200 Subject: [PATCH 3/9] chore: bump playwright image (#4326) bump playwright image and dependancies --- .circleci/config.yml | 7 +++--- apps/nuxt3-ssr/package.json | 4 ++-- apps/tailwind-components/package.json | 2 +- apps/yarn.lock | 33 ++++++++++++--------------- e2e/package.json | 2 +- 5 files changed, 22 insertions(+), 26 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 251a81ce82..398173d5de 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -246,11 +246,10 @@ jobs: - post_steps e2e-test: docker: - - image: mcr.microsoft.com/playwright:v1.47.0-jammy + - image: mcr.microsoft.com/playwright:v1.48.0-jammy steps: - checkout - run: npm i -D @playwright/test - - run: npx playwright install chrome - run: name: Install dependencies command: yarn install @@ -274,6 +273,7 @@ jobs: - run: name: Test nuxt ssr catalogue tests command: | + yarn playwright install chrome echo "PR number: ${CIRCLE_PULL_REQUEST##*/}" export E2E_BASE_URL=https://preview-emx2-pr-${CIRCLE_PULL_REQUEST##*/}.dev.molgenis.org/ echo $E2E_BASE_URL @@ -281,7 +281,8 @@ jobs: working_directory: apps/nuxt3-ssr - run: name: Run non ssr e2e tests - command: | + command: | + npx playwright install chrome echo "PR number: ${CIRCLE_PULL_REQUEST##*/}" export E2E_BASE_URL=https://preview-emx2-pr-${CIRCLE_PULL_REQUEST##*/}.dev.molgenis.org/ echo $E2E_BASE_URL diff --git a/apps/nuxt3-ssr/package.json b/apps/nuxt3-ssr/package.json index 777e9be738..931cca4be2 100644 --- a/apps/nuxt3-ssr/package.json +++ b/apps/nuxt3-ssr/package.json @@ -20,7 +20,7 @@ "devDependencies": { "@nuxt/image": "1.7.0", "@nuxt/test-utils": "3.14.2", - "@playwright/test": "1.47.1", + "@playwright/test": "1.48.0", "@tailwindcss/forms": "0.5.7", "@tailwindcss/typography": "0.5.13", "@vitejs/plugin-vue": "4.6.2", @@ -31,7 +31,7 @@ "if-env": "^1.0.4", "jsdom": "22.1.0", "metadata-utils": "*", - "playwright-core": "1.47.1", + "playwright-core": "1.48.0", "postcss": "8.4.39", "postcss-custom-properties": "13.3.11", "prettier": "2.8.8", diff --git a/apps/tailwind-components/package.json b/apps/tailwind-components/package.json index 49ac700645..6e61affa47 100644 --- a/apps/tailwind-components/package.json +++ b/apps/tailwind-components/package.json @@ -34,7 +34,7 @@ "if-env": "1.0.4", "pa11y-ci": "^3.1.0", "pa11y-ci-reporter-html": "^7.0.0", - "playwright-core": "1.45.0", + "playwright-core": "1.48.0", "prettier": "2.8.8", "svgo": "2.8.0", "typescript": "5.6.2", diff --git a/apps/yarn.lock b/apps/yarn.lock index 1a670ed59a..2f4fecfb60 100644 --- a/apps/yarn.lock +++ b/apps/yarn.lock @@ -1711,12 +1711,12 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== -"@playwright/test@1.47.1": - version "1.47.1" - resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.47.1.tgz#568a46229a5aef54b74977297a7946bb5ac4b67b" - integrity sha512-dbWpcNQZ5nj16m+A5UNScYx7HX5trIy7g4phrcitn+Nk83S32EBX/CLU4hiF4RGKX/yRc93AAqtfaXB7JWBd4Q== +"@playwright/test@1.48.0": + version "1.48.0" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.48.0.tgz#4b81434a3ca75e2a6f82a645287784223a45434c" + integrity sha512-W5lhqPUVPqhtc/ySvZI5Q8X2ztBOUgZ8LbAFy0JQgrXZs2xaILrUcNO3rQjwbLPfGK13+rZsDa1FpG+tqYkT5w== dependencies: - playwright "1.47.1" + playwright "1.48.0" "@polka/url@^1.0.0-next.24": version "1.0.0-next.27" @@ -10388,22 +10388,17 @@ pkg-up@^2.0.0: dependencies: find-up "^2.1.0" -playwright-core@1.45.0: - version "1.45.0" - resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.45.0.tgz#5741a670b7c9060ce06852c0051d84736fb94edc" - integrity sha512-lZmHlFQ0VYSpAs43dRq1/nJ9G/6SiTI7VPqidld9TDefL9tX87bTKExWZZUF5PeRyqtXqd8fQi2qmfIedkwsNQ== +playwright-core@1.48.0: + version "1.48.0" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.48.0.tgz#34d209dd4aba8fccd4a96116f1c4f7630f868722" + integrity sha512-RBvzjM9rdpP7UUFrQzRwR8L/xR4HyC1QXMzGYTbf1vjw25/ya9NRAVnXi/0fvFopjebvyPzsmoK58xxeEOaVvA== -playwright-core@1.47.1: - version "1.47.1" - resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.47.1.tgz#bb45bdfb0d48412c535501aa3805867282857df8" - integrity sha512-i1iyJdLftqtt51mEk6AhYFaAJCDx0xQ/O5NU8EKaWFgMjItPVma542Nh/Aq8aLCjIJSzjaiEQGW/nyqLkGF1OQ== - -playwright@1.47.1: - version "1.47.1" - resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.47.1.tgz#cdc1116f5265b8d2ff7be0d8942d49900634dc6c" - integrity sha512-SUEKi6947IqYbKxRiqnbUobVZY4bF1uu+ZnZNJX9DfU1tlf2UhWfvVjLf01pQx9URsOr18bFVUKXmanYWhbfkw== +playwright@1.48.0: + version "1.48.0" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.48.0.tgz#00855d9a25f1991d422867f1c32af5d90f457b48" + integrity sha512-qPqFaMEHuY/ug8o0uteYJSRfMGFikhUysk8ZvAtfKmUK3kc/6oNl/y3EczF8OFGYIi/Ex2HspMfzYArk6+XQSA== dependencies: - playwright-core "1.47.1" + playwright-core "1.48.0" optionalDependencies: fsevents "2.3.2" diff --git a/e2e/package.json b/e2e/package.json index 06963d5d92..7dbf15c0a5 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -13,7 +13,7 @@ "test:tw-forms": "playwright test --grep '@tw-forms' --reporter=line" }, "dependencies": { - "@playwright/test": "1.47.1", + "@playwright/test": "1.48.0", "install": "0.13.0" } } From fe87336e33971bf81bebf8514fda0c9a5624e2c9 Mon Sep 17 00:00:00 2001 From: connoratrug <47183404+connoratrug@users.noreply.github.com> Date: Thu, 10 Oct 2024 09:09:51 +0200 Subject: [PATCH 4/9] feat: fetch last update column in single gql query (#4319) feat: Fetch last update column in single gql query --- apps/central/src/components/Groups.vue | 78 ++++++------------ .../src/components/LastUpdateField.vue | 63 ++++---------- .../emx2/graphql/GraphqlApiFactory.java | 4 + .../graphql/GraphqlDatabaseFieldFactory.java | 28 +++++++ .../molgenis/emx2/sql/ChangeLogExecutor.java | 82 ++++++++++++++++--- .../org/molgenis/emx2/sql/SqlDatabase.java | 5 ++ .../emx2/sql/ChangeLogExecutorTest.java | 53 ++++++++++++ .../main/java/org/molgenis/emx2/Database.java | 2 + .../java/org/molgenis/emx2/LastUpdate.java | 6 ++ 9 files changed, 211 insertions(+), 110 deletions(-) create mode 100644 backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/ChangeLogExecutorTest.java create mode 100644 backend/molgenis-emx2/src/main/java/org/molgenis/emx2/LastUpdate.java diff --git a/apps/central/src/components/Groups.vue b/apps/central/src/components/Groups.vue index 50430fd574..a66487715b 100644 --- a/apps/central/src/components/Groups.vue +++ b/apps/central/src/components/Groups.vue @@ -20,20 +20,6 @@ /> - - Show changelog - - - Hide changelog - @@ -90,14 +76,8 @@ @@ -167,8 +147,7 @@ export default { search: null, sortColumn: "name", sortOrder: null, - changelogSchemas: [], - showChangeColumn: false, + lastUpdates: [], }; }, computed: { @@ -186,8 +165,8 @@ export default { this.session.roles.includes("Manager")) ); }, - showChangeColumnButton() { - return this.hasManagerPermission; + showChangeColumn() { + return this.session.email == "admin"; }, }, created() { @@ -219,34 +198,31 @@ export default { }, getSchemaList() { this.loading = true; - request("graphql", "{_schemas{id,label,description}}") + const schemaFragment = "_schemas{id,label,description}"; + const lastUpdateFragment = + "_lastUpdate{schemaName, tableName, stamp, userId, operation}"; + request( + "graphql", + `{${schemaFragment} ${this.showChangeColumn ? lastUpdateFragment : ""}}` + ) .then((data) => { this.schemas = data._schemas; - this.loading = false; - if (this.hasManagerPermission && this.showChangeColumn) { - this.fetchChangelogStatus(); - } - }) - .catch( - (error) => - (this.graphqlError = "internal server graphqlError" + error) - ); - }, - fetchChangelogStatus() { - this.schemas.forEach((schema) => { - request( - `/${schema.id}/settings/graphql`, - `{_settings (keys: ["isChangelogEnabled"]){ key, value }}` - ) - .then((data) => { - if (data._settings[0].value.toLowerCase() === "true") { - this.changelogSchemas.push(schema.id); + const lastUpdates = data._lastUpdate ?? []; + lastUpdates.forEach((lastUpdate) => { + const schemaLastUpdate = this.schemas.find( + (schema) => schema.id === lastUpdate.schemaName + ); + if (schemaLastUpdate) { + schemaLastUpdate.update = lastUpdate; } - }) - .catch((error) => { - console.log(error); }); - }); + this.loading = false; + }) + .catch((error) => { + console.error("internal server error", error); + this.graphqlError = "internal server error" + error; + this.loading = false; + }); }, filterSchema(unfiltered) { let filtered = unfiltered; @@ -268,7 +244,7 @@ export default { if (this.sortColumn === "lastUpdate") { sorted = unsortedCopy.sort((a, b) => { if (a.update && b.update) { - return a.update.getTime() - b.update.getTime(); + return new Date(a.update.stamp) - new Date(b.update.stamp); } else if (a.update && !b.update) { return 1; } else if (!a.update && b.update) { diff --git a/apps/central/src/components/LastUpdateField.vue b/apps/central/src/components/LastUpdateField.vue index dc6c6a86cb..63384bd52f 100644 --- a/apps/central/src/components/LastUpdateField.vue +++ b/apps/central/src/components/LastUpdateField.vue @@ -1,56 +1,23 @@ - diff --git a/backend/molgenis-emx2-graphql/src/main/java/org/molgenis/emx2/graphql/GraphqlApiFactory.java b/backend/molgenis-emx2-graphql/src/main/java/org/molgenis/emx2/graphql/GraphqlApiFactory.java index c4d1b66dac..6b5e2d011f 100644 --- a/backend/molgenis-emx2-graphql/src/main/java/org/molgenis/emx2/graphql/GraphqlApiFactory.java +++ b/backend/molgenis-emx2-graphql/src/main/java/org/molgenis/emx2/graphql/GraphqlApiFactory.java @@ -70,6 +70,10 @@ public GraphQL createGraphqlForDatabase(Database database, TaskService taskServi queryBuilder.field(db.schemasQuery(database)); queryBuilder.field(db.settingsQueryField(database)); queryBuilder.field(db.tasksQueryField(taskService)); + // todo need to allow for owner ? ( need to filter the query to include only owned schema's) + if (database.isAdmin()) { + queryBuilder.field(db.lastUpdateQuery(database)); + } mutationBuilder.field(db.createMutation(database, taskService)); mutationBuilder.field(db.deleteMutation(database)); diff --git a/backend/molgenis-emx2-graphql/src/main/java/org/molgenis/emx2/graphql/GraphqlDatabaseFieldFactory.java b/backend/molgenis-emx2-graphql/src/main/java/org/molgenis/emx2/graphql/GraphqlDatabaseFieldFactory.java index dae024cc68..abf9c9dd2e 100644 --- a/backend/molgenis-emx2-graphql/src/main/java/org/molgenis/emx2/graphql/GraphqlDatabaseFieldFactory.java +++ b/backend/molgenis-emx2-graphql/src/main/java/org/molgenis/emx2/graphql/GraphqlDatabaseFieldFactory.java @@ -19,6 +19,27 @@ public class GraphqlDatabaseFieldFactory { + static final GraphQLType lastUpdateMetadataType = + new GraphQLObjectType.Builder() + .name("ChangesType") + .field( + GraphQLFieldDefinition.newFieldDefinition() + .name(OPERATION) + .type(Scalars.GraphQLString)) + .field( + GraphQLFieldDefinition.newFieldDefinition().name(STAMP).type(Scalars.GraphQLString)) + .field( + GraphQLFieldDefinition.newFieldDefinition().name(USERID).type(Scalars.GraphQLString)) + .field( + GraphQLFieldDefinition.newFieldDefinition() + .name(TABLENAME) + .type(Scalars.GraphQLString)) + .field( + GraphQLFieldDefinition.newFieldDefinition() + .name(SCHEMA_NAME) + .type(Scalars.GraphQLString)) + .build(); + public static final GraphQLType outputSchemasType = new GraphQLObjectType.Builder() .name("SchemaInfo") @@ -268,6 +289,13 @@ public GraphQLFieldDefinition dropMutation(Database database) { .build(); } + public GraphQLFieldDefinition.Builder lastUpdateQuery(Database database) { + return GraphQLFieldDefinition.newFieldDefinition() + .name("_lastUpdate") + .type(GraphQLList.list(lastUpdateMetadataType)) + .dataFetcher(dataFetchingEnvironment -> database.getLastUpdated()); + } + private static void dropUsers( Database database, List> userList, StringBuilder messageBuilder) { if (userList != null) { diff --git a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/ChangeLogExecutor.java b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/ChangeLogExecutor.java index 5a2f734553..86c949195d 100644 --- a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/ChangeLogExecutor.java +++ b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/ChangeLogExecutor.java @@ -1,9 +1,6 @@ package org.molgenis.emx2.sql; -import static org.jooq.impl.DSL.field; -import static org.jooq.impl.DSL.name; -import static org.jooq.impl.DSL.select; -import static org.jooq.impl.DSL.table; +import static org.jooq.impl.DSL.*; import static org.jooq.impl.SQLDataType.CHAR; import static org.jooq.impl.SQLDataType.JSON; import static org.jooq.impl.SQLDataType.TIMESTAMP; @@ -15,16 +12,12 @@ import java.sql.Timestamp; import java.util.Collections; +import java.util.Comparator; import java.util.List; -import org.jooq.DSLContext; -import org.jooq.Field; +import org.jooq.*; import org.jooq.Record; -import org.jooq.Record6; -import org.jooq.Result; -import org.molgenis.emx2.Change; +import org.molgenis.emx2.*; import org.molgenis.emx2.Schema; -import org.molgenis.emx2.SchemaMetadata; -import org.molgenis.emx2.TableMetadata; public class ChangeLogExecutor { @@ -36,6 +29,9 @@ public class ChangeLogExecutor { private static final Field OLD = field(name("old"), JSON.nullable(true)); private static final Field NEW = field(name("new"), JSON.nullable(true)); + private static final Field SCHEMA_NAME = + field(name("table_schema"), VARCHAR.nullable(false)); + private ChangeLogExecutor() { // hide } @@ -125,6 +121,60 @@ static List executeGetChanges(DSLContext jooq, SchemaMetadata schema, in .toList(); } + static List executeLastUpdates(DSLContext jooq) { + + // get a list of schema's with changelogs, need due to limited support for cross schema queries + List schemasWithChangeLog = getSchemasWithChangeLog(jooq); + + if (schemasWithChangeLog.isEmpty()) { + return Collections.emptyList(); + } + + // get the last updated table and details from all schema's that have a change log + SelectLimitPercentStep> query = + jooq.select( + OPERATION, + STAMP, + USERID, + TABLENAME, + inline(schemasWithChangeLog.get(0)).as(SCHEMA_NAME)) + .from(table(name(schemasWithChangeLog.get(0), MG_CHANGLOG))) + .orderBy(STAMP.desc()) + .limit(1); + + // union the select for schema's in a loop + for (int i = 1; i < schemasWithChangeLog.size(); i++) { + query.unionAll( + jooq.select( + OPERATION, + STAMP, + USERID, + TABLENAME, + inline(schemasWithChangeLog.get(1)).as(SCHEMA_NAME)) + .from(table(name(schemasWithChangeLog.get(i), MG_CHANGLOG))) + .orderBy(STAMP.desc()) + .limit(1)); + } + + // execute to query with all the unions + Result> result = query.fetch(); + + // transform the result in to records + return result.stream() + .map( + r -> { + char operation = r.getValue(OPERATION, char.class); + Timestamp stamp = r.getValue(STAMP, Timestamp.class); + String userId = r.getValue(USERID, String.class); + String tableName = r.getValue(TABLENAME, String.class); + String schemaName = r.getValue(SCHEMA_NAME, String.class); + + return new LastUpdate(operation, stamp, userId, tableName, schemaName); + }) + .sorted(Comparator.comparing(LastUpdate::stamp)) + .toList(); + } + static Integer executeGetChangesCount(DSLContext jooq, SchemaMetadata schema) { if (hasChangeLogTable(jooq, schema)) { return jooq.fetchCount(table(name(schema.getName(), MG_CHANGLOG))); @@ -141,4 +191,14 @@ static boolean hasChangeLogTable(DSLContext jooq, SchemaMetadata schema) { .where(field("table_schema").eq(schema.getName())) .and(field("table_name").eq(MG_CHANGLOG))); } + + static List getSchemasWithChangeLog(DSLContext jooq) { + Result result = + jooq.select() + .from(table(name("information_schema", "tables"))) + .where(field("table_name").eq(MG_CHANGLOG)) + .fetch(); + + return result.stream().map(r -> r.getValue(SCHEMA_NAME, String.class)).toList(); + } } diff --git a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java index 6c0fdfabaf..1e859d2f80 100644 --- a/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java +++ b/backend/molgenis-emx2-sql/src/main/java/org/molgenis/emx2/sql/SqlDatabase.java @@ -800,4 +800,9 @@ public TableListener getTableListener(String schemaName, String tableName) { } return null; } + + @Override + public List getLastUpdated() { + return ChangeLogExecutor.executeLastUpdates(jooq); + } } diff --git a/backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/ChangeLogExecutorTest.java b/backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/ChangeLogExecutorTest.java new file mode 100644 index 0000000000..746fd47d3c --- /dev/null +++ b/backend/molgenis-emx2-sql/src/test/java/org/molgenis/emx2/sql/ChangeLogExecutorTest.java @@ -0,0 +1,53 @@ +package org.molgenis.emx2.sql; + +import static org.junit.jupiter.api.Assertions.*; +import static org.molgenis.emx2.Column.column; +import static org.molgenis.emx2.Constants.IS_CHANGELOG_ENABLED; +import static org.molgenis.emx2.Row.row; +import static org.molgenis.emx2.TableMetadata.table; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.molgenis.emx2.LastUpdate; + +class ChangeLogExecutorTest { + + static SqlDatabase sqlDatabase; + + @BeforeEach + void setUp() { + sqlDatabase = (SqlDatabase) TestDatabaseFactory.getTestDatabase(); + + sqlDatabase.dropCreateSchema("ChangeLogExecutorTestA"); + sqlDatabase.dropCreateSchema("ChangeLogExecutorTestB"); + + Map settings = new LinkedHashMap<>(); + settings.put(IS_CHANGELOG_ENABLED, "true"); + SqlSchema schemaA = sqlDatabase.getSchema("ChangeLogExecutorTestA"); + schemaA.getMetadata().setSettings(settings); + sqlDatabase.getSchema("ChangeLogExecutorTestB").getMetadata().setSettings(settings); + + schemaA.create(table("test", column("A").setPkey(), column("B"))); + schemaA.getTable("test").insert(List.of(row("A", "a1", "B", "B"))); + } + + @Test + void getSchemasWithChangeLog() { + List schemasWithChangeLog = + ChangeLogExecutor.getSchemasWithChangeLog(sqlDatabase.getJooq()); + assertEquals(2, schemasWithChangeLog.size()); + assertTrue(schemasWithChangeLog.contains("ChangeLogExecutorTestA")); + assertTrue(schemasWithChangeLog.contains("ChangeLogExecutorTestB")); + } + + @Test + void executeLastUpdates() { + List lastUpdates = ChangeLogExecutor.executeLastUpdates(sqlDatabase.getJooq()); + assertEquals(1, lastUpdates.size()); + LastUpdate lastUpdate = lastUpdates.get(0); + assertEquals("ChangeLogExecutorTestA", lastUpdate.schemaName()); + } +} diff --git a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Database.java b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Database.java index 103de3c278..380cf673cf 100644 --- a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Database.java +++ b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/Database.java @@ -99,4 +99,6 @@ public interface Database extends HasSettingsInterface { Database setBindings(Map> bindings); Map> getJavaScriptBindings(); + + List getLastUpdated(); } diff --git a/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/LastUpdate.java b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/LastUpdate.java new file mode 100644 index 0000000000..f1cbaf8cf8 --- /dev/null +++ b/backend/molgenis-emx2/src/main/java/org/molgenis/emx2/LastUpdate.java @@ -0,0 +1,6 @@ +package org.molgenis.emx2; + +import java.sql.Timestamp; + +public record LastUpdate( + char operation, Timestamp stamp, String userId, String tableName, String schemaName) {} From 30277caceabb377dd78331b80ef355c0b3dd9653 Mon Sep 17 00:00:00 2001 From: connoratrug <47183404+connoratrug@users.noreply.github.com> Date: Thu, 10 Oct 2024 11:16:44 +0200 Subject: [PATCH 5/9] feat: Add demo style page to tailwind playground app (#4329) feat: Add demo style page to tailwind playground app it show 'all' the styles that can be changed for a theme and the names of the classes --- apps/tailwind-components/app.vue | 6 +- .../components/ColorTile.vue | 64 +++ apps/tailwind-components/layouts/default.vue | 4 + .../pages/Styles.other.vue | 418 ++++++++++++++++++ 4 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 apps/tailwind-components/components/ColorTile.vue create mode 100644 apps/tailwind-components/pages/Styles.other.vue diff --git a/apps/tailwind-components/app.vue b/apps/tailwind-components/app.vue index 1fcfeac8d2..e3c9ba9cc1 100644 --- a/apps/tailwind-components/app.vue +++ b/apps/tailwind-components/app.vue @@ -62,7 +62,11 @@ function toggleLayout () {
- + +
+
+ +
diff --git a/apps/tailwind-components/components/ColorTile.vue b/apps/tailwind-components/components/ColorTile.vue new file mode 100644 index 0000000000..3d8fdb8d23 --- /dev/null +++ b/apps/tailwind-components/components/ColorTile.vue @@ -0,0 +1,64 @@ + + diff --git a/apps/tailwind-components/layouts/default.vue b/apps/tailwind-components/layouts/default.vue index d6ca022c33..e0f22446fa 100644 --- a/apps/tailwind-components/layouts/default.vue +++ b/apps/tailwind-components/layouts/default.vue @@ -39,6 +39,10 @@ const stories = Object.keys(modules)