From c97e31a4fd382ff4cacc463ef443b8f78aa2b143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20B=C3=BChlmann?= Date: Fri, 20 Sep 2024 18:45:08 +0200 Subject: [PATCH] Upgrade JDK, Gradle, Jooq Switch to Testcontainers --- .github/workflows/build_and_test.yml | 17 ++----- .github/workflows/publish_gradle_plugin.yml | 6 +-- .github/workflows/publish_to_central.yml | 6 +-- build.gradle | 13 +++++ core/build.gradle | 15 +++--- gradle-plugin/build.gradle | 6 +-- .../gradle-plugins/ch.ergon.adam.properties | 1 - gradle/wrapper/gradle-wrapper.properties | 2 +- integration-test-db/build.gradle | 2 +- integration-test/build.gradle | 25 ++++++---- .../integrationtest/AbstractDbTestBase.java | 7 +++ .../PostgreSqlToSqliteTestDbUrlProvider.java | 14 +++--- .../postgresql/PostgreSqlArrayFieldTest.java | 3 +- .../postgresql/PostgreSqlCreateTableTest.java | 7 +-- .../PostgreSqlTestDbUrlProvider.java | 10 ++-- .../sqlite/SqliteCreateTableTest.java | 4 +- jooq/build.gradle | 11 ++--- .../java/ch/ergon/adam/jooq/JooqSink.java | 14 +++--- .../java/ch/ergon/adam/jooq/JooqSource.java | 47 +++++++++++++------ postgresql/build.gradle | 11 ++--- .../ergon/adam/postgresql/PostgreSqlSink.java | 8 +--- .../adam/postgresql/PostgreSqlSource.java | 13 ++--- sqlite/build.gradle | 11 ++--- .../ch/ergon/adam/sqlite/SqliteSource.java | 21 --------- yml/build.gradle | 15 +++--- 25 files changed, 148 insertions(+), 141 deletions(-) delete mode 100644 gradle-plugin/src/main/resources/META-INF/gradle-plugins/ch.ergon.adam.properties diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index efec5c3..69a265a 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -5,18 +5,7 @@ on: [ push, pull_request] jobs: container-job: runs-on: ubuntu-latest - container: adoptopenjdk:8-jdk - - services: - postgres: - image: postgres:10 - env: - POSTGRES_PASSWORD: password - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 + container: eclipse-temurin:21-jdk steps: - name: Install dependencies @@ -27,9 +16,9 @@ jobs: - name: Build run: ./gradlew -S -i assemble - name: Test - run: ./gradlew -S -i test -Ppostgresql_database_url="jdbc:postgresql://postgres:5432/postgres?user=postgres&password=password" + run: ./gradlew -S -i test - name: Publish Unit Test Results - uses: EnricoMi/publish-unit-test-result-action@v1 + uses: EnricoMi/publish-unit-test-result-action@v2 if: always() with: files: ./**/build/test-results/**/*.xml diff --git a/.github/workflows/publish_gradle_plugin.yml b/.github/workflows/publish_gradle_plugin.yml index 469dd6f..298ab66 100644 --- a/.github/workflows/publish_gradle_plugin.yml +++ b/.github/workflows/publish_gradle_plugin.yml @@ -14,11 +14,11 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 8 + - name: Set up JDK 21 uses: actions/setup-java@v2 with: - java-version: '8' - distribution: 'adopt' + java-version: '21' + distribution: 'temurin' - name: Prepare GPG key run: mkdir build && echo '${{secrets.SIGNING_KEY_FILE_BASE64}}' | base64 -d > build/adam_signing_key.gpg diff --git a/.github/workflows/publish_to_central.yml b/.github/workflows/publish_to_central.yml index 5123bbb..cbab6e1 100644 --- a/.github/workflows/publish_to_central.yml +++ b/.github/workflows/publish_to_central.yml @@ -14,11 +14,11 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up JDK 8 + - name: Set up JDK 21 uses: actions/setup-java@v2 with: - java-version: '8' - distribution: 'adopt' + java-version: '21' + distribution: 'temurin' - name: Prepare GPG key run: mkdir build && echo '${{secrets.SIGNING_KEY_FILE_BASE64}}' | base64 -d > build/adam_signing_key.gpg diff --git a/build.gradle b/build.gradle index 2c96342..9c4b614 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,5 @@ +import static java.nio.charset.StandardCharsets.UTF_8 + buildscript { repositories { mavenCentral() @@ -38,3 +40,14 @@ nexusPublishing { } } } + +ext.getLocalProperty = { key, file = "local.properties" -> + def properties = new Properties() + def localProperties = new File(rootProject.rootDir, file) + if (localProperties.isFile()) { + new InputStreamReader(new FileInputStream(localProperties), UTF_8).with {properties.load} + return properties.getProperty(key, "") + } else { + return null + } +} diff --git a/core/build.gradle b/core/build.gradle index f169e1e..2bc83ef 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -1,22 +1,21 @@ plugins { - id 'java' + id 'java-library' id "ch.ergon.gradle.goodies.versioning" - id 'maven' } group 'ch.ergon.adam' description 'The core functionality of ADAM' -sourceCompatibility = 1.8 +sourceCompatibility = 21 dependencies { testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.1' - compile group: 'com.google.guava', name: 'guava', version: '30.1-jre' - compile group: 'org.reflections', name: 'reflections', version: '0.9.12' - compile group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '5.10.0.202012080955-r' - compile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.30' + api 'com.google.guava:guava:33.3.0-jre' + implementation group: 'org.reflections', name: 'reflections', version: '0.9.12' + api group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '5.10.0.202012080955-r' + implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.30' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.4.0' - testCompile group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3' + testImplementation group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3' } apply from: "${rootProject.projectDir}/publish.gradle" diff --git a/gradle-plugin/build.gradle b/gradle-plugin/build.gradle index 6e0f19c..3c32219 100644 --- a/gradle-plugin/build.gradle +++ b/gradle-plugin/build.gradle @@ -8,11 +8,11 @@ plugins { group 'ch.ergon.adam' description = 'The ADAM Gradle Plugin' -sourceCompatibility = 1.8 +sourceCompatibility = 21 dependencies { - compile gradleApi() - compile project(':core') + implementation gradleApi() + implementation project(':core') } apply from: "${rootProject.projectDir}/common.gradle" diff --git a/gradle-plugin/src/main/resources/META-INF/gradle-plugins/ch.ergon.adam.properties b/gradle-plugin/src/main/resources/META-INF/gradle-plugins/ch.ergon.adam.properties deleted file mode 100644 index 1b85b1e..0000000 --- a/gradle-plugin/src/main/resources/META-INF/gradle-plugins/ch.ergon.adam.properties +++ /dev/null @@ -1 +0,0 @@ -implementation-class=ch.ergon.adam.gradleplugin.AdamPlugin diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 442d913..d642e7f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.8.3-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/integration-test-db/build.gradle b/integration-test-db/build.gradle index 6eeec73..b4d0814 100644 --- a/integration-test-db/build.gradle +++ b/integration-test-db/build.gradle @@ -4,4 +4,4 @@ plugins { group 'ch.ergon.adam' -sourceCompatibility = 1.8 +sourceCompatibility = 21 diff --git a/integration-test/build.gradle b/integration-test/build.gradle index 6f61b18..d17a971 100644 --- a/integration-test/build.gradle +++ b/integration-test/build.gradle @@ -5,22 +5,27 @@ plugins { group 'ch.ergon.adam' -sourceCompatibility = 1.8 +sourceCompatibility = 21 dependencies { - testCompile group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.14.1' - testCompile group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.1' - testCompile project(':core') - testCompile project(':yml') - testCompile project(':postgresql') - testCompile project(':sqlite') - testCompile project(':integration-test-db') + testImplementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.14.1' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.1' + testImplementation project(':core') + testImplementation project(':yml') + testImplementation project(':postgresql') + testImplementation project(':sqlite') + testImplementation project(':integration-test-db') testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.1' testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.7.1' - testCompile group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3' + testImplementation group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3' + testImplementation("org.testcontainers:testcontainers:1.20.1") + testImplementation("org.testcontainers:junit-jupiter:1.20.1") + testImplementation("org.testcontainers:postgresql:1.20.1") + testImplementation("org.postgresql:postgresql:42.7.1") + testImplementation("org.testcontainers:oracle-free:1.20.1") + testImplementation('com.oracle.database.jdbc:ojdbc11:23.5.0.24.07') } test { - systemProperty "postgresql_database_url", project.findProperty("postgresql_database_url") ?: 'jdbc:postgresql://localhost:5432/test?user=test&password=test' useJUnitPlatform() } diff --git a/integration-test/src/test/java/ch/ergon/adam/integrationtest/AbstractDbTestBase.java b/integration-test/src/test/java/ch/ergon/adam/integrationtest/AbstractDbTestBase.java index 4d2b7e3..aba8f70 100644 --- a/integration-test/src/test/java/ch/ergon/adam/integrationtest/AbstractDbTestBase.java +++ b/integration-test/src/test/java/ch/ergon/adam/integrationtest/AbstractDbTestBase.java @@ -109,5 +109,12 @@ protected String targetToYml() { return baos.toString(); } + protected String sourceToYml() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + YmlSink ymlSink = new YmlSink(baos); + new SchemaMigrator(new EmptySource(), getSourceDbSource(), ymlSink).migrate(); + return baos.toString(); + } + } diff --git a/integration-test/src/test/java/ch/ergon/adam/integrationtest/PostgreSqlToSqliteTestDbUrlProvider.java b/integration-test/src/test/java/ch/ergon/adam/integrationtest/PostgreSqlToSqliteTestDbUrlProvider.java index 484cd86..b7f7c21 100644 --- a/integration-test/src/test/java/ch/ergon/adam/integrationtest/PostgreSqlToSqliteTestDbUrlProvider.java +++ b/integration-test/src/test/java/ch/ergon/adam/integrationtest/PostgreSqlToSqliteTestDbUrlProvider.java @@ -1,23 +1,22 @@ package ch.ergon.adam.integrationtest; +import org.testcontainers.containers.PostgreSQLContainer; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.util.HashMap; -import java.util.Map; import static java.lang.String.format; public class PostgreSqlToSqliteTestDbUrlProvider extends TestDbUrlProvider { - private static final String DATABASE_URL_PROPERTY = "postgresql_database_url"; - private static final String DATABASE_URL_DEFAULT = "jdbc:postgresql://localhost:5432/test?user=test&password=test"; + private final static PostgreSQLContainer container = new PostgreSQLContainer<>("postgres:15-alpine"); + private static final String SOURCE_SCHEMA = "test-source"; protected static final String TARGET_SCHEMA = "test-target"; - private static Map connectionsByUrl = new HashMap<>(); private final Path tempFolder; public PostgreSqlToSqliteTestDbUrlProvider() throws IOException { @@ -55,7 +54,10 @@ protected String getPostgreSqlDbUrl(String schema) { } private String getPostgreSqlDbUrl() { - return System.getProperty(DATABASE_URL_PROPERTY, DATABASE_URL_DEFAULT); + if (!container.isRunning()) { + container.start(); + } + return container.getJdbcUrl() + "&user=" + container.getUsername() + "&password=" + container.getPassword(); } } diff --git a/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlArrayFieldTest.java b/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlArrayFieldTest.java index 17a78e5..936fa3a 100644 --- a/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlArrayFieldTest.java +++ b/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlArrayFieldTest.java @@ -7,6 +7,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; public class PostgreSqlArrayFieldTest extends AbstractDbTestBase { @@ -39,7 +40,7 @@ public void testCreateTableToYml() throws Exception { schema.getTable("test_table").getField("array_col").setDefaultValue("'{}'"); migrateTargetWithSchema(schema); String yml = targetToYml(); - assertThat(yml, is(YML)); + assertEquals(YML, yml); } } diff --git a/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlCreateTableTest.java b/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlCreateTableTest.java index 4c2fdee..b0b8a3a 100644 --- a/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlCreateTableTest.java +++ b/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlCreateTableTest.java @@ -10,10 +10,7 @@ import static ch.ergon.adam.core.db.schema.DataType.NUMERIC; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; public class PostgreSqlCreateTableTest extends AbstractDbTestBase { @@ -185,7 +182,7 @@ public void testCreateTableToYml() throws Exception { getSourceDbConnection().createStatement().execute(CREATE_TABLE_SQL); sourceToTarget(); String yml = targetToYml(); - assertThat(yml, is(YML)); + assertEquals(YML, yml); } } diff --git a/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlTestDbUrlProvider.java b/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlTestDbUrlProvider.java index f26c248..3a4e171 100644 --- a/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlTestDbUrlProvider.java +++ b/integration-test/src/test/java/ch/ergon/adam/integrationtest/postgresql/PostgreSqlTestDbUrlProvider.java @@ -4,6 +4,7 @@ import ch.ergon.adam.integrationtest.TestDbUrlProvider; import ch.ergon.adam.postgresql.PostgreSqlFactory; import ch.ergon.adam.postgresql.PostgreSqlTransactionWrapper; +import org.testcontainers.containers.PostgreSQLContainer; import java.sql.Connection; import java.sql.DriverManager; @@ -13,8 +14,8 @@ public class PostgreSqlTestDbUrlProvider extends TestDbUrlProvider { - private static final String DATABASE_URL_PROPERTY = "postgresql_database_url"; - private static final String DATABASE_URL_DEFAULT = "jdbc:postgresql://localhost:5432/test?user=test&password=test"; + private static final PostgreSQLContainer container = new PostgreSQLContainer<>("postgres:15-alpine"); + private static final String SOURCE_SCHEMA = "test-source"; protected static final String TARGET_SCHEMA = "test-target"; @@ -43,7 +44,10 @@ protected String getDbUrl(String schema) { } protected String getDbUrl() { - return System.getProperty(DATABASE_URL_PROPERTY, DATABASE_URL_DEFAULT); + if (!container.isRunning()) { + container.start(); + } + return container.getJdbcUrl() + "&user=" + container.getUsername() + "&password=" + container.getPassword(); } @Override diff --git a/integration-test/src/test/java/ch/ergon/adam/integrationtest/sqlite/SqliteCreateTableTest.java b/integration-test/src/test/java/ch/ergon/adam/integrationtest/sqlite/SqliteCreateTableTest.java index 07e13e8..a7de5d4 100644 --- a/integration-test/src/test/java/ch/ergon/adam/integrationtest/sqlite/SqliteCreateTableTest.java +++ b/integration-test/src/test/java/ch/ergon/adam/integrationtest/sqlite/SqliteCreateTableTest.java @@ -90,14 +90,14 @@ private void verifySchema(Schema schema) { assertNull(field.getDbEnum()); assertThat(field.getDataType(), is(DataType.VARCHAR)); - Index index = table.getIndex("pk"); + Index index = table.getIndexes().stream().filter(Index::isPrimary).findFirst().orElse(null); assertNotNull(index); assertTrue(index.isPrimary()); assertTrue(index.isUnique()); assertThat(index.getFields().size(), is(1)); assertThat(index.getFields().get(0).getName(), is("id")); - index = table.getIndex("test_table_col1_idx"); + index = table.getIndexes().stream().filter(i -> !i.isPrimary()).findFirst().orElse(null); assertNotNull(index); assertFalse(index.isPrimary()); assertTrue(index.isUnique()); diff --git a/jooq/build.gradle b/jooq/build.gradle index a86a73c..05f631e 100644 --- a/jooq/build.gradle +++ b/jooq/build.gradle @@ -1,18 +1,17 @@ plugins { - id 'java' - id 'maven' + id 'java-library' id "ch.ergon.gradle.goodies.versioning" } group 'ch.ergon.adam' description 'The jOOQ plugin for ADAM' -sourceCompatibility = 1.8 +sourceCompatibility = 21 dependencies { - testCompile group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.1' - compile project(':core') - compile group: 'org.jooq', name: 'jooq', version: '3.14.8' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.1' + api project(':core') + api group: 'org.jooq', name: 'jooq', version: '3.19.11' } apply from: "${rootProject.projectDir}/publish.gradle" diff --git a/jooq/src/main/java/ch/ergon/adam/jooq/JooqSink.java b/jooq/src/main/java/ch/ergon/adam/jooq/JooqSink.java index 184b93c..2953f2d 100644 --- a/jooq/src/main/java/ch/ergon/adam/jooq/JooqSink.java +++ b/jooq/src/main/java/ch/ergon/adam/jooq/JooqSink.java @@ -9,10 +9,10 @@ import ch.ergon.adam.core.db.schema.Sequence; import ch.ergon.adam.core.db.schema.Table; import ch.ergon.adam.core.db.schema.*; -import com.google.common.base.Strings; import org.jooq.DataType; import org.jooq.*; import org.jooq.impl.DSL; +import org.jooq.impl.DefaultDataType; import org.jooq.impl.SQLDataType; import java.sql.Connection; @@ -27,7 +27,6 @@ import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; import static java.util.stream.Stream.concat; -import static org.jooq.impl.DSL.noCondition; import static org.jooq.impl.SQLDataType.TIMESTAMPWITHTIMEZONE; public class JooqSink implements SchemaSink { @@ -74,6 +73,7 @@ protected Name getFieldName(Field field) { @Override public void setTargetSchema(Schema targetSchema) { + targetSchema.getEnums().forEach( e -> new DefaultDataType<>(null, Object.class, e.getName())); } @Override @@ -107,7 +107,7 @@ public void dropIndex(Index index) { @Override public void createIndex(Index index) { if (index.isPrimary()) { - ConstraintFinalStep primaryKey = DSL.constraint(index.getName()).primaryKey(createSchemaItemNameArray(index.getFields())); + ConstraintEnforcementStep primaryKey = DSL.constraint(index.getName()).primaryKey(createSchemaItemNameArray(index.getFields())); context.alterTable(index.getTable().getName()).add(primaryKey).execute(); } else { Collection fieldNames = index.getFields().stream().map(this::getFieldName).collect(toList()); @@ -118,7 +118,7 @@ public void createIndex(Index index) { createIndex = context.createIndex(index.getName()); } CreateIndexIncludeStep indexStep = createIndex.on(getTableName(index.getTable()), fieldNames); - if (Strings.isNullOrEmpty(index.getWhere())) { + if (index.getWhere() == null || index.getWhere().isEmpty()) { indexStep.execute(); } else { indexStep.where(DSL.condition(index.getWhere())).execute(); @@ -155,8 +155,8 @@ public void setDefault(Field field) { @Override public void createTable(Table table) { - CreateTableColumnStep createTable = context.createTable(getTableName(table)); - CreateTableColumnStep addRows = null; + CreateTableElementListStep createTable = context.createTable(getTableName(table)); + CreateTableElementListStep addRows = null; for (Field field : table.getFields()) { if (addRows == null) { addRows = createTable.column(field.getName(), mapType(field)); @@ -212,7 +212,7 @@ public void dropConstraint(Constraint constraint) { public void createConstraint(Constraint constraint) { if (constraint instanceof RuleConstraint) { RuleConstraint rule = (RuleConstraint)constraint; - ConstraintFinalStep check = DSL.constraint(constraint.getName()).check(DSL.condition(rule.getRule())); + ConstraintEnforcementStep check = DSL.constraint(constraint.getName()).check(DSL.condition(rule.getRule())); context.alterTable(constraint.getTable().getName()).add(check).execute(); } else if (constraint instanceof PrimaryKeyConstraint) { // Handled together with index diff --git a/jooq/src/main/java/ch/ergon/adam/jooq/JooqSource.java b/jooq/src/main/java/ch/ergon/adam/jooq/JooqSource.java index 0ed15f4..365a3d9 100644 --- a/jooq/src/main/java/ch/ergon/adam/jooq/JooqSource.java +++ b/jooq/src/main/java/ch/ergon/adam/jooq/JooqSource.java @@ -142,23 +142,40 @@ private ForeignKey mapForeignKeyFromJooq(Map tables, org.jooq.For private Table mapTableFromJooq(org.jooq.Table jooqTable) { Table table = new Table(jooqTable.getName()); table.setFields(stream(jooqTable.fields()).map(this::mapFieldFromJooq).collect(toList())); - table.setIndexes(jooqTable.getIndexes().stream().map(jooqIndex -> mapIndexFromJooq(table, jooqIndex)).collect(toList())); + List indexes = jooqTable.getIndexes().stream().map(jooqIndex -> mapIndexFromJooq(table, jooqIndex)).collect(toList()); + if (jooqTable.getPrimaryKey() != null) { + indexes.add(mapPrimaryKeyFromJooq(table, jooqTable.getPrimaryKey())); + } + jooqTable.getUniqueKeys().stream().map(jooqKey -> mapUniqueKeyFromJooq(table, jooqKey)).forEach(indexes::add); + table.setIndexes(indexes); return table; } + private Index mapPrimaryKeyFromJooq(Table table, UniqueKey primaryKey) { + Index index = mapUniqueKeyFromJooq(table, primaryKey); + index.setPrimary(true); + return index; + } + + private Index mapUniqueKeyFromJooq(Table table, UniqueKey uniqueKey) { + Index index = mapKeyFromJooq(table, uniqueKey); + index.setUnique(true); + return index; + } + private Index mapIndexFromJooq(Table table, org.jooq.Index jooqIndex) { Index index = new Index(jooqIndex.getName()); - index.setFields(jooqIndex.getFields().stream().map(jooqField -> table.getField(jooqField.getName())).collect(toList())); - index.setUnique(jooqIndex.getUnique()); if (jooqIndex.getWhere() != null) { index.setWhere(jooqIndex.getWhere().toString()); } - UniqueKey primaryKey = jooqIndex.getTable().getPrimaryKey(); - if (primaryKey != null) { - String[] primaryKeyFieldNames = primaryKey.getFields().stream().map(TableField::getName).toArray(String[]::new); - String[] indexFieldNames = jooqIndex.getFields().stream().map(SortField::getName).toArray(String[]::new); - index.setPrimary(Arrays.equals(primaryKeyFieldNames, indexFieldNames)); - } + index.setUnique(jooqIndex.getUnique()); + index.setFields(jooqIndex.getFields().stream().map(jooqField -> table.getField(jooqField.getName())).collect(toList())); + return index; + } + + private Index mapKeyFromJooq(Table table, Key jooqKey) { + Index index = new Index(jooqKey.getName()); + index.setFields(jooqKey.getFields().stream().map(jooqField -> table.getField(jooqField.getName())).collect(toList())); return index; } @@ -177,9 +194,9 @@ private Field mapFieldFromJooq(org.jooq.Field jooqField) { elementType = jooqType; } - field.setLength(elementType.hasLength() && elementType.length() < 20000000 ? elementType.length() : null); - field.setPrecision(elementType.hasPrecision() && elementType.precision() < 10000 ? elementType.precision() : null); - field.setScale(elementType.hasScale() ? elementType.scale() : null); + field.setLength(elementType.hasLength() && elementType.length() > 0 && elementType.length() < 20000000 ? elementType.length() : null); + field.setPrecision(elementType.hasPrecision() && elementType.precision() > 0 && elementType.precision() < 10000 ? elementType.precision() : null); + field.setScale(elementType.hasScale() && elementType.scale() > 0 ? elementType.scale() : null); field.setSequence(isSequence(jooqField)); @@ -191,7 +208,7 @@ private Field mapFieldFromJooq(org.jooq.Field jooqField) { protected String getDefaultValue(org.jooq.Field jooqField) { org.jooq.Field defaultValue = jooqField.getDataType().defaultValue(); - return defaultValue.getName(); + return defaultValue.toString(); } protected boolean isSequence(org.jooq.Field jooqField) { @@ -211,8 +228,10 @@ protected DataType mapDataTypeFromJooq(org.jooq.Field jooqField) { } else { throw new RuntimeException("Unsupported interval type [" + type.getName() + "]"); } + } else if (sqlDataType.isNumeric() && jooqField.getDataType().precision() == 0 && jooqField.getDataType().scale() == 0) { + return DataType.DECIMAL_INTEGER; } - String typeName = sqlDataType.getTypeName(); + String typeName = jooqField.getDataType().isArray() ? sqlDataType.getArrayBaseDataType().getTypeName() : sqlDataType.getTypeName(); try { return DataType.valueOf(typeName.toUpperCase().replace(" ", "")); } catch (IllegalArgumentException e) { diff --git a/postgresql/build.gradle b/postgresql/build.gradle index 934a232..f830241 100644 --- a/postgresql/build.gradle +++ b/postgresql/build.gradle @@ -1,18 +1,17 @@ plugins { - id 'java' + id 'java-library' id "ch.ergon.gradle.goodies.versioning" - id 'maven' } group 'ch.ergon.adam' description 'The PostgreSQL plugin for ADAM' -sourceCompatibility = 1.8 +sourceCompatibility = 21 dependencies { - testCompile group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.1' - compile project(':jooq') - compile group: 'org.postgresql', name: 'postgresql', version: '42.2.19' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.1' + implementation project(':jooq') + implementation group: 'org.postgresql', name: 'postgresql', version: '42.2.19' } apply from: "${rootProject.projectDir}/publish.gradle" diff --git a/postgresql/src/main/java/ch/ergon/adam/postgresql/PostgreSqlSink.java b/postgresql/src/main/java/ch/ergon/adam/postgresql/PostgreSqlSink.java index 088db6e..ff10b28 100644 --- a/postgresql/src/main/java/ch/ergon/adam/postgresql/PostgreSqlSink.java +++ b/postgresql/src/main/java/ch/ergon/adam/postgresql/PostgreSqlSink.java @@ -6,6 +6,7 @@ import ch.ergon.adam.core.db.schema.Sequence; import ch.ergon.adam.core.db.schema.Table; import org.jooq.*; +import org.jooq.Record; import org.jooq.impl.DSL; import org.jooq.impl.DefaultDataType; @@ -46,7 +47,6 @@ public void changeFieldType(Field oldField, Field newField, ch.ergon.adam.core.d using = String.format(" USING %s::%s", newField.getName(), newField.getDbEnum().getName()); newTypeName = mapType(newField).getTypeName(); if (newField.isArray()) { - newTypeName += "[]"; using += "[]"; } } else { @@ -73,11 +73,7 @@ protected DataType mapType(Field field) { protected DataType mapFieldToJooqType(Field field) { switch (field.getDataType()) { case ENUM: - if (field.isArray()) { - return new DefaultDataType<>(null, String[].class, field.getDbEnum().getName() + "[]", field.getDbEnum().getName() + "[]"); - } else { - return new DefaultDataType<>(null, VARCHAR, field.getDbEnum().getName(), field.getDbEnum().getName()); - } + return new DefaultDataType<>(null, VARCHAR, field.getDbEnum().getName(), field.getDbEnum().getName()); default: return super.mapFieldToJooqType(field); } diff --git a/postgresql/src/main/java/ch/ergon/adam/postgresql/PostgreSqlSource.java b/postgresql/src/main/java/ch/ergon/adam/postgresql/PostgreSqlSource.java index a510e46..152fb88 100644 --- a/postgresql/src/main/java/ch/ergon/adam/postgresql/PostgreSqlSource.java +++ b/postgresql/src/main/java/ch/ergon/adam/postgresql/PostgreSqlSource.java @@ -26,6 +26,8 @@ public class PostgreSqlSource extends JooqSource { private static final Pattern CHECK_CONSTRAINT_PATTERN = Pattern.compile("^CHECK \\((.*)\\)$"); + private static final Pattern DEFAULT_CAST_PATTERN = Pattern.compile("cast\\((.*) as [^ ^)]+( array)?\\)"); + private final String schemaName; private Map enums; @@ -108,10 +110,9 @@ protected String getDefaultValue(org.jooq.Field jooqField) { if (defaultValue == null) { return null; } - if (jooqField.getDataType().isArray() && defaultValue.endsWith("[]")) { - defaultValue = defaultValue.substring(0, defaultValue.length() - 2); - } - return defaultValue.replaceAll("::[a-z A-Z_]*", ""); + Matcher matcher = DEFAULT_CAST_PATTERN.matcher(defaultValue); + defaultValue = matcher.replaceAll(r -> r.group(1) ); + return defaultValue; } private void fetchViewDependencies(Schema schema) { @@ -249,7 +250,7 @@ private View mapViewFromJooq(Record record) { @Override protected DataType mapDataTypeFromJooq(org.jooq.Field jooqField) { - String typeName = jooqField.getDataType().getTypeName(); + String typeName = jooqField.getDataType().isArray() ? jooqField.getDataType().getArrayComponentDataType().getTypeName() : jooqField.getDataType().getTypeName(); switch (typeName) { case "other": return null; // Will be set later @@ -257,7 +258,7 @@ protected DataType mapDataTypeFromJooq(org.jooq.Field jooqField) { // Only if type is array of text return DataType.CLOB; } - if (enums.containsKey(typeName)) { + if (enums.containsKey(typeName) || enums.containsKey(typeName.replaceFirst("_", ""))) { return ENUM; } return super.mapDataTypeFromJooq(jooqField); diff --git a/sqlite/build.gradle b/sqlite/build.gradle index 109e7b6..36d909d 100644 --- a/sqlite/build.gradle +++ b/sqlite/build.gradle @@ -1,18 +1,17 @@ plugins { - id 'java' + id 'java-library' id "ch.ergon.gradle.goodies.versioning" - id 'maven' } group 'ch.ergon.adam' description 'The SQLite plugin for ADAM' -sourceCompatibility = 1.8 +sourceCompatibility = 21 dependencies { - testCompile group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.1' - compile project(':jooq') - compile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.34.0' + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.1' + implementation project(':jooq') + implementation group: 'org.xerial', name: 'sqlite-jdbc', version: '3.34.0' } apply from: "${rootProject.projectDir}/publish.gradle" diff --git a/sqlite/src/main/java/ch/ergon/adam/sqlite/SqliteSource.java b/sqlite/src/main/java/ch/ergon/adam/sqlite/SqliteSource.java index 133e6b5..dd80135 100644 --- a/sqlite/src/main/java/ch/ergon/adam/sqlite/SqliteSource.java +++ b/sqlite/src/main/java/ch/ergon/adam/sqlite/SqliteSource.java @@ -38,7 +38,6 @@ public Schema getSchema() { schema.setViews(getViews()); schema.getViews().forEach(view -> view.setFields(schema.getTable(view.getName()).getFields())); schema.getTables().removeIf(table -> schema.getView(table.getName()) != null); - setPrimaryKeys(schema); setSequences(schema); return schema; } @@ -57,26 +56,6 @@ private View mapViewFromJooq(Record record) { return view; } - private void setPrimaryKeys(Schema schema) { - for (Table table : schema.getTables()) { - Result result = getContext().resultQuery("PRAGMA table_info('test_table')").fetch(); - for (Record record : result) { - if (record.getValue("pk", Integer.class) == 0) { - continue; - } - String fieldName = record.getValue("name", String.class); - Index primaryKeyIndex = new Index("pk"); - primaryKeyIndex.setPrimary(true); - primaryKeyIndex.setUnique(true); - primaryKeyIndex.setFields(newArrayList(table.getField(fieldName))); - List newIndexList = Lists.newArrayList(primaryKeyIndex); - newIndexList.addAll(table.getIndexes()); - table.setIndexes(newIndexList); - break; - } - } - } - private void setSequences(Schema schema) { Result result = getContext().resultQuery("SELECT tbl_name FROM sqlite_master WHERE sql LIKE \"%AUTOINCREMENT%\"").fetch(); result.forEach(record -> { diff --git a/yml/build.gradle b/yml/build.gradle index e62a46a..7f9c551 100644 --- a/yml/build.gradle +++ b/yml/build.gradle @@ -1,20 +1,19 @@ plugins { - id 'java' + id 'java-library' id "ch.ergon.gradle.goodies.versioning" - id 'maven' } group 'ch.ergon.adam' description 'The YML plugin for ADAM' -sourceCompatibility = 1.8 +sourceCompatibility = 21 dependencies { - testCompile group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.1' - compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.12.2' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.2' - compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.12.2' - compile project(':core') + testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.7.1' + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.12.2' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.2' + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.12.2' + implementation project(':core') } apply from: "${rootProject.projectDir}/publish.gradle"