diff --git a/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java index 89aa9e491..5911fa0e1 100644 --- a/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/CrawlerExtensionContextBuilder.java @@ -33,7 +33,6 @@ import de.sovity.edc.ext.catalog.crawler.dao.CatalogPatchApplier; import de.sovity.edc.ext.catalog.crawler.dao.config.DataSourceFactory; import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; -import de.sovity.edc.ext.catalog.crawler.dao.config.FlywayService; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorQueries; import de.sovity.edc.ext.catalog.crawler.dao.connectors.ConnectorStatusUpdater; import de.sovity.edc.ext.catalog.crawler.dao.contract_offers.ContractOfferQueries; @@ -112,8 +111,6 @@ public static CrawlerExtensionContext buildContext( // DB var dataSourceFactory = new DataSourceFactory(config); var dataSource = dataSourceFactory.newDataSource(); - var flywayService = new FlywayService(config, monitor, dataSource); - flywayService.validateOrMigrateInTests(); // Dao var dataOfferQueries = new DataOfferQueries(); @@ -183,10 +180,10 @@ public static CrawlerExtensionContext buildContext( // Schedules List> jobs = List.of( - getOnlineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), - getOfflineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), - getDeadConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller), - getOfflineConnectorCleanerCronJob(dslContextFactory, offlineConnectorCleaner) + getOnlineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller, config), + getOfflineConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller, config), + getDeadConnectorRefreshCronJob(dslContextFactory, connectorQueueFiller, config), + getOfflineConnectorCleanerCronJob(dslContextFactory, offlineConnectorCleaner, config) ); // Startup @@ -264,9 +261,9 @@ private static AssetMapper newAssetMapper( @NotNull private static CronJobRef getOfflineConnectorCleanerCronJob(DslContextFactory dslContextFactory, - OfflineConnectorCleaner offlineConnectorCleaner) { + OfflineConnectorCleaner offlineConnectorCleaner, Config config) { return new CronJobRef<>( - CrawlerExtension.SCHEDULED_KILL_OFFLINE_CONNECTORS, + CrawlerConfigProps.CRAWLER_SCHEDULED_KILL_OFFLINE_CONNECTORS.getStringOrThrow(config), OfflineConnectorCleanerJob.class, () -> new OfflineConnectorCleanerJob(dslContextFactory, offlineConnectorCleaner) ); @@ -275,10 +272,11 @@ private static CronJobRef getOfflineConnectorCleaner @NotNull private static CronJobRef getOnlineConnectorRefreshCronJob( DslContextFactory dslContextFactory, - ConnectorQueueFiller connectorQueueFiller + ConnectorQueueFiller connectorQueueFiller, + Config config ) { return new CronJobRef<>( - CrawlerExtension.CRON_ONLINE_CONNECTOR_REFRESH, + CrawlerConfigProps.CRAWLER_CRON_ONLINE_CONNECTOR_REFRESH.getStringOrThrow(config), OnlineConnectorRefreshJob.class, () -> new OnlineConnectorRefreshJob(dslContextFactory, connectorQueueFiller) ); @@ -287,10 +285,11 @@ private static CronJobRef getOnlineConnectorRefreshCr @NotNull private static CronJobRef getOfflineConnectorRefreshCronJob( DslContextFactory dslContextFactory, - ConnectorQueueFiller connectorQueueFiller + ConnectorQueueFiller connectorQueueFiller, + Config config ) { return new CronJobRef<>( - CrawlerExtension.CRON_OFFLINE_CONNECTOR_REFRESH, + CrawlerConfigProps.CRAWLER_CRON_OFFLINE_CONNECTOR_REFRESH.getStringOrThrow(config), OfflineConnectorRefreshJob.class, () -> new OfflineConnectorRefreshJob(dslContextFactory, connectorQueueFiller) ); @@ -298,9 +297,9 @@ private static CronJobRef getOfflineConnectorRefresh @NotNull private static CronJobRef getDeadConnectorRefreshCronJob(DslContextFactory dslContextFactory, - ConnectorQueueFiller connectorQueueFiller) { + ConnectorQueueFiller connectorQueueFiller, Config config) { return new CronJobRef<>( - CrawlerExtension.CRON_DEAD_CONNECTOR_REFRESH, + CrawlerConfigProps.CRAWLER_CRON_DEAD_CONNECTOR_REFRESH.getStringOrThrow(config), DeadConnectorRefreshJob.class, () -> new DeadConnectorRefreshJob(dslContextFactory, connectorQueueFiller) ); diff --git a/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java index b66997470..24d0b0d13 100644 --- a/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/DataSourceFactory.java @@ -15,6 +15,7 @@ package de.sovity.edc.ext.catalog.crawler.dao.config; import com.zaxxer.hikari.HikariDataSource; +import de.sovity.edc.ext.catalog.crawler.CrawlerConfigProps; import de.sovity.edc.ext.catalog.crawler.CrawlerExtension; import de.sovity.edc.extension.postgresql.HikariDataSourceFactory; import de.sovity.edc.extension.postgresql.JdbcCredentials; @@ -36,8 +37,8 @@ public class DataSourceFactory { */ public HikariDataSource newDataSource() { var jdbcCredentials = getJdbcCredentials(); - int maxPoolSize = config.getInteger(CrawlerExtension.DB_CONNECTION_POOL_SIZE); - int connectionTimeoutInMs = config.getInteger(CrawlerExtension.DB_CONNECTION_TIMEOUT_IN_MS); + int maxPoolSize = CrawlerConfigProps.CRAWLER_DB_CONNECTION_POOL_SIZE.getInt(config); + int connectionTimeoutInMs = CrawlerConfigProps.CRAWLER_DB_CONNECTION_TIMEOUT_IN_MS.getInt(config); return HikariDataSourceFactory.newDataSource( jdbcCredentials, maxPoolSize, @@ -48,16 +49,9 @@ public HikariDataSource newDataSource() { public JdbcCredentials getJdbcCredentials() { return new JdbcCredentials( - getRequiredStringProperty(config, CrawlerExtension.JDBC_URL), - getRequiredStringProperty(config, CrawlerExtension.JDBC_USER), - getRequiredStringProperty(config, CrawlerExtension.JDBC_PASSWORD) + CrawlerConfigProps.CRAWLER_DB_JDBC_URL.getStringOrThrow(config), + CrawlerConfigProps.CRAWLER_DB_JDBC_USER.getStringOrThrow(config), + CrawlerConfigProps.CRAWLER_DB_JDBC_PASSWORD.getStringOrThrow(config) ); } - - private String getRequiredStringProperty(Config config, String name) { - String value = config.getString(name, ""); - Validate.notBlank(value, "EDC Property '%s' is required".formatted(name)); - return value; - } - } diff --git a/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/FlywayService.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/FlywayService.java deleted file mode 100644 index d0d8d53ff..000000000 --- a/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/dao/config/FlywayService.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2023 sovity GmbH - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * sovity GmbH - initial implementation - * - */ - -package de.sovity.edc.ext.catalog.crawler.dao.config; - -import de.sovity.edc.ext.catalog.crawler.CrawlerExtension; -import de.sovity.edc.extension.postgresql.FlywayExecutionParams; -import de.sovity.edc.extension.postgresql.FlywayUtils; -import lombok.RequiredArgsConstructor; -import org.eclipse.edc.spi.monitor.Monitor; -import org.eclipse.edc.spi.system.configuration.Config; - -import javax.sql.DataSource; - -@RequiredArgsConstructor -public class FlywayService { - private final Config config; - private final Monitor monitor; - private final DataSource dataSource; - - public void validateOrMigrateInTests() { - var additionalLocations = config.getString(CrawlerExtension.DB_ADDITIONAL_FLYWAY_MIGRATION_LOCATIONS, ""); - - var params = baseConfig(additionalLocations) - .clean(config.getBoolean(CrawlerExtension.DB_CLEAN, false)) - .cleanEnabled(config.getBoolean(CrawlerExtension.DB_CLEAN_ENABLED, false)) - .migrate(config.getBoolean(CrawlerExtension.DB_MIGRATE, false)) - .infoLogger(monitor::info) - .build(); - - FlywayUtils.cleanAndMigrate(params, dataSource); - } - - public static FlywayExecutionParams.FlywayExecutionParamsBuilder baseConfig(String additionalMigrationLocations) { - var migrationLocations = FlywayUtils.parseFlywayLocations( - "classpath:/db/migration,%s".formatted(additionalMigrationLocations) - ); - - return FlywayExecutionParams.builder() - .migrationLocations(migrationLocations) - .table("flyway_schema_history"); - } -} diff --git a/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java index f40e6b6af..f436405eb 100644 --- a/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/main/java/de/sovity/edc/ext/catalog/crawler/orchestration/config/CrawlerConfigFactory.java @@ -14,6 +14,7 @@ package de.sovity.edc.ext.catalog.crawler.orchestration.config; +import de.sovity.edc.ext.catalog.crawler.CrawlerConfigProps; import de.sovity.edc.ext.catalog.crawler.CrawlerExtension; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -27,11 +28,11 @@ public class CrawlerConfigFactory { private final Config config; public CrawlerConfig buildCrawlerConfig() { - var environmentId = config.getString(CrawlerExtension.ENVIRONMENT_ID); - var numThreads = config.getInteger(CrawlerExtension.NUM_THREADS, 1); - var killOfflineConnectorsAfter = getDuration(CrawlerExtension.KILL_OFFLINE_CONNECTORS_AFTER, Duration.ofDays(5)); - var maxDataOffers = config.getInteger(CrawlerExtension.MAX_DATA_OFFERS_PER_CONNECTOR, -1); - var maxContractOffers = config.getInteger(CrawlerExtension.MAX_CONTRACT_OFFERS_PER_DATA_OFFER, -1); + var environmentId = CrawlerConfigProps.CRAWLER_ENVIRONMENT_ID.getStringOrThrow(config); + var numThreads = CrawlerConfigProps.CRAWLER_NUM_THREADS.getInt(config); + var killOfflineConnectorsAfter = Duration.parse(CrawlerConfigProps.CRAWLER_KILL_OFFLINE_CONNECTORS_AFTER.getStringOrThrow(config)); + var maxDataOffers = CrawlerConfigProps.CRAWLER_MAX_DATA_OFFERS_PER_CONNECTOR.getInt(config); + var maxContractOffers = CrawlerConfigProps.CRAWLER_MAX_CONTRACT_OFFERS_PER_DATA_OFFER.getInt(config); return CrawlerConfig.builder() .environmentId(environmentId) @@ -41,14 +42,4 @@ public CrawlerConfig buildCrawlerConfig() { .maxContractOffersPerDataOffer(maxContractOffers) .build(); } - - private Duration getDuration(@NonNull String configProperty, Duration defaultValue) { - var value = config.getString(configProperty, ""); - - if (StringUtils.isBlank(value)) { - return defaultValue; - } - - return Duration.parse(value); - } } diff --git a/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java index a46890706..801f3d311 100644 --- a/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java +++ b/authority-portal-backend/catalog-crawler/catalog-crawler/src/test/java/de/sovity/edc/ext/catalog/crawler/CrawlerTestDb.java @@ -2,8 +2,8 @@ import com.zaxxer.hikari.HikariDataSource; import de.sovity.edc.ext.catalog.crawler.dao.config.DslContextFactory; -import de.sovity.edc.ext.catalog.crawler.dao.config.FlywayService; import de.sovity.edc.extension.e2e.db.TestDatabaseViaTestcontainers; +import de.sovity.edc.extension.postgresql.FlywayExecutionParams; import de.sovity.edc.extension.postgresql.FlywayUtils; import de.sovity.edc.extension.postgresql.HikariDataSourceFactory; import de.sovity.edc.extension.postgresql.JdbcCredentials; @@ -39,7 +39,7 @@ public void beforeAll(ExtensionContext extensionContext) throws Exception { dslContextFactory = new DslContextFactory(dataSource); // Migrate DB - var params = FlywayService.baseConfig("classpath:/migration-test-utils") + var params = baseConfig("classpath:/migration-test-utils") .migrate(true) .build(); try { @@ -59,4 +59,14 @@ public void afterAll(ExtensionContext extensionContext) throws Exception { // Close DB db.afterAll(extensionContext); } + + public static FlywayExecutionParams.FlywayExecutionParamsBuilder baseConfig(String additionalMigrationLocations) { + var migrationLocations = FlywayUtils.parseFlywayLocations( + "classpath:/db/migration,%s".formatted(additionalMigrationLocations) + ); + + return FlywayExecutionParams.builder() + .migrationLocations(migrationLocations) + .table("flyway_schema_history"); + } }