diff --git a/server/db/pom.xml b/server/db/pom.xml index ca4b24478a..d74c68b8e3 100644 --- a/server/db/pom.xml +++ b/server/db/pom.xml @@ -16,7 +16,8 @@ library/postgres:10.4-alpine ${project.build.directory}/db - com/walmartlabs/concord/server/db/liquibase.xml + com/walmartlabs/concord/server/db/mainDb.xml + com/walmartlabs/concord/server/db/logDb.xml localhost postgres q1 @@ -164,14 +165,27 @@ ${liquibase.version} + main-db generate-sources update + + src/main/resources/${mainDb.changeLogPath} + + + + log-db + generate-sources + + update + + + src/main/resources/${logDb.changeLogPath} + - src/main/resources/${db.changeLogPath} org.postgresql.Driver jdbc:postgresql://${db.host}:${db.port}/postgres ${db.username} diff --git a/server/db/src/main/java/com/walmartlabs/concord/db/DatabaseModule.java b/server/db/src/main/java/com/walmartlabs/concord/db/DatabaseModule.java index 1942bea364..5b44d68df9 100644 --- a/server/db/src/main/java/com/walmartlabs/concord/db/DatabaseModule.java +++ b/server/db/src/main/java/com/walmartlabs/concord/db/DatabaseModule.java @@ -27,8 +27,10 @@ import javax.inject.Singleton; import javax.sql.DataSource; +import java.lang.annotation.Annotation; import java.util.Comparator; import java.util.Set; +import java.util.stream.Collectors; import static com.google.inject.multibindings.Multibinder.newSetBinder; @@ -47,46 +49,83 @@ public DatabaseModule(boolean migrateDb) { @Override public void configure(Binder binder) { newSetBinder(binder, DatabaseChangeLogProvider.class).addBinding().to(MainDBChangeLogProvider.class); + newSetBinder(binder, DatabaseChangeLogProvider.class).addBinding().to(LogDBChangeLogProvider.class); } @Provides @MainDB @Singleton - public DataSource appDataSource(@MainDB DatabaseConfiguration cfg, + public DataSource mainDbDataSource(@MainDB DatabaseConfiguration cfg, + MetricRegistry metricRegistry, + Set changeLogProviders) { + + var ds = DataSourceUtils.createDataSource(cfg, "app" /* not called "main" for backward compatibility */, cfg.username(), cfg.password(), metricRegistry); + if (migrateDb) { + migrateDb(changeLogProviders, ds, cfg, MainDB.class); + } + return ds; + } + + @Provides + @LogDB + @Singleton + public DataSource logDataSource(@LogDB DatabaseConfiguration cfg, MetricRegistry metricRegistry, Set changeLogProviders) { - DataSource ds = DataSourceUtils.createDataSource(cfg, "app", cfg.username(), cfg.password(), metricRegistry); - + var ds = DataSourceUtils.createDataSource(cfg, "log", cfg.username(), cfg.password(), metricRegistry); if (migrateDb) { - changeLogProviders.stream() - // can't inject a set of objects with the same qualifier, filter manually - .filter(p -> p.getClass().getAnnotation(MainDB.class) != null) - .sorted(Comparator.comparingInt(DatabaseChangeLogProvider::order)) - .forEach(p -> DataSourceUtils.migrateDb(ds, p, cfg.changeLogParameters())); + migrateDb(changeLogProviders, ds, cfg, LogDB.class); } - return ds; } @Provides @JsonStorageDB @Singleton - public DataSource inventoryDataSource(@JsonStorageDB DatabaseConfiguration cfg, MetricRegistry metricRegistry) { + public DataSource jsonStorageDbDataSource(@JsonStorageDB DatabaseConfiguration cfg, MetricRegistry metricRegistry) { return DataSourceUtils.createDataSource(cfg, "inventory", cfg.username(), cfg.password(), metricRegistry); } @Provides @MainDB @Singleton - public Configuration appJooqConfiguration(@MainDB DataSource ds) { + public Configuration mainDbJooqConfiguration(@MainDB DataSource ds) { + return DataSourceUtils.createJooqConfiguration(ds); + } + + @Provides + @LogDB + @Singleton + public Configuration logDbJooqConfiguration(@LogDB DataSource ds) { return DataSourceUtils.createJooqConfiguration(ds); } @Provides @JsonStorageDB @Singleton - public Configuration inventoryJooqConfiguration(@JsonStorageDB DataSource ds) { + public Configuration jsonStorageDbJooqConfiguration(@JsonStorageDB DataSource ds) { return DataSourceUtils.createJooqConfiguration(ds); } + + private static void migrateDb(Set changeLogProviders, + DataSource ds, + DatabaseConfiguration cfg, + Class annotation) { + + var providers = changeLogProviders.stream() + // can't inject a set of objects with the same qualifier, filter manually + .filter(p -> p.getClass().getAnnotation(annotation) != null) + .sorted(Comparator.comparingInt(DatabaseChangeLogProvider::order)) + .toList(); + + if (providers.isEmpty()) { + // classpath issue or a bug? + var availableProviders = changeLogProviders.stream().map(DatabaseChangeLogProvider::getChangeLogPath).sorted(); + throw new IllegalStateException("Can't find a DatabaseChangeLogProvider for %s (most likely a bug). Available providers: %s" + .formatted(annotation.getName(), availableProviders.collect(Collectors.joining(", ")))); + } + + providers.forEach(p -> DataSourceUtils.migrateDb(ds, p, cfg.changeLogParameters())); + } } diff --git a/server/db/src/main/java/com/walmartlabs/concord/db/LogDB.java b/server/db/src/main/java/com/walmartlabs/concord/db/LogDB.java new file mode 100644 index 0000000000..2266fec791 --- /dev/null +++ b/server/db/src/main/java/com/walmartlabs/concord/db/LogDB.java @@ -0,0 +1,30 @@ +package com.walmartlabs.concord.db; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2019 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import javax.inject.Qualifier; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@Qualifier +public @interface LogDB { +} diff --git a/server/db/src/main/java/com/walmartlabs/concord/db/LogDBChangeLogProvider.java b/server/db/src/main/java/com/walmartlabs/concord/db/LogDBChangeLogProvider.java new file mode 100644 index 0000000000..2e5670613a --- /dev/null +++ b/server/db/src/main/java/com/walmartlabs/concord/db/LogDBChangeLogProvider.java @@ -0,0 +1,41 @@ +package com.walmartlabs.concord.db; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2024 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +@LogDB +public class LogDBChangeLogProvider implements DatabaseChangeLogProvider { + + @Override + public String getChangeLogPath() { + return "com/walmartlabs/concord/server/db/logDb.xml"; + } + + @Override + public int order() { + // we expect the log DB to be migrated after the main DB + return 1; + } + + @Override + public String toString() { + return "log-db"; + } +} diff --git a/server/db/src/main/java/com/walmartlabs/concord/db/MainDBChangeLogProvider.java b/server/db/src/main/java/com/walmartlabs/concord/db/MainDBChangeLogProvider.java index a9696ea080..f61c274713 100644 --- a/server/db/src/main/java/com/walmartlabs/concord/db/MainDBChangeLogProvider.java +++ b/server/db/src/main/java/com/walmartlabs/concord/db/MainDBChangeLogProvider.java @@ -25,17 +25,17 @@ public class MainDBChangeLogProvider implements DatabaseChangeLogProvider { @Override public String getChangeLogPath() { - return "com/walmartlabs/concord/server/db/liquibase.xml"; + return "com/walmartlabs/concord/server/db/mainDb.xml"; } @Override public int order() { - // we expect the server's DB to be migrated first + // we expect the main DB to be migrated first return 0; } @Override public String toString() { - return "server-db"; + return "main-db"; } } diff --git a/server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb-v1.49.0.xml b/server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb-v1.49.0.xml new file mode 100644 index 0000000000..175e10e4d2 --- /dev/null +++ b/server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb-v1.49.0.xml @@ -0,0 +1,167 @@ + + + + + 7:b70de8118884fd89025bb9ed8a8b0572 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 7:10ab12d5b908de504f897e6e2a773036 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + create or replace function PROCESS_LOG_DATA_SEGMENT_NEXT_RANGE(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_SEGMENT_ID bigint, P_DATA_LEN int) + returns int4range as $$ + declare + R_START int; + begin + select coalesce(max(upper(SEGMENT_RANGE)), 0) into R_START + from PROCESS_LOG_DATA + where + INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT and SEGMENT_ID = P_SEGMENT_ID; + + if R_START is null then + R_START := 0; + end if; + + return int4range(R_START, R_START + P_DATA_LEN); + end; + $$ language plpgsql; + + + + create or replace function PROCESS_LOG_DATA_NEXT_RANGE(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_DATA_LEN int) + returns int4range as $$ + declare + R_START int; + begin + select coalesce(max(upper(LOG_RANGE)), 0) into R_START + from PROCESS_LOG_DATA + where + INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT; + + if R_START is null then + R_START := 0; + end if; + + return int4range(R_START, R_START + P_DATA_LEN); + end; + $$ language plpgsql; + + + + + + create or replace function PROCESS_LOG_DATA_SEGMENT_LAST_N_BYTES(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_SEGMENT_ID bigint, P_DATA_LEN int) + returns int4range as $$ + declare + R_START int; + begin + select coalesce(max(upper(SEGMENT_RANGE)), 0) into R_START + from PROCESS_LOG_DATA + where + INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT and SEGMENT_ID = P_SEGMENT_ID; + + if R_START is null then + R_START := 0; + end if; + + return int4range(R_START - P_DATA_LEN, R_START); + end; + $$ language plpgsql; + + + + + + create or replace function PROCESS_LOG_DATA_LAST_N_BYTES(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_DATA_LEN int) + returns int4range as $$ + declare + R_START int; + begin + select coalesce(max(upper(LOG_RANGE)), 0) into R_START + from PROCESS_LOG_DATA + where + INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT; + + if R_START is null then + R_START := 0; + end if; + + return int4range(R_START - P_DATA_LEN, R_START); + end; + $$ language plpgsql; + + + diff --git a/server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb-v1.58.0.xml b/server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb-v1.58.0.xml new file mode 100644 index 0000000000..bd8d51cc52 --- /dev/null +++ b/server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb-v1.58.0.xml @@ -0,0 +1,185 @@ + + + + + + + create or replace function ts_to_tstz(t text) + returns bool as $$ + declare + v_cnt numeric; + begin + v_cnt := 0; + + update pg_attribute + set atttypid = 'timestamp with time zone'::regtype + from pg_class + where attrelid = pg_class.oid + and relnamespace = current_schema()::regnamespace + and atttypid = 'timestamp'::regtype + and relname ilike t; + + get diagnostics v_cnt = row_count; + if v_cnt = 0 then + raise warning 'Relation not found (or is already converted): %', t; + end if; + + update pg_index + set indclass = array_to_string(array_replace(indclass::oid[], 3128::oid, 3127::oid), ' ')::oidvector + from pg_class + where indrelid = pg_class.oid + and relnamespace = current_schema()::regnamespace + and indclass::oid[] @> ARRAY[3128::oid] + and relname ilike t; + + return v_cnt > 0; + end; + $$ language plpgsql + + + + + + + + + + + + + select ts_to_tstz('process_log_data%') + + + + + + + + + + alter table PROCESS_LOG_DATA + alter column INSTANCE_CREATED_AT type timestamptz using INSTANCE_CREATED_AT at time zone 'UTC' + + + + + + + + + + + select ts_to_tstz('process_log_segments%') + + + + + + + + + + alter table PROCESS_LOG_SEGMENTS + alter column INSTANCE_CREATED_AT type timestamptz using INSTANCE_CREATED_AT at time zone 'UTC', + alter column SEGMENT_TS type timestamptz using SEGMENT_TS at time zone 'UTC' + + + + + + + + create or replace function PROCESS_LOG_DATA_SEGMENT_NEXT_RANGE(P_INSTANCE_ID uuid, P_CREATED_AT timestamptz, P_SEGMENT_ID bigint, P_DATA_LEN int) + returns int4range as $$ + declare + R_START int; + begin + select coalesce(max(upper(SEGMENT_RANGE)), 0) into R_START + from PROCESS_LOG_DATA + where + INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT and SEGMENT_ID = P_SEGMENT_ID; + + if R_START is null then + R_START := 0; + end if; + + return int4range(R_START, R_START + P_DATA_LEN); + end; + $$ language plpgsql; + + + + create or replace function PROCESS_LOG_DATA_NEXT_RANGE(P_INSTANCE_ID uuid, P_CREATED_AT timestamptz, P_DATA_LEN int) + returns int4range as $$ + declare + R_START int; + begin + select coalesce(max(upper(LOG_RANGE)), 0) into R_START + from PROCESS_LOG_DATA + where + INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT; + + if R_START is null then + R_START := 0; + end if; + + return int4range(R_START, R_START + P_DATA_LEN); + end; + $$ language plpgsql; + + + + + + create or replace function PROCESS_LOG_DATA_SEGMENT_LAST_N_BYTES(P_INSTANCE_ID uuid, P_CREATED_AT timestamptz, P_SEGMENT_ID bigint, P_DATA_LEN int) + returns int4range as $$ + declare + R_START int; + begin + select coalesce(max(upper(SEGMENT_RANGE)), 0) into R_START + from PROCESS_LOG_DATA + where + INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT and SEGMENT_ID = P_SEGMENT_ID; + + if R_START is null then + R_START := 0; + end if; + + return int4range(R_START - P_DATA_LEN, R_START); + end; + $$ language plpgsql; + + + + + + create or replace function PROCESS_LOG_DATA_LAST_N_BYTES(P_INSTANCE_ID uuid, P_CREATED_AT timestamptz, P_DATA_LEN int) + returns int4range as $$ + declare + R_START int; + begin + select coalesce(max(upper(LOG_RANGE)), 0) into R_START + from PROCESS_LOG_DATA + where + INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT; + + if R_START is null then + R_START := 0; + end if; + + return int4range(R_START - P_DATA_LEN, R_START); + end; + $$ language plpgsql; + + + diff --git a/server/db/src/main/resources/com/walmartlabs/concord/server/db/v1.81.0.xml b/server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb-v1.81.0.xml similarity index 100% rename from server/db/src/main/resources/com/walmartlabs/concord/server/db/v1.81.0.xml rename to server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb-v1.81.0.xml diff --git a/server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb.xml b/server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb.xml new file mode 100644 index 0000000000..8b707bdf73 --- /dev/null +++ b/server/db/src/main/resources/com/walmartlabs/concord/server/db/logDb.xml @@ -0,0 +1,10 @@ + + + + + + + diff --git a/server/db/src/main/resources/com/walmartlabs/concord/server/db/liquibase.xml b/server/db/src/main/resources/com/walmartlabs/concord/server/db/mainDb.xml similarity index 99% rename from server/db/src/main/resources/com/walmartlabs/concord/server/db/liquibase.xml rename to server/db/src/main/resources/com/walmartlabs/concord/server/db/mainDb.xml index e0f1f06686..631f76d3ec 100644 --- a/server/db/src/main/resources/com/walmartlabs/concord/server/db/liquibase.xml +++ b/server/db/src/main/resources/com/walmartlabs/concord/server/db/mainDb.xml @@ -95,7 +95,6 @@ - diff --git a/server/db/src/main/resources/com/walmartlabs/concord/server/db/v1.49.0.xml b/server/db/src/main/resources/com/walmartlabs/concord/server/db/v1.49.0.xml index d3a3947217..fd441bb927 100644 --- a/server/db/src/main/resources/com/walmartlabs/concord/server/db/v1.49.0.xml +++ b/server/db/src/main/resources/com/walmartlabs/concord/server/db/v1.49.0.xml @@ -11,165 +11,4 @@ - - - 7:b70de8118884fd89025bb9ed8a8b0572 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 7:10ab12d5b908de504f897e6e2a773036 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - create or replace function PROCESS_LOG_DATA_SEGMENT_NEXT_RANGE(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_SEGMENT_ID bigint, P_DATA_LEN int) - returns int4range as $$ - declare - R_START int; - begin - select coalesce(max(upper(SEGMENT_RANGE)), 0) into R_START - from PROCESS_LOG_DATA - where - INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT and SEGMENT_ID = P_SEGMENT_ID; - - if R_START is null then - R_START := 0; - end if; - - return int4range(R_START, R_START + P_DATA_LEN); - end; - $$ language plpgsql; - - - - create or replace function PROCESS_LOG_DATA_NEXT_RANGE(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_DATA_LEN int) - returns int4range as $$ - declare - R_START int; - begin - select coalesce(max(upper(LOG_RANGE)), 0) into R_START - from PROCESS_LOG_DATA - where - INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT; - - if R_START is null then - R_START := 0; - end if; - - return int4range(R_START, R_START + P_DATA_LEN); - end; - $$ language plpgsql; - - - - - - create or replace function PROCESS_LOG_DATA_SEGMENT_LAST_N_BYTES(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_SEGMENT_ID bigint, P_DATA_LEN int) - returns int4range as $$ - declare - R_START int; - begin - select coalesce(max(upper(SEGMENT_RANGE)), 0) into R_START - from PROCESS_LOG_DATA - where - INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT and SEGMENT_ID = P_SEGMENT_ID; - - if R_START is null then - R_START := 0; - end if; - - return int4range(R_START - P_DATA_LEN, R_START); - end; - $$ language plpgsql; - - - - - - create or replace function PROCESS_LOG_DATA_LAST_N_BYTES(P_INSTANCE_ID uuid, P_CREATED_AT timestamp, P_DATA_LEN int) - returns int4range as $$ - declare - R_START int; - begin - select coalesce(max(upper(LOG_RANGE)), 0) into R_START - from PROCESS_LOG_DATA - where - INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT; - - if R_START is null then - R_START := 0; - end if; - - return int4range(R_START - P_DATA_LEN, R_START); - end; - $$ language plpgsql; - - diff --git a/server/db/src/main/resources/com/walmartlabs/concord/server/db/v1.58.0.xml b/server/db/src/main/resources/com/walmartlabs/concord/server/db/v1.58.0.xml index 21ea29759e..f41173c977 100644 --- a/server/db/src/main/resources/com/walmartlabs/concord/server/db/v1.58.0.xml +++ b/server/db/src/main/resources/com/walmartlabs/concord/server/db/v1.58.0.xml @@ -301,51 +301,6 @@ - - - - - - - - select ts_to_tstz('process_log_data%') - - - - - - - - - - alter table PROCESS_LOG_DATA - alter column INSTANCE_CREATED_AT type timestamptz using INSTANCE_CREATED_AT at time zone 'UTC' - - - - - - - - - - - select ts_to_tstz('process_log_segments%') - - - - - - - - - - alter table PROCESS_LOG_SEGMENTS - alter column INSTANCE_CREATED_AT type timestamptz using INSTANCE_CREATED_AT at time zone 'UTC', - alter column SEGMENT_TS type timestamptz using SEGMENT_TS at time zone 'UTC' - - - @@ -387,92 +342,4 @@ ); - - - - - - create or replace function PROCESS_LOG_DATA_SEGMENT_NEXT_RANGE(P_INSTANCE_ID uuid, P_CREATED_AT timestamptz, P_SEGMENT_ID bigint, P_DATA_LEN int) - returns int4range as $$ - declare - R_START int; - begin - select coalesce(max(upper(SEGMENT_RANGE)), 0) into R_START - from PROCESS_LOG_DATA - where - INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT and SEGMENT_ID = P_SEGMENT_ID; - - if R_START is null then - R_START := 0; - end if; - - return int4range(R_START, R_START + P_DATA_LEN); - end; - $$ language plpgsql; - - - - create or replace function PROCESS_LOG_DATA_NEXT_RANGE(P_INSTANCE_ID uuid, P_CREATED_AT timestamptz, P_DATA_LEN int) - returns int4range as $$ - declare - R_START int; - begin - select coalesce(max(upper(LOG_RANGE)), 0) into R_START - from PROCESS_LOG_DATA - where - INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT; - - if R_START is null then - R_START := 0; - end if; - - return int4range(R_START, R_START + P_DATA_LEN); - end; - $$ language plpgsql; - - - - - - create or replace function PROCESS_LOG_DATA_SEGMENT_LAST_N_BYTES(P_INSTANCE_ID uuid, P_CREATED_AT timestamptz, P_SEGMENT_ID bigint, P_DATA_LEN int) - returns int4range as $$ - declare - R_START int; - begin - select coalesce(max(upper(SEGMENT_RANGE)), 0) into R_START - from PROCESS_LOG_DATA - where - INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT and SEGMENT_ID = P_SEGMENT_ID; - - if R_START is null then - R_START := 0; - end if; - - return int4range(R_START - P_DATA_LEN, R_START); - end; - $$ language plpgsql; - - - - - - create or replace function PROCESS_LOG_DATA_LAST_N_BYTES(P_INSTANCE_ID uuid, P_CREATED_AT timestamptz, P_DATA_LEN int) - returns int4range as $$ - declare - R_START int; - begin - select coalesce(max(upper(LOG_RANGE)), 0) into R_START - from PROCESS_LOG_DATA - where - INSTANCE_ID = P_INSTANCE_ID and INSTANCE_CREATED_AT = P_CREATED_AT; - - if R_START is null then - R_START := 0; - end if; - - return int4range(R_START - P_DATA_LEN, R_START); - end; - $$ language plpgsql; - - diff --git a/server/dist/src/main/resources/concord-server.conf b/server/dist/src/main/resources/concord-server.conf index 1473632e8f..7724c01479 100644 --- a/server/dist/src/main/resources/concord-server.conf +++ b/server/dist/src/main/resources/concord-server.conf @@ -117,6 +117,22 @@ concord-server { } } + log { + db { + url = ${db.url} + url = ${?LOG_DB_URL} + + username = ${db.appUsername} + username = ${?LOG_DB_URL} + + password = ${db.appPassword} + password = ${?LOG_DB_URL} + + maxPoolSize = 10 + maxLifetime = "5 minutes" + } + } + # "remember me" cookie support rememberMe { # max age of the "remember me" cookie diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/DatabaseConfigurationModule.java b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/DatabaseConfigurationModule.java index fdfd66fc2b..7bfd8e1af2 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/DatabaseConfigurationModule.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/DatabaseConfigurationModule.java @@ -24,6 +24,7 @@ import com.google.inject.Module; import com.walmartlabs.concord.db.DatabaseConfiguration; import com.walmartlabs.concord.db.JsonStorageDB; +import com.walmartlabs.concord.db.LogDB; import com.walmartlabs.concord.db.MainDB; import static com.google.inject.Scopes.SINGLETON; @@ -33,6 +34,7 @@ public class DatabaseConfigurationModule implements Module { @Override public void configure(Binder binder) { binder.bind(DatabaseConfiguration.class).annotatedWith(MainDB.class).to(MainDBConfiguration.class).in(SINGLETON); + binder.bind(DatabaseConfiguration.class).annotatedWith(LogDB.class).to(LogDBConfiguration.class).in(SINGLETON); binder.bind(DatabaseConfiguration.class).annotatedWith(JsonStorageDB.class).to(JsonStorageDBConfiguration.class).in(SINGLETON); } } diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/LogDBConfiguration.java b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/LogDBConfiguration.java new file mode 100644 index 0000000000..c8f068d8f3 --- /dev/null +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/cfg/LogDBConfiguration.java @@ -0,0 +1,77 @@ +package com.walmartlabs.concord.server.cfg; + +/*- + * ***** + * Concord + * ----- + * Copyright (C) 2017 - 2018 Walmart Inc. + * ----- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ===== + */ + +import com.walmartlabs.concord.config.Config; +import com.walmartlabs.concord.db.DatabaseConfiguration; +import com.walmartlabs.concord.db.LogDB; + +import javax.inject.Inject; +import java.time.Duration; + +@LogDB +public class LogDBConfiguration implements DatabaseConfiguration { + + @Inject + @Config("log.db.url") + private String url; + + @Inject + @Config("log.db.username") + private String username; + + @Inject + @Config("log.db.password") + private String password; + + @Inject + @Config("log.db.maxPoolSize") + private int maxPoolSize; + + @Inject + @Config("log.db.maxLifetime") + private Duration maxLifetime; + + @Override + public String url() { + return url; + } + + @Override + public String username() { + return username; + } + + @Override + public String password() { + return password; + } + + @Override + public int maxPoolSize() { + return maxPoolSize; + } + + @Override + public Duration maxLifetime() { + return maxLifetime; + } +} diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/process/logs/ProcessLogManager.java b/server/impl/src/main/java/com/walmartlabs/concord/server/process/logs/ProcessLogManager.java index 40c83d9976..2d64413d0b 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/process/logs/ProcessLogManager.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/process/logs/ProcessLogManager.java @@ -22,12 +22,14 @@ import com.codahale.metrics.Counter; import com.walmartlabs.concord.common.LogUtils; +import com.walmartlabs.concord.db.LogDB; import com.walmartlabs.concord.server.Listeners; import com.walmartlabs.concord.server.process.LogSegment; import com.walmartlabs.concord.server.sdk.ProcessKey; import com.walmartlabs.concord.server.sdk.Range; import com.walmartlabs.concord.server.sdk.log.ProcessLogEntry; import com.walmartlabs.concord.server.sdk.metrics.InjectCounter; +import org.jooq.Configuration; import org.jooq.DSLContext; import javax.inject.Inject; @@ -82,8 +84,8 @@ public List listSegments(ProcessKey processKey, int limit, int offse return logsDao.listSegments(processKey, limit, offset); } - public void createSystemSegment(DSLContext tx, ProcessKey processKey) { - logsDao.createSegment(tx, SYSTEM_SEGMENT_ID, processKey, null, SYSTEM_SEGMENT_NAME, null); + public void createSystemSegment(ProcessKey processKey) { + logsDao.tx(tx -> logsDao.createSegment(tx, SYSTEM_SEGMENT_ID, processKey, null, SYSTEM_SEGMENT_NAME, null)); } public long createSegment(ProcessKey processKey, UUID correlationId, String name, OffsetDateTime createdAt) { diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/process/logs/ProcessLogsDao.java b/server/impl/src/main/java/com/walmartlabs/concord/server/process/logs/ProcessLogsDao.java index d41ee934c3..01b8526a1f 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/process/logs/ProcessLogsDao.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/process/logs/ProcessLogsDao.java @@ -21,7 +21,7 @@ */ import com.walmartlabs.concord.db.AbstractDao; -import com.walmartlabs.concord.db.MainDB; +import com.walmartlabs.concord.db.LogDB; import com.walmartlabs.concord.db.PgIntRange; import com.walmartlabs.concord.server.jooq.tables.records.ProcessLogDataRecord; import com.walmartlabs.concord.server.jooq.tables.records.ProcessLogSegmentsRecord; @@ -45,10 +45,15 @@ public class ProcessLogsDao extends AbstractDao { @Inject - public ProcessLogsDao(@MainDB Configuration cfg) { + public ProcessLogsDao(@LogDB Configuration cfg) { super(cfg); } + @Override + protected void tx(Tx t) { + super.tx(t); + } + /** * Appends a chunk to the process log. Automatically calculates the chunk's range. * diff --git a/server/impl/src/main/java/com/walmartlabs/concord/server/process/queue/ProcessQueueManager.java b/server/impl/src/main/java/com/walmartlabs/concord/server/process/queue/ProcessQueueManager.java index c9b3e1e67a..5f6493c302 100644 --- a/server/impl/src/main/java/com/walmartlabs/concord/server/process/queue/ProcessQueueManager.java +++ b/server/impl/src/main/java/com/walmartlabs/concord/server/process/queue/ProcessQueueManager.java @@ -88,8 +88,9 @@ public void insert(Payload payload, ProcessStatus status) { queueDao.tx(tx -> { queueDao.insert(tx, processKey, status, kind, parentInstanceId, projectId, repoId, branchOrTag, commitId, initiatorId, meta, triggeredBy); notifyStatusChange(tx, processKey, status); - processLogManager.createSystemSegment(tx, payload.getProcessKey()); }); + + processLogManager.createSystemSegment(processKey); } /** diff --git a/server/impl/src/test/java/com/walmartlabs/concord/server/AbstractDaoTest.java b/server/impl/src/test/java/com/walmartlabs/concord/server/AbstractDaoTest.java index b13fedca09..4d392cee21 100644 --- a/server/impl/src/test/java/com/walmartlabs/concord/server/AbstractDaoTest.java +++ b/server/impl/src/test/java/com/walmartlabs/concord/server/AbstractDaoTest.java @@ -58,9 +58,9 @@ public void initDataSource() { DatabaseConfiguration cfg = new DatabaseConfigurationImpl("jdbc:postgresql://localhost:5432/postgres", "postgres", "q1", 3); DatabaseModule db = new DatabaseModule(migrateDb); - this.dataSource = db.appDataSource(cfg, new MetricRegistry(), Collections.singleton(new MainDBChangeLogProvider())); + this.dataSource = db.mainDbDataSource(cfg, new MetricRegistry(), Collections.singleton(new MainDBChangeLogProvider())); - this.cfg = db.appJooqConfiguration(this.dataSource); + this.cfg = db.mainDbJooqConfiguration(this.dataSource); } @AfterEach