From f137c1d30f0f83b842a12c1b4dc0b31e147b2387 Mon Sep 17 00:00:00 2001 From: reeferman Date: Sun, 9 Jul 2023 17:27:26 +0200 Subject: [PATCH 01/17] - comments maven central upload --- pom.xml | 110 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/pom.xml b/pom.xml index 36eb5e8..fe63ce4 100644 --- a/pom.xml +++ b/pom.xml @@ -59,61 +59,61 @@ - - org.apache.maven.plugins - maven-gpg-plugin - ${maven-gpg-plugin.version} - - - sign-artifacts - verify - - sign - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 3.2.0 - - - attach-javadoc - - jar - - - - - io.es4j.core.objects - io.es4j.infrastructure.models - - - - org.apache.maven.plugins - maven-source-plugin - ${maven-source-plugin.version} - - - attach-source - - jar - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - ${nexus-staging-maven-plugin.version} - true - - ossrh - https://s01.oss.sonatype.org/ - true - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From fb0869dc1e5d4243f037b0dbb41b6e989228f70b Mon Sep 17 00:00:00 2001 From: reeferman Date: Sun, 9 Jul 2023 19:45:49 +0200 Subject: [PATCH 02/17] - change config - debug logs --- es4j-core/src/main/java/io/es4j/core/CommandHandler.java | 4 ++-- .../main/java/io/es4j/infrastructure/bus/AggregateBus.java | 2 +- es4j-core/src/main/resources/logback.xml | 1 + .../src/main/java/io/es4j/http/HttpBridge.java | 1 - .../es4j-sql/src/main/java/io/es4j/sql/RepositoryHandler.java | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/es4j-core/src/main/java/io/es4j/core/CommandHandler.java b/es4j-core/src/main/java/io/es4j/core/CommandHandler.java index abb8bef..e59fcbc 100644 --- a/es4j-core/src/main/java/io/es4j/core/CommandHandler.java +++ b/es4j-core/src/main/java/io/es4j/core/CommandHandler.java @@ -136,11 +136,11 @@ private Uni> replayAggregateAndCache(String aggregateId, Strin AggregateState state = null; final var key = new AggregateKey<>(aggregateClass, aggregateId, tenant); if (infrastructure.cache().isPresent()) { - LOGGER.info("Fetching from cache-store {}", key); + LOGGER.debug("Fetching from cache-store {}", key); state = infrastructure.cache().get().get(key); } if (state == null) { - LOGGER.info("Fetching from event-store {}", key); + LOGGER.debug("Fetching from event-store {}", key); return playFromLastJournalOffset(aggregateId, tenant, new AggregateState<>(aggregateClass)); } else { return Uni.createFrom().item(state); diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/bus/AggregateBus.java b/es4j-core/src/main/java/io/es4j/infrastructure/bus/AggregateBus.java index 4ce58a8..5c330e7 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/bus/AggregateBus.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/bus/AggregateBus.java @@ -98,7 +98,7 @@ public static Uni waitForRegistration(String deploym } public static void broadcastActorAddress(Vertx vertx, Class entityClass, String deploymentID) { - LOGGER.info("Publishing [{}] address[{}] ", entityClass.getSimpleName(), AddressResolver.nodeAddress(entityClass, deploymentID)); + LOGGER.debug("Publishing [{}] address[{}] ", entityClass.getSimpleName(), AddressResolver.nodeAddress(entityClass, deploymentID)); vertx.eventBus().publish( AddressResolver.broadcastChannel(entityClass), AddressResolver.nodeAddress(entityClass, deploymentID), diff --git a/es4j-core/src/main/resources/logback.xml b/es4j-core/src/main/resources/logback.xml index 42c70c3..158d54d 100644 --- a/es4j-core/src/main/resources/logback.xml +++ b/es4j-core/src/main/resources/logback.xml @@ -7,6 +7,7 @@ + diff --git a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java index d06d041..b41850f 100644 --- a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java +++ b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java @@ -215,7 +215,6 @@ public Uni stop() { private HttpServer httpServer() { return vertx.createHttpServer(new HttpServerOptions() .setTracingPolicy(TracingPolicy.ALWAYS) - .setLogActivity(true) .setRegisterWebSocketWriteHandlers(true) ); } diff --git a/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/RepositoryHandler.java b/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/RepositoryHandler.java index eb2f0b4..dfc046d 100644 --- a/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/RepositoryHandler.java +++ b/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/RepositoryHandler.java @@ -81,7 +81,7 @@ private static PoolOptions pooledOptions(JsonObject config) { .setConnectionTimeoutUnit(TimeUnit.SECONDS) .setConnectionTimeout(config.getInteger("pgConnectionTimeOut", EnvVars.PG_CONNECTION_TIMEOUT)) .setPoolCleanerPeriod(config.getInteger("pgPoolCleanerPeriod", EnvVars.PG_CLEANER_PERIOD)) - .setMaxSize(config.getInteger("pgPoolMaxSize", CpuCoreSensor.availableProcessors() * 2)) + .setMaxSize(config.getInteger("pgPoolMaxSize", CpuCoreSensor.availableProcessors() * 8)) .setMaxWaitQueueSize(config.getInteger("pgMaxWaitQueueSize", -1)) .setShared(true) .setName(config.getString("schema", EnvVars.SCHEMA) + "-postgres-pooled"); @@ -249,7 +249,7 @@ public Function>>, Uni> handleInsert(Object objec return upstreamSupplier -> upstreamSupplier.get() .map(row -> { final var end = Instant.now(); - logger.info(" Inserted in " + Duration.between(start, end).toMillis() + "ms"); + logger.debug(" Inserted in " + Duration.between(start, end).toMillis() + "ms"); // return row != null && row.iterator().hasNext() ? row.iterator().next().getLong(ID) : null; return row.rowCount() == 0 ? null : (long) row.rowCount(); } From f5350b49484883eda898626c0b0b251e3c22594a Mon Sep 17 00:00:00 2001 From: reeferman Date: Sun, 9 Jul 2023 20:13:19 +0200 Subject: [PATCH 03/17] - batchsize adjustment --- .../src/main/java/io/es4j/infra/pg/PgEventStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java index cadfd86..1063797 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java @@ -220,7 +220,7 @@ private EventRecordQuery eventJournalQuery(EventStream eventStream) { null, null, null, - null, + eventStream.batchSize(), null, eventStream.tenantId() ) From 4d1ae477afed00b0bb7e1945afac464f50167421 Mon Sep 17 00:00:00 2001 From: reeferman Date: Sun, 9 Jul 2023 20:14:05 +0200 Subject: [PATCH 04/17] - page number --- .../src/main/java/io/es4j/infra/pg/PgEventStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java index 1063797..10d3913 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java @@ -219,7 +219,7 @@ private EventRecordQuery eventJournalQuery(EventStream eventStream) { eventStream.to(), null, null, - null, + 0, eventStream.batchSize(), null, eventStream.tenantId() From 7de7729ed3b3519a6ece4b4bdf369abccd1ac761 Mon Sep 17 00:00:00 2001 From: reeferman Date: Sun, 9 Jul 2023 22:01:21 +0200 Subject: [PATCH 05/17] - page number --- .../es4j/infrastructure/bus/Es4jService.java | 37 ++++++++++++++----- es4j-dependencies/pom.xml | 6 --- .../java/io/es4j/infra/pg/PgEventStore.java | 2 +- pom.xml | 4 +- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/bus/Es4jService.java b/es4j-core/src/main/java/io/es4j/infrastructure/bus/Es4jService.java index cf2ead8..e6c2c1f 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/bus/Es4jService.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/bus/Es4jService.java @@ -7,10 +7,12 @@ import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder; import com.github.victools.jsonschema.generator.SchemaVersion; import io.es4j.Aggregate; +import io.es4j.Event; import io.es4j.core.exceptions.Es4jException; import io.es4j.core.objects.*; import io.es4j.infrastructure.EventStore; import io.es4j.infrastructure.OffsetStore; +import io.es4j.infrastructure.misc.EventParser; import io.es4j.infrastructure.models.*; import io.smallrye.mutiny.Uni; import io.smallrye.mutiny.tuples.Tuple2; @@ -172,22 +174,37 @@ public Uni> fetchNextEvents(FetchNextE ); } - public Uni> fetchEvents(EventFilter eventStreamQuery) { + public Uni> fetchEvents(EventFilter eventFilter) { return eventStore.fetch( EventStreamBuilder.builder() - .offset(eventStreamQuery.offset()) - .batchSize(eventStreamQuery.batchSize()) - .tenantId(eventStreamQuery.tenantId()) - .tags(eventStreamQuery.tags()) - .to(eventStreamQuery.to()) - .from(eventStreamQuery.from()) - .versionFrom(eventStreamQuery.versionFrom()) - .versionTo(eventStreamQuery.versionTo()) - .aggregateIds(eventStreamQuery.aggregateIds()) + .offset(eventFilter.offset()) + .batchSize(eventFilter.batchSize()) + .tenantId(eventFilter.tenantId()) + .tags(eventFilter.tags()) + .to(eventFilter.to()) + .from(eventFilter.from()) + .events(figureEventClass(eventFilter.events())) + .versionFrom(eventFilter.versionFrom()) + .versionTo(eventFilter.versionTo()) + .aggregateIds(eventFilter.aggregateIds()) .build() ); } + private List> figureEventClass(List classNames) { + final var arrayList = new ArrayList>(); + classNames.forEach( + className -> { + try { + arrayList.add((Class) Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + ); + return arrayList; + } + public Uni> offsets(OffsetFilter offsetFilter) { return offsetStore.projections(offsetFilter); } diff --git a/es4j-dependencies/pom.xml b/es4j-dependencies/pom.xml index 1207a44..7cb1c9b 100644 --- a/es4j-dependencies/pom.xml +++ b/es4j-dependencies/pom.xml @@ -31,7 +31,6 @@ 9.0.0 2.0.7 0.10.2 - 5.4.3 0.1.3 @@ -111,11 +110,6 @@ reflections ${reflections.version} - - io.activej - activej-inject - ${activej.version} - io.vertx vertx-sockjs-service-proxy diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java index 10d3913..46295d1 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java @@ -208,7 +208,7 @@ private EventRecordQuery eventJournalQuery(EventStream eventStream) { eventStream.events() != null ? eventStream.events().stream().map(Class::getName).toList() : null, null, eventStream.tags(), - null, + eventStream.versionFrom(), eventStream.versionTo(), eventStream.offset(), null, diff --git a/pom.xml b/pom.xml index fe63ce4..0989894 100644 --- a/pom.xml +++ b/pom.xml @@ -27,8 +27,8 @@ 17 UTF-8 3.6.0.1398 - 4.4.1 - 3.4.2 + 4.4.3 + 3.5.0 3.3.0 1.6.13 3.1.0 From 9978d92f0f8e24edcd516059600ab3e9d0c94e9f Mon Sep 17 00:00:00 2001 From: reeferman Date: Mon, 10 Jul 2023 02:31:14 +0200 Subject: [PATCH 06/17] - page number --- diagrams/command-handler.puml | 60 +++++++++++++++++++ diagrams/hash-ring-2.puml | 15 ----- diagrams/hash-ring-append.puml | 8 +-- diagrams/projection.puml | 27 +++++++++ diagrams/retention.puml | 44 ++++++++++++++ .../java/io/es4j/LiveEventProjection.java | 12 ---- .../main/java/io/es4j/LiveEventStream.java | 30 ++++++++++ ...teProjection.java => LiveStateStream.java} | 2 +- .../java/io/es4j/PollingEventProjection.java | 39 +++++++++++- .../java/io/es4j/PollingStateProjection.java | 36 ++++++++++- .../core/projections/EventStreamListener.java | 10 ++-- .../misc/Es4jServiceLoader.java | 8 +-- 12 files changed, 247 insertions(+), 44 deletions(-) create mode 100644 diagrams/command-handler.puml delete mode 100644 diagrams/hash-ring-2.puml create mode 100644 diagrams/projection.puml create mode 100644 diagrams/retention.puml delete mode 100644 es4j-core/src/main/java/io/es4j/LiveEventProjection.java create mode 100644 es4j-core/src/main/java/io/es4j/LiveEventStream.java rename es4j-core/src/main/java/io/es4j/{LiveStateProjection.java => LiveStateStream.java} (80%) diff --git a/diagrams/command-handler.puml b/diagrams/command-handler.puml new file mode 100644 index 0000000..2a25a69 --- /dev/null +++ b/diagrams/command-handler.puml @@ -0,0 +1,60 @@ +@startuml +skinparam sequence { + ArrowColor DeepSkyBlue + ActorBorderColor DeepSkyBlue + LifeLineBorderColor blue + LifeLineBackgroundColor #A9DCDF + ParticipantBorderColor DeepSkyBlue + ParticipantBackgroundColor DodgerBlue + ParticipantFontName Impact + ParticipantFontSize 17 + ParticipantFontColor #A9DCDF + ActorBackgroundColor aqua + ActorFontColor DeepSkyBlue + ActorFontSize 17 + ActorFontName Aapex + NoteBackgroundColor wheat + NoteBorderColor sienna + NoteFontColor black +} + + +actor CommandIssuer +participant es4j +database EventStore + +CommandIssuer -> es4j: Issue Command 1 +activate es4j +es4j -> EventStore: Fetch Events +activate EventStore +EventStore --> es4j: Return Events +deactivate EventStore +es4j -> es4j: Process Command and Events in Memory +es4j -> EventStore: Append New Event +activate EventStore +EventStore --> es4j: Confirm Event Appended +deactivate EventStore +es4j --> CommandIssuer: Reply to Command 1 +deactivate es4j + +CommandIssuer -> es4j: Issue Command 2 +activate es4j +es4j -> es4j: Process Command in Memory +es4j -> EventStore: Attempt to Append New Event +activate EventStore +EventStore --> es4j: Event Append Failed +deactivate EventStore +es4j -> EventStore: Fetch Events Again +activate EventStore +EventStore --> es4j: Return Events +deactivate EventStore +es4j -> es4j: Process Command Again and Events in Memory +es4j -> EventStore: Append New Event +activate EventStore +EventStore --> es4j: Confirm Event Appended +deactivate EventStore +es4j --> CommandIssuer: Reply to Command 2 +deactivate es4j + +@enduml + diff --git a/diagrams/hash-ring-2.puml b/diagrams/hash-ring-2.puml deleted file mode 100644 index 4d2ecd6..0000000 --- a/diagrams/hash-ring-2.puml +++ /dev/null @@ -1,15 +0,0 @@ -@startuml -skinparam monochrome true -skinparam state { - BackgroundColor White - BorderColor Black - FontColor Black -} - -state "Hash Ring" as HR { - [*] --> Node1 - Node1 --> Node2 : "hash function" - Node2 --> Node3 : "hash function" - Node3 --> Node1 : "hash function" -} -@enduml diff --git a/diagrams/hash-ring-append.puml b/diagrams/hash-ring-append.puml index 784ad05..b32589c 100644 --- a/diagrams/hash-ring-append.puml +++ b/diagrams/hash-ring-append.puml @@ -19,8 +19,8 @@ skinparam sequence { } actor Client -participant "Event.x Node 1" as N1 -participant "Event.x Node 2" as N2 +participant "es4j Node 1" as N1 +participant "es4j Node 2" as N2 database "PG Event Store" as PG @@ -30,7 +30,7 @@ database "PG Event Store" as PG note right: worse case scenario 1 network hop Client -> N1 : Command(aggregateId) N1 -> N1 : findNode(aggregateId) -note right: returns "Event.x Node 2" +note right: returns "es4j Node 2" N1 -> N2: proxyCommand(aggregateId) N2 -> N2: aggregateCacheMiss N2 <-> PG: streamEvent() @@ -44,7 +44,7 @@ N1 ->> Client : AggregateState() Client -> N1 : Command2(aggregateId) N1 -> N1 : findNode(aggregateId) -note right: returns "Event.x Node 1" +note right: returns "es4j Node 1" N1 -> N1: cacheHit() N1 -> N1: handleCommand(aggregateId) N1 -> PG: appendEvents() diff --git a/diagrams/projection.puml b/diagrams/projection.puml new file mode 100644 index 0000000..f70ebf5 --- /dev/null +++ b/diagrams/projection.puml @@ -0,0 +1,27 @@ +@startuml + +actor Client +participant Bridge +participant CommandHandler +database EventStore +participant Projection + +Client -> Bridge: Issue Command +activate Bridge +Bridge -> CommandHandler: Proxy Command +activate CommandHandler +CommandHandler -> EventStore: Append Event +activate EventStore +EventStore --> CommandHandler: Confirm Event Appended +deactivate CommandHandler +EventStore -> Projection: Project Event +activate Projection +Projection --> EventStore: Confirm Projection Updated +deactivate EventStore +Client -> Projection: Query Projection +Projection --> Client: Return Query Result +deactivate Projection +Bridge --> Client: Reply to Command +deactivate Bridge + +@enduml diff --git a/diagrams/retention.puml b/diagrams/retention.puml new file mode 100644 index 0000000..d3ee317 --- /dev/null +++ b/diagrams/retention.puml @@ -0,0 +1,44 @@ +@startuml +skinparam sequence { + ArrowColor DeepSkyBlue + ActorBorderColor DeepSkyBlue + LifeLineBorderColor blue + LifeLineBackgroundColor #A9DCDF + ParticipantBorderColor DeepSkyBlue + ParticipantBackgroundColor DodgerBlue + ParticipantFontName Impact + ParticipantFontSize 17 + ParticipantFontColor #A9DCDF + ActorBackgroundColor aqua + ActorFontColor DeepSkyBlue + ActorFontSize 17 + ActorFontName Aapex +} +participant JournalCompressor as "Es4j" +database MainEventStore +database SecondaryEventStore + +activate JournalCompressor +JournalCompressor -> MainEventStore: Query for latest snapshot +activate MainEventStore +MainEventStore --> JournalCompressor: Return latest snapshot +deactivate MainEventStore + +JournalCompressor -> MainEventStore: Get events before snapshot +activate MainEventStore +MainEventStore --> JournalCompressor: Return old events +deactivate MainEventStore + +JournalCompressor -> SecondaryEventStore: Copy old events +activate SecondaryEventStore +SecondaryEventStore --> JournalCompressor: Confirm events copied +deactivate SecondaryEventStore + +JournalCompressor -> MainEventStore: Delete old events +activate MainEventStore +MainEventStore --> JournalCompressor: Confirm old events deleted +deactivate MainEventStore + +deactivate JournalCompressor + +@enduml diff --git a/es4j-core/src/main/java/io/es4j/LiveEventProjection.java b/es4j-core/src/main/java/io/es4j/LiveEventProjection.java deleted file mode 100644 index 94f63e9..0000000 --- a/es4j-core/src/main/java/io/es4j/LiveEventProjection.java +++ /dev/null @@ -1,12 +0,0 @@ -package io.es4j; - -import io.es4j.infrastructure.models.Event; -import io.smallrye.mutiny.Uni; - -public interface LiveEventProjection { - Uni apply(Event event); - default String tenant() { - return "default"; - } - -} diff --git a/es4j-core/src/main/java/io/es4j/LiveEventStream.java b/es4j-core/src/main/java/io/es4j/LiveEventStream.java new file mode 100644 index 0000000..232e1c2 --- /dev/null +++ b/es4j-core/src/main/java/io/es4j/LiveEventStream.java @@ -0,0 +1,30 @@ +package io.es4j; + +import io.es4j.infrastructure.models.Event; +import io.smallrye.mutiny.Uni; + +/** + * The LiveEventStream interface defines the structure for live streams of events + * within the framework. Implementors of this interface will be called in a best-effort + * manner for emitted events. + */ +public interface LiveEventStream { + + /** + * Apply an Event to the live stream. This method will be called in a best-effort + * manner for emitted events. + * + * @param event the Event to apply + * @return a Uni representing the completion of the operation + */ + Uni apply(Event event); + + /** + * Retrieve the tenant associated with the live event stream. + * + * @return a String representing the tenant. Defaults to "default". + */ + default String tenant() { + return "default"; + } +} diff --git a/es4j-core/src/main/java/io/es4j/LiveStateProjection.java b/es4j-core/src/main/java/io/es4j/LiveStateStream.java similarity index 80% rename from es4j-core/src/main/java/io/es4j/LiveStateProjection.java rename to es4j-core/src/main/java/io/es4j/LiveStateStream.java index 749637d..12d682b 100644 --- a/es4j-core/src/main/java/io/es4j/LiveStateProjection.java +++ b/es4j-core/src/main/java/io/es4j/LiveStateStream.java @@ -4,7 +4,7 @@ import io.smallrye.mutiny.Uni; // create javadoc for this interface -public interface LiveStateProjection { +public interface LiveStateStream { Uni update(AggregateState currentState); diff --git a/es4j-core/src/main/java/io/es4j/PollingEventProjection.java b/es4j-core/src/main/java/io/es4j/PollingEventProjection.java index 5179aff..b890a16 100644 --- a/es4j-core/src/main/java/io/es4j/PollingEventProjection.java +++ b/es4j-core/src/main/java/io/es4j/PollingEventProjection.java @@ -13,19 +13,43 @@ import java.util.List; import java.util.Optional; +/** + * The PollingEventProjection interface defines the structure for event projections + * that use a polling strategy. + */ public interface PollingEventProjection { + /** + * Apply the list of AggregateEvent objects to the projection. + * + * @param events the list of AggregateEvent objects to apply + * @return a Uni representing the completion of the operation + */ Uni apply(List events); + /** + * Optional filter for the event journal. + * + * @return an Optional containing the EventJournalFilter, if one is defined + */ default Optional filter() { return Optional.empty(); } + /** + * Retrieve the tenant associated with the projection. + * + * @return a String representing the tenant. Defaults to "default". + */ default String tenant() { return "default"; } - // + /** + * Defines the polling policy for the projection. + * + * @return a Cron object representing the polling policy. Defaults to once every minute. + */ default Cron pollingPolicy() { return new CronParser(CronDefinitionBuilder .instanceDefinitionFor(CronType.UNIX) @@ -33,9 +57,20 @@ default Cron pollingPolicy() { .parse("*/1 * * * *"); } + /** + * Setup the projection with the given Vertx and configuration. + * + * @param vertx the Vertx to use for the setup + * @param configuration the configuration to use for the setup + * @return a Uni representing the completion of the setup operation + */ Uni setup(Vertx vertx, JsonObject configuration); + /** + * Retrieve the class of the aggregate associated with the projection. + * + * @return the Class of the aggregate associated with the projection + */ Class aggregateClass(); - } diff --git a/es4j-core/src/main/java/io/es4j/PollingStateProjection.java b/es4j-core/src/main/java/io/es4j/PollingStateProjection.java index 61bd2cb..eac75f6 100644 --- a/es4j-core/src/main/java/io/es4j/PollingStateProjection.java +++ b/es4j-core/src/main/java/io/es4j/PollingStateProjection.java @@ -7,18 +7,52 @@ import io.es4j.core.objects.AggregateState; import io.smallrye.mutiny.Uni; +/** + * An interface for polling state projections of aggregates in an event sourcing system. + * + *

This interface defines methods for updating the current state of an aggregate, and provides + * default implementations for retrieving the tenant and the polling policy.

+ * + *

Implementors of this interface should provide a specific implementation of the + * {@link #update(AggregateState)} method, which is used to update the current state + * of an aggregate.

+ * + * @param the type of the aggregate + */ public interface PollingStateProjection { + /** + * Updates the current state of an aggregate. + * + * @param currentState the current state of the aggregate + * @return a Uni which represents the completion of the update operation + */ Uni update(AggregateState currentState); + + /** + * Returns the tenant for this polling state projection. + * + *

The default implementation returns "default", but this can be overridden by subclasses.

+ * + * @return a string representing the tenant + */ default String tenant() { return "default"; } + /** + * Returns the polling policy for this polling state projection. + * + *

The default implementation returns a Cron policy that triggers every minute. This can be + * overridden by subclasses to provide a different polling policy.

+ * + * @return a Cron object representing the polling policy + */ default Cron pollingPolicy() { return new CronParser(CronDefinitionBuilder .instanceDefinitionFor(CronType.UNIX) ) .parse("*/1 * * * *"); } - } + diff --git a/es4j-core/src/main/java/io/es4j/core/projections/EventStreamListener.java b/es4j-core/src/main/java/io/es4j/core/projections/EventStreamListener.java index fff9c2e..294cade 100644 --- a/es4j-core/src/main/java/io/es4j/core/projections/EventStreamListener.java +++ b/es4j-core/src/main/java/io/es4j/core/projections/EventStreamListener.java @@ -1,7 +1,7 @@ package io.es4j.core.projections; import io.es4j.Aggregate; -import io.es4j.LiveEventProjection; +import io.es4j.LiveEventStream; import io.smallrye.mutiny.Uni; import io.vertx.mutiny.core.Vertx; import org.slf4j.Logger; @@ -13,15 +13,15 @@ public class EventStreamListener { private final Vertx vertx; private final Class aggregateClass; - private final List liveEventProjectionConsumer; + private final List liveEventStreamConsumer; protected static final Logger LOGGER = LoggerFactory.getLogger(EventStreamListener.class); - public EventStreamListener(Vertx vertx, Class aggregateClass, List liveEventProjectionConsumer) { + public EventStreamListener(Vertx vertx, Class aggregateClass, List liveEventStreamConsumer) { this.vertx = vertx; this.aggregateClass = aggregateClass; - this.liveEventProjectionConsumer = liveEventProjectionConsumer; + this.liveEventStreamConsumer = liveEventStreamConsumer; } @@ -41,7 +41,7 @@ public Uni start() { // .replaceWithVoid(); } - public static void handle(Throwable throwable, LiveEventProjection consumer) { + public static void handle(Throwable throwable, LiveEventStream consumer) { LOGGER.error("Error in live event stream {}::{}", consumer.getClass().getName(), consumer.tenant(), throwable); } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java b/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java index 00ede09..d1c84da 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java @@ -47,14 +47,14 @@ public static List pollingEventProjections() { .toList(); } - public static List liveEventProjections() { - return ServiceLoader.load(LiveEventProjection.class).stream() + public static List liveEventProjections() { + return ServiceLoader.load(LiveEventStream.class).stream() .map(ServiceLoader.Provider::get) .toList(); } - public static List liveStateProjections() { - return ServiceLoader.load(LiveStateProjection.class).stream() + public static List liveStateProjections() { + return ServiceLoader.load(LiveStateStream.class).stream() .map(ServiceLoader.Provider::get) .toList(); } From 2e42a3066decf6825ae4cb0908b0fbf984479158 Mon Sep 17 00:00:00 2001 From: reeferman Date: Mon, 10 Jul 2023 03:33:29 +0200 Subject: [PATCH 07/17] - page number --- diagrams/hash-ring-synchronization.puml | 2 +- diagrams/hash-ring.puml | 8 ++--- diagrams/sharding.puml | 45 +++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 diagrams/sharding.puml diff --git a/diagrams/hash-ring-synchronization.puml b/diagrams/hash-ring-synchronization.puml index 44b5490..3559a6f 100644 --- a/diagrams/hash-ring-synchronization.puml +++ b/diagrams/hash-ring-synchronization.puml @@ -15,7 +15,7 @@ skinparam sequence { ActorFontName Aapex } -participant "Event.x Bus" as LB +participant "Es4j Bus" as LB note over LB: the bus is embeded in each node \n this is just to make the visualization easier database "Node 1" as N1 database "Node 2" as N2 diff --git a/diagrams/hash-ring.puml b/diagrams/hash-ring.puml index 4e4dc80..bb1806d 100644 --- a/diagrams/hash-ring.puml +++ b/diagrams/hash-ring.puml @@ -19,8 +19,8 @@ skinparam sequence { } actor Client -participant "Event.x Node 1" as N1 -participant "Event.x Node 2" as N2 +participant "es4j Node 1" as N1 +participant "es4j Node 2" as N2 == Worse Case Scenario == @@ -29,7 +29,7 @@ participant "Event.x Node 2" as N2 note right: worse case scenario 1 network hop Client -> N1 : Command(aggregateId) N1 -> N1 : findNode(aggregateId) -note right: returns "Event.x Node 2" +note right: returns "es4j Node 2" N1 -> N2: proxyCommand(aggregateId) N2 -> N2: handleCommand(aggregateId) N2 ->> N1 : AggregateState() @@ -40,7 +40,7 @@ N1 ->> Client : AggregateState() Client -> N1 : Command2(aggregateId) N1 -> N1 : findNode(aggregateId) -note right: returns "Event.x Node 1" +note right: returns "es4j Node 1" N1 -> N1: handleCommand(aggregateId) N1 ->> Client : AggregateState() diff --git a/diagrams/sharding.puml b/diagrams/sharding.puml new file mode 100644 index 0000000..31800eb --- /dev/null +++ b/diagrams/sharding.puml @@ -0,0 +1,45 @@ +@startuml + +skinparam sequence { + ArrowColor DeepSkyBlue + ActorBorderColor DeepSkyBlue + LifeLineBorderColor blue + LifeLineBackgroundColor #A9DCDF + ParticipantBorderColor DeepSkyBlue + ParticipantBackgroundColor DodgerBlue + ParticipantFontName Impact + ParticipantFontSize 17 + ParticipantFontColor #A9DCDF + ActorBackgroundColor aqua + ActorFontColor DeepSkyBlue + ActorFontSize 17 + ActorFontName Aapex + NoteBackgroundColor wheat + NoteBorderColor sienna + NoteFontColor black +} +!define DEFAULT_MONO_FONT_SIZE 14 + +skinparam monochrome true + +title es4j Architecture + +actor Client +component HashingFunction +node Node1 +node Node2 +node Node3 +database Aggregate1 +database Aggregate2 +database Aggregate3 + +Client --> HashingFunction : Sends Command with Aggregate Key +HashingFunction --> Node1 : Directs command for Aggregate1 +HashingFunction --> Node2 : Directs command for Aggregate2 +HashingFunction --> Node3 : Directs command for Aggregate3 + +Node1 --> Aggregate1 : Executes command +Node2 --> Aggregate2 : Executes command +Node3 --> Aggregate3 : Executes command + +@enduml From 4f6c1881fefc62cc456eed166c18ad5b5d25352e Mon Sep 17 00:00:00 2001 From: reeferman Date: Wed, 12 Jul 2023 22:09:19 +0200 Subject: [PATCH 08/17] - various fixes - uses evenType instead of event classes - removes event related schema handling to aggregators --- diagrams/command-handler.puml | 2 + diagrams/config.puml | 48 ++++++++ diagrams/configmaps.puml | 47 ++++++++ diagrams/event-sequence.puml | 50 +++++++++ diagrams/hash-ring-append.puml | 3 + diagrams/hash-ring-synchronization.puml | 3 + diagrams/hash-ring.puml | 2 + diagrams/projection.puml | 22 ++++ diagrams/retention.puml | 4 + diagrams/sharding.puml | 41 ++++--- .../src/main/java/io/es4j/Aggregator.java | 6 +- .../src/main/java/io/es4j/Behaviour.java | 6 +- .../src/main/java/io/es4j/Bootstrap.java | 4 + es4j-core/src/main/java/io/es4j/Event.java | 18 +-- .../main/java/io/es4j/LiveEventStream.java | 6 +- .../java/io/es4j/core/CommandHandler.java | 106 +++++++++--------- .../io/es4j/core/exceptions/UnknownEvent.java | 5 +- .../io/es4j/core/objects/AggregateState.java | 1 + .../es4j/core/objects/EventJournalFilter.java | 4 +- .../io/es4j/core/tasks/EventLogTrimmer.java | 2 +- .../core/tasks/EventProjectionPoller.java | 4 +- .../core/tasks/StateProjectionPoller.java | 2 +- .../es4j/infrastructure/bus/Es4jService.java | 33 ++---- .../cache/CaffeineAggregateCache.java | 9 +- .../es4j/infrastructure/misc/EventParser.java | 9 ++ .../models/AggregateEventStream.java | 4 +- .../models/AvailableAggregate.java | 12 ++ .../infrastructure/models/AvailableTypes.java | 4 +- .../io/es4j/infrastructure/models/Event.java | 4 +- .../infrastructure/models/EventFilter.java | 2 +- .../infrastructure/models/EventStream.java | 3 +- .../main/java/io/es4j/launcher/Es4jMain.java | 18 +++ .../java/io/es4j/http/OpenApiGenerator.java | 2 +- .../java/io/es4j/infra/pg/PgEventStore.java | 13 +-- .../es4j/infra/pg/PgSecondaryEventStore.java | 14 +-- .../io/es4j/infra/pg/models/EventRecord.java | 2 +- .../src/main/resources/pg-event-store.xml | 9 +- .../io/es4j/infra/redis/RedisEventStore.java | 2 +- es4j-test/pom.xml | 5 + .../src/main/java/io/es4j/Es4jExtension.java | 2 +- .../src/test/java/io/es4j/BridgeTest.java | 4 +- .../io/es4j/behaviours/ChangedAggregator.java | 5 +- .../io/es4j/behaviours/CreateAggregator.java | 5 + .../es4j/hashing/ConsistentHashingTest.java | 6 + .../es4j/infrastructure/EventStoreTest.java | 6 +- 45 files changed, 393 insertions(+), 166 deletions(-) create mode 100644 diagrams/config.puml create mode 100644 diagrams/configmaps.puml create mode 100644 diagrams/event-sequence.puml create mode 100644 es4j-core/src/main/java/io/es4j/infrastructure/models/AvailableAggregate.java diff --git a/diagrams/command-handler.puml b/diagrams/command-handler.puml index 2a25a69..6679169 100644 --- a/diagrams/command-handler.puml +++ b/diagrams/command-handler.puml @@ -17,7 +17,9 @@ skinparam sequence { NoteBorderColor sienna NoteFontColor black } +!define DEFAULT_MONO_FONT_SIZE 14 +skinparam monochrome true actor CommandIssuer participant es4j diff --git a/diagrams/config.puml b/diagrams/config.puml new file mode 100644 index 0000000..cce9442 --- /dev/null +++ b/diagrams/config.puml @@ -0,0 +1,48 @@ +@startuml + +skinparam sequence { + ArrowColor DeepSkyBlue + ActorBorderColor DeepSkyBlue + LifeLineBorderColor blue + LifeLineBackgroundColor #A9DCDF + ParticipantBorderColor DeepSkyBlue + ParticipantBackgroundColor DodgerBlue + ParticipantFontName Impact + ParticipantFontSize 17 + ParticipantFontColor #A9DCDF + ActorBackgroundColor aqua + ActorFontColor DeepSkyBlue + ActorFontSize 17 + ActorFontName Aapex + NoteBackgroundColor wheat + NoteBorderColor sienna + NoteFontColor black +} +!define DEFAULT_MONO_FONT_SIZE 14 +skinparam monochrome true + +actor "Client" as f +participant "ES4J Node1" as B +database "Config Store" as C +participant "ES4J Node2" as D +participant "ES4J Node3" as E + + +f -> B: Send configuration +activate B +activate C +B -> C: Update table with configuration +C -> B: Emit event +B -> B: Update internal cache +deactivate B +activate D +C -> D: Emit event +D -> D: Update internal cache +deactivate D + +activate E +C -> E: Emit event +deactivate C +E -> E: Update internal cache +deactivate E +@enduml diff --git a/diagrams/configmaps.puml b/diagrams/configmaps.puml new file mode 100644 index 0000000..85285ea --- /dev/null +++ b/diagrams/configmaps.puml @@ -0,0 +1,47 @@ +@startuml + +skinparam sequence { + ArrowColor DeepSkyBlue + ActorBorderColor DeepSkyBlue + LifeLineBorderColor blue + LifeLineBackgroundColor #A9DCDF + ParticipantBorderColor DeepSkyBlue + ParticipantBackgroundColor DodgerBlue + ParticipantFontName Impact + ParticipantFontSize 17 + ParticipantFontColor #A9DCDF + ActorBackgroundColor aqua + ActorFontColor DeepSkyBlue + ActorFontSize 17 + ActorFontName Aapex + NoteBackgroundColor wheat + NoteBorderColor sienna + NoteFontColor black +} +!define DEFAULT_MONO_FONT_SIZE 14 + +skinparam monochrome true + +participant "ES4J Node1" as A +participant "ES4J Node2" as B +participant "ES4J Node3" as C +participant "ES4J Node4" as E +participant "ES4J Node5" as F +database "Kubernetes ConfigMap1" as D +database "EventStore ConfigMap" as G + +A -> D: Put Watch +activate D +B -> D: Put Watch +C -> D: Put Watch +note right of D: Watches by 3 ES4J Nodes are active + +E -> G: Put Watch +activate G +F -> G: Put Watch +A -> G: Put Watch +B -> G: Put Watch +C -> G: Put Watch +note right of G: Watches by 5 ES4J Nodes are active + +@enduml diff --git a/diagrams/event-sequence.puml b/diagrams/event-sequence.puml new file mode 100644 index 0000000..98ec0f5 --- /dev/null +++ b/diagrams/event-sequence.puml @@ -0,0 +1,50 @@ +@startuml +skinparam sequence { + ArrowColor DeepSkyBlue + ActorBorderColor DeepSkyBlue + LifeLineBorderColor blue + LifeLineBackgroundColor #A9DCDF + ParticipantBorderColor DeepSkyBlue + ParticipantBackgroundColor DodgerBlue + ParticipantFontName Impact + ParticipantFontSize 17 + ParticipantFontColor #A9DCDF + ActorBackgroundColor aqua + ActorFontColor DeepSkyBlue + ActorFontSize 17 + ActorFontName Aapex + NoteBackgroundColor wheat + NoteBorderColor sienna + NoteFontColor black +} + +!define DEFAULT_MONO_FONT_SIZE 14 +skinparam monochrome true + +actor Client +participant es4j as A +database "Event Store" as C + +Client -> A : Sends Command with Aggregate Key(aggregateID, tenant) +activate A +A -> A : hash(AggregateKey) +A -> A : handleCommand(Command) + +note over A: Generate 3 events +A -> A: Event 1 with tuple (aggregateID, tenant, 1) +A -> A: Event 2 with tuple (aggregateID, tenant, 2) +A -> A: Event 3 with tuple (aggregateID, tenant, 3) + +A -> C : Persist Events to Event Store +activate C +deactivate A +note right of C: Events are stored + +C --> A : Acknowledgement +deactivate C +activate A + +A --> Client : Return Aggregate State +deactivate A + +@enduml diff --git a/diagrams/hash-ring-append.puml b/diagrams/hash-ring-append.puml index b32589c..ce97c8a 100644 --- a/diagrams/hash-ring-append.puml +++ b/diagrams/hash-ring-append.puml @@ -17,6 +17,9 @@ skinparam sequence { NoteBorderColor sienna NoteFontColor black } +!define DEFAULT_MONO_FONT_SIZE 14 + +skinparam monochrome true actor Client participant "es4j Node 1" as N1 diff --git a/diagrams/hash-ring-synchronization.puml b/diagrams/hash-ring-synchronization.puml index 3559a6f..feec62c 100644 --- a/diagrams/hash-ring-synchronization.puml +++ b/diagrams/hash-ring-synchronization.puml @@ -14,6 +14,9 @@ skinparam sequence { ActorFontSize 17 ActorFontName Aapex } +!define DEFAULT_MONO_FONT_SIZE 14 + +skinparam monochrome true participant "Es4j Bus" as LB note over LB: the bus is embeded in each node \n this is just to make the visualization easier diff --git a/diagrams/hash-ring.puml b/diagrams/hash-ring.puml index bb1806d..51596ec 100644 --- a/diagrams/hash-ring.puml +++ b/diagrams/hash-ring.puml @@ -17,7 +17,9 @@ skinparam sequence { NoteBorderColor sienna NoteFontColor black } +!define DEFAULT_MONO_FONT_SIZE 14 +skinparam monochrome true actor Client participant "es4j Node 1" as N1 participant "es4j Node 2" as N2 diff --git a/diagrams/projection.puml b/diagrams/projection.puml index f70ebf5..37a2a68 100644 --- a/diagrams/projection.puml +++ b/diagrams/projection.puml @@ -1,5 +1,27 @@ @startuml +skinparam sequence { + ArrowColor DeepSkyBlue + ActorBorderColor DeepSkyBlue + LifeLineBorderColor blue + LifeLineBackgroundColor #A9DCDF + ParticipantBorderColor DeepSkyBlue + ParticipantBackgroundColor DodgerBlue + ParticipantFontName Impact + ParticipantFontSize 17 + ParticipantFontColor #A9DCDF + ActorBackgroundColor aqua + ActorFontColor DeepSkyBlue + ActorFontSize 17 + ActorFontName Aapex + NoteBackgroundColor wheat + NoteBorderColor sienna + NoteFontColor black +} +!define DEFAULT_MONO_FONT_SIZE 14 + +skinparam monochrome true + actor Client participant Bridge participant CommandHandler diff --git a/diagrams/retention.puml b/diagrams/retention.puml index d3ee317..4436501 100644 --- a/diagrams/retention.puml +++ b/diagrams/retention.puml @@ -14,6 +14,10 @@ skinparam sequence { ActorFontSize 17 ActorFontName Aapex } +!define DEFAULT_MONO_FONT_SIZE 14 + +skinparam monochrome true + participant JournalCompressor as "Es4j" database MainEventStore database SecondaryEventStore diff --git a/diagrams/sharding.puml b/diagrams/sharding.puml index 31800eb..51866fb 100644 --- a/diagrams/sharding.puml +++ b/diagrams/sharding.puml @@ -1,5 +1,4 @@ @startuml - skinparam sequence { ArrowColor DeepSkyBlue ActorBorderColor DeepSkyBlue @@ -19,27 +18,37 @@ skinparam sequence { NoteFontColor black } !define DEFAULT_MONO_FONT_SIZE 14 - skinparam monochrome true -title es4j Architecture - actor Client component HashingFunction -node Node1 -node Node2 -node Node3 -database Aggregate1 -database Aggregate2 -database Aggregate3 +node Node1 as "Es4j Node" +node Node2 as "Es4j Node" +node Node3 as "Es4j Node" +database Aggregate1_1 +database Aggregate1_2 +database Aggregate1_3 +database Aggregate2_1 +database Aggregate2_2 +database Aggregate2_3 +database Aggregate3_1 +database Aggregate3_2 +database Aggregate3_3 Client --> HashingFunction : Sends Command with Aggregate Key -HashingFunction --> Node1 : Directs command for Aggregate1 -HashingFunction --> Node2 : Directs command for Aggregate2 -HashingFunction --> Node3 : Directs command for Aggregate3 +HashingFunction --> Node1 : Directs command for Node1 Aggregates +HashingFunction --> Node2 : Directs command for Node2 Aggregates +HashingFunction --> Node3 : Directs command for Node3 Aggregates + +Node1 --> Aggregate1_1 : Executes command +Node1 --> Aggregate1_2 : Executes command +Node1 --> Aggregate1_3 : Executes command -Node1 --> Aggregate1 : Executes command -Node2 --> Aggregate2 : Executes command -Node3 --> Aggregate3 : Executes command +Node2 --> Aggregate2_1 : Executes command +Node2 --> Aggregate2_2 : Executes command +Node2 --> Aggregate2_3 : Executes command +Node3 --> Aggregate3_1 : Executes command +Node3 --> Aggregate3_2 : Executes command +Node3 --> Aggregate3_3 : Executes command @enduml diff --git a/es4j-core/src/main/java/io/es4j/Aggregator.java b/es4j-core/src/main/java/io/es4j/Aggregator.java index 1732c28..e6c8e50 100644 --- a/es4j-core/src/main/java/io/es4j/Aggregator.java +++ b/es4j-core/src/main/java/io/es4j/Aggregator.java @@ -7,7 +7,7 @@ /** * The Aggregator interface represents an entity responsible for - * applying events to an aggregate root and handling event version migration + * applying eventTypes to an aggregate root and handling event version migration * in an event sourcing based system. * * @param The type of the aggregate, which must implement the Aggregate interface. @@ -31,7 +31,7 @@ public interface Aggregator { * * @return The current schema version as an integer, default is 0. */ - default int currentSchemaVersion() { + default int schemaVersion() { return 0; } @@ -57,5 +57,7 @@ default E transformFrom(int schemaVersion, JsonObject event) { ); } + String eventType(); + } diff --git a/es4j-core/src/main/java/io/es4j/Behaviour.java b/es4j-core/src/main/java/io/es4j/Behaviour.java index fac9532..b8aa9c2 100644 --- a/es4j-core/src/main/java/io/es4j/Behaviour.java +++ b/es4j-core/src/main/java/io/es4j/Behaviour.java @@ -6,7 +6,7 @@ /** * The Behaviour interface represents the behavior of an aggregate in response * to a command. It is responsible for processing commands, validating them, and - * emitting events that should be applied to the aggregate. + * emitting eventTypes that should be applied to the aggregate. * * @param The type of the aggregate, which must implement the Aggregate interface. * @param The type of the command, which must implement the Command interface. @@ -15,12 +15,12 @@ public interface Behaviour { /** * Processes the given command against the provided aggregate state. - * This involves validating the command and emitting events that represent + * This involves validating the command and emitting eventTypes that represent * the changes to be applied to the aggregate. * * @param state The current state of the aggregate. * @param command The command to be processed. - * @return A list of events that should be applied to the aggregate. + * @return A list of eventTypes that should be applied to the aggregate. */ List process(T state, C command); diff --git a/es4j-core/src/main/java/io/es4j/Bootstrap.java b/es4j-core/src/main/java/io/es4j/Bootstrap.java index 8e30ed0..d32d7d4 100644 --- a/es4j-core/src/main/java/io/es4j/Bootstrap.java +++ b/es4j-core/src/main/java/io/es4j/Bootstrap.java @@ -28,4 +28,8 @@ default List fileConfigurations() { return Collections.emptyList(); } + default List tenants() { + return List.of("default"); + } + } diff --git a/es4j-core/src/main/java/io/es4j/Event.java b/es4j-core/src/main/java/io/es4j/Event.java index bb31c8a..06b98fe 100644 --- a/es4j-core/src/main/java/io/es4j/Event.java +++ b/es4j-core/src/main/java/io/es4j/Event.java @@ -10,27 +10,16 @@ /** * The Event interface represents an event that is emitted from the Behaviour - * interface as a result of processing commands. It must be implemented by all events. + * interface as a result of processing commands. It must be implemented by all eventTypes. * Events are used to record state changes in an aggregate. Whenever the content * of the event changes, the schema version must be incremented. * Event extends Shareable and Serializable interfaces. */ public interface Event extends Shareable, Serializable { - /** - * Retrieves the schema version of this event. The schema version must be - * incremented whenever the content of the event changes. - * This method provides a default implementation and returns 0. - * - * @return The schema version as an integer, default is 0. - */ - default int schemaVersion() { - return 0; - } - /** * Retrieves the list of tags associated with this event. - * Tags can be used for categorization or filtering of events. + * Tags can be used for categorization or filtering of eventTypes. * This method provides a default implementation that returns an empty list. * * @return A list of tags as strings. Default is an empty list. @@ -39,8 +28,5 @@ default List tags() { return Collections.emptyList(); } - // TODO: Add mandatory event-type and use that instead of the event class when storing in event log - // At startup, save the map of event-type -> class - } diff --git a/es4j-core/src/main/java/io/es4j/LiveEventStream.java b/es4j-core/src/main/java/io/es4j/LiveEventStream.java index 232e1c2..a3e209e 100644 --- a/es4j-core/src/main/java/io/es4j/LiveEventStream.java +++ b/es4j-core/src/main/java/io/es4j/LiveEventStream.java @@ -4,15 +4,15 @@ import io.smallrye.mutiny.Uni; /** - * The LiveEventStream interface defines the structure for live streams of events + * The LiveEventStream interface defines the structure for live streams of eventTypes * within the framework. Implementors of this interface will be called in a best-effort - * manner for emitted events. + * manner for emitted eventTypes. */ public interface LiveEventStream { /** * Apply an Event to the live stream. This method will be called in a best-effort - * manner for emitted events. + * manner for emitted eventTypes. * * @param event the Event to apply * @return a Uni representing the completion of the operation diff --git a/es4j-core/src/main/java/io/es4j/core/CommandHandler.java b/es4j-core/src/main/java/io/es4j/core/CommandHandler.java index e59fcbc..09361f8 100644 --- a/es4j-core/src/main/java/io/es4j/core/CommandHandler.java +++ b/es4j-core/src/main/java/io/es4j/core/CommandHandler.java @@ -76,11 +76,11 @@ private Uni replayAndAppend(Command command) { return replayAggregateAndCache(command.aggregateId(), command.tenant()) .flatMap(aggregateState -> processCommand(aggregateState, command) .onFailure(Conflict.class).recoverWithUni( - () -> playFromLastJournalOffset(command.aggregateId(), command.tenant(), aggregateState) + () -> playFromLastSnapshot(command.aggregateId(), command.tenant(), aggregateState) .flatMap(reconstructedState -> processCommand(reconstructedState, command)) .onFailure(Conflict.class).retry().atMost(5) ) - .onFailure().invoke(throwable -> logRejectedCommand(throwable, command)) + .onFailure().invoke(throwable -> logRejectedCommand(throwable, command, aggregateState)) ) .map(AggregateState::toJson); } @@ -90,19 +90,19 @@ private Uni replayAndSimulate(Command command) { .map(aggregateState -> { checkCommandId(aggregateState, command); final var events = applyCommandBehaviour(aggregateState, command); - applyEvents(aggregateState, events); + aggregateEvents(aggregateState, events); return aggregateState.toJson(); } ); } - private T aggregateEvent(T aggregateState, final Event event) { + private T aggregateEvent(T aggregateState, final Event event, Integer eventSchemaVersion) { Event finalEvent = event; final var aggregator = findAggregator(event); - LOGGER.debug("Applying {} schema versionTo {} ", aggregator.delegate().getClass().getSimpleName(), aggregator.delegate().currentSchemaVersion()); - if (aggregator.delegate().currentSchemaVersion() != event.schemaVersion()) { + LOGGER.debug("Applying {} schema versionTo {} ", aggregator.delegate().getClass().getSimpleName(), aggregator.delegate().schemaVersion()); + if (aggregator.delegate().schemaVersion() != eventSchemaVersion) { LOGGER.debug("Schema versionTo mismatch, migrating event {} {} ", event.getClass().getName(), JsonObject.mapFrom(event).encodePrettily()); - finalEvent = aggregator.delegate().transformFrom(event.schemaVersion(), JsonObject.mapFrom(event)); + finalEvent = aggregator.delegate().transformFrom(eventSchemaVersion, JsonObject.mapFrom(event)); } final var newAggregateState = (T) aggregator.delegate().apply(aggregateState, finalEvent); LOGGER.debug("State after aggregation {}", newAggregateState); @@ -113,7 +113,14 @@ private AggregatorWrap findAggregator(Event event) { return aggregators.stream() .filter(aggregator -> aggregator.eventClass().getName().equals(event.getClass().getName())) .findFirst() - .orElseThrow(() -> UnknownEvent.unknown(event.getClass())); + .orElseThrow(() -> UnknownEvent.unknown(event)); + } + + private AggregatorWrap findAggregator(String eventType) { + return aggregators.stream() + .filter(aggregator -> aggregator.delegate().eventType().equals(eventType)) + .findFirst() + .orElseThrow(); } private List applyCommandBehaviour(final T aggregateState, final Command command) { @@ -141,14 +148,14 @@ private Uni> replayAggregateAndCache(String aggregateId, Strin } if (state == null) { LOGGER.debug("Fetching from event-store {}", key); - return playFromLastJournalOffset(aggregateId, tenant, new AggregateState<>(aggregateClass)); + return playFromLastSnapshot(aggregateId, tenant, new AggregateState<>(aggregateClass)); } else { return Uni.createFrom().item(state); } } - private Uni> playFromLastJournalOffset(String aggregateId, String tenant, AggregateState state) { - final var instruction = streamInstruction(aggregateId, tenant, state); + private Uni> playFromLastSnapshot(String aggregateId, String tenant, AggregateState state) { + final var instruction = streamInstruction(aggregateId, tenant, state, true); LOGGER.debug("Playing aggregate stream with instruction {}", instruction.toJson().encodePrettily()); return infrastructure.eventStore().fetch(instruction) .map(events -> { @@ -162,44 +169,39 @@ private Uni> replayAndAggregate(LoadAggregate loadAggregate) { final var state = new AggregateState<>(aggregateClass); final var instruction = eventStreamInstruction(loadAggregate); LOGGER.debug("Playing aggregate stream with instruction {}", instruction); - return infrastructure.eventStore().stream(instruction, event -> applyEvent(state, event)) - .replaceWith(state); + return infrastructure.eventStore().stream(instruction, event -> applyEvent(state, event)).replaceWith(state); } private EventStream eventStreamInstruction(LoadAggregate loadAggregate) { - return EventStreamBuilder - .builder() + return EventStreamBuilder.builder() .aggregateIds(List.of(loadAggregate.aggregateId())) .tenantId(loadAggregate.tenant()) - .offset(0L) .to(loadAggregate.dateTo()) .versionTo(loadAggregate.versionTo()) .build(); } - private AggregateEventStream streamInstruction(String aggregateId, String tenant, AggregateState state) { - // todo use aggregate configuration to limit the size of the instruction - // the log from the last compression can only be as long as the compression policy - // when policy is applied should trim and move events to secondary event-store + private AggregateEventStream streamInstruction(String aggregateId, String tenant, AggregateState state, Boolean replayFromSnapshot) { return new AggregateEventStream<>( aggregateId, tenant, state.currentVersion(), state.currentJournalOffset(), - SnapshotEvent.class, + replayFromSnapshot, aggregateConfiguration.snapshotThreshold() ); } - private void applyEvents(final AggregateState state, final List events) { + private void aggregateEvents(final AggregateState state, final List events) { events.stream() .sorted(Comparator.comparingLong(io.es4j.infrastructure.models.Event::eventVersion)) .forEachOrdered(event -> { - final var parsedEvent = EventParser.getEvent(event.eventClass(), event.event()); + final var aggregator = findAggregator(event.eventType()); + final var parsedEvent = EventParser.getEvent(aggregator.eventClass(), event.event()); final var isSnapshot = parsedEvent.getClass().isAssignableFrom(SnapshotEvent.class); if (isSnapshot) { LOGGER.debug("Aggregating snapshot {}", event.event().encodePrettily()); - applySnapshot(state, event, parsedEvent); + applySnapshot(state, event, parsedEvent, event.schemaVersion()); } else { applyEvent(state, event, parsedEvent); } @@ -212,7 +214,7 @@ private void applyEvents(final AggregateState state, final List state, io.es4j.infrastructure.models.Event event, Event parsedEvent) { - final var newState = aggregateEvent(state.state(), parsedEvent); + final var newState = aggregateEvent(state.state(), parsedEvent, event.schemaVersion()); if (state.knownCommands().stream().noneMatch(txId -> txId.equals(event.commandId()))) { LOGGER.debug("Acknowledging command {}", event.commandId()); state.knownCommands().add(event.commandId()); @@ -220,29 +222,30 @@ private void applyEvent(AggregateState state, io.es4j.infrastructure.models.E state.setState(newState); } - private void applySnapshot(AggregateState state, io.es4j.infrastructure.models.Event event, Event parsedEvent) { + private void applySnapshot(AggregateState state, io.es4j.infrastructure.models.Event event, Event parsedEvent, Integer eventSchemaVersion) { LOGGER.debug("Applying snapshot {}", JsonObject.mapFrom(event).encodePrettily()); - if (parsedEvent.schemaVersion() != state.state().schemaVersion()) { - LOGGER.debug("Aggregate schema versionTo mismatch, migrating schema from {} to {}", parsedEvent.schemaVersion(), state.state().schemaVersion()); - state.setState(state.aggregateClass().cast(state.state().transformSnapshot(parsedEvent.schemaVersion(), event.event()))); + if (eventSchemaVersion != state.state().schemaVersion()) { + LOGGER.debug("Aggregate schema versionTo mismatch, migrating schema from {} to {}", eventSchemaVersion, state.state().schemaVersion()); + state.setState(state.aggregateClass().cast(state.state().transformSnapshot(eventSchemaVersion, event.event()))); } else { - LOGGER.debug("Applying snapshot with schema versionTo {} to aggregate with schema versionTo {}", parsedEvent.schemaVersion(), state.state().schemaVersion()); + LOGGER.debug("Applying snapshot with schema versionTo {} to aggregate with schema versionTo {}", eventSchemaVersion, state.state().schemaVersion()); state.setState(JsonObject.mapFrom(((SnapshotEvent) parsedEvent).state()).mapTo(state.aggregateClass())); } } private void applyEvent(final AggregateState state, final io.es4j.infrastructure.models.Event event) { - LOGGER.info("Aggregating event {} ", event.eventClass()); - final var parsedEvent = EventParser.getEvent(event.eventClass(), event.event()); - if (parsedEvent.getClass().isAssignableFrom(SnapshotEvent.class)) { - final var snapshot = (SnapshotEvent) parsedEvent; + LOGGER.info("Aggregating event {} ", event); + if (event.eventType().equals("snapshot")) { + final var snapshot = event.event().mapTo(SnapshotEvent.class); state.knownCommands().clear(); state.setState(JsonObject.mapFrom(snapshot.state()).mapTo(state.aggregateClass())) .addKnownCommands(snapshot.knownCommands()) .setCurrentVersion(event.eventVersion()) .setCurrentJournalOffset(event.journalOffset()); } else { - final var newState = aggregateEvent(state.state(), parsedEvent); + final var aggregator = findAggregator(event.eventType()); + final var parsedEvent = EventParser.getEvent(aggregator.eventClass(), event.event()); + final var newState = aggregateEvent(state.state(), parsedEvent, event.schemaVersion()); LOGGER.debug("State after aggregation {} ", JsonObject.mapFrom(newState).encodePrettily()); state.setState(newState) .addKnownCommand(event.commandId()) @@ -259,21 +262,21 @@ private List applyCommandBehaviour(final Ag return resultingEvents; } - public static ArrayList transformEvents(AggregateState state, Command finalCommand, Event[] array) { + public ArrayList transformEvents(AggregateState state, Command finalCommand, Event[] array) { final var currentVersion = Objects.requireNonNullElse(state.currentVersion(), 0L); return new ArrayList<>(IntStream.range(1, array.length + 1) .mapToObj(index -> { final var ev = array[index - 1]; - final var eventVersion = currentVersion + index; + final var aggregator = findAggregator(ev); return new io.es4j.infrastructure.models.Event( finalCommand.aggregateId(), - ev.getClass().getName(), - eventVersion, + aggregator.delegate().eventType(), + currentVersion + index, JsonObject.mapFrom(ev), finalCommand.tenant(), finalCommand.uniqueId(), ev.tags(), - ev.schemaVersion() + aggregator.delegate().schemaVersion() ); } ).toList() @@ -284,7 +287,6 @@ private void addOptionalSnapshot(AggregateState state, Command finalCommand, if (state.state() != null && Objects.nonNull(aggregateConfiguration.snapshotThreshold())) { final var shouldSnapshot = aggregateConfiguration.snapshotThreshold() <= Math.floorMod(state.currentVersion(), aggregateConfiguration.snapshotThreshold()); if (shouldSnapshot) { - state.setCurrentVersion(state.currentVersion() + 1); final var snapshotEvent = new SnapshotEvent( JsonObject.mapFrom(state.state()).getMap(), state.knownCommands().stream().toList(), @@ -293,13 +295,13 @@ private void addOptionalSnapshot(AggregateState state, Command finalCommand, LOGGER.debug("Appending a snapshot {}", JsonObject.mapFrom(snapshotEvent).encodePrettily()); resultingEvents.add(new io.es4j.infrastructure.models.Event( finalCommand.aggregateId(), - SnapshotEvent.class.getName(), - state.currentVersion(), + "snapshot", + resultingEvents.stream().map(io.es4j.infrastructure.models.Event::eventVersion).max(Long::compareTo).orElseThrow() + 1, JsonObject.mapFrom( snapshotEvent), finalCommand.tenant(), finalCommand.uniqueId(), - List.of("system"), + List.of("system-snapshot"), state.state().schemaVersion() )); } @@ -310,7 +312,7 @@ private void addOptionalSnapshot(AggregateState state, Command finalCommand, private Uni> processCommand(final AggregateState state, final C command) { checkCommandId(state, command); final var events = applyCommandBehaviour(state, command); - applyEvents(state, events); + aggregateEvents(state, events); return appendEvents(state, events) .map(avoid -> cacheState(state)) .invoke(avoid -> publishToEventStream(state, events)) @@ -406,13 +408,13 @@ private Uni appendEvents(AggregateState state, List state, List events) { - if (infrastructure.secondaryEventStore().isPresent() && events.stream().anyMatch(event -> event.eventClass().equals(SnapshotEvent.class.getName()))) { + if (infrastructure.secondaryEventStore().isPresent() && events.stream().anyMatch(event -> event.eventType().equals(SnapshotEvent.class.getName()))) { infrastructure.eventStore().fetch(new AggregateEventStream<>( state.state().aggregateId(), state.state().tenant(), - 0L, null, null, + false, null ) ) @@ -454,14 +456,18 @@ private static List getEventsToTrim(List eventsThatCanBeTrimmed) { return eventsThatCanBeTrimmed.stream() - .filter(ev -> ev.eventClass().equals(SnapshotEvent.class.getName())) + .filter(ev -> ev.eventType().equals(SnapshotEvent.class.getName())) .max(Comparator.comparingLong(io.es4j.infrastructure.models.Event::eventVersion)) .map(io.es4j.infrastructure.models.Event::eventVersion) .orElseThrow(() -> new IllegalStateException("Snapshot not found")); } - private void logRejectedCommand(final Throwable throwable, final Command command) { - LOGGER.error("{} command rejected {}", command.getClass().getName(), JsonObject.mapFrom(command).encodePrettily(), throwable); + private void logRejectedCommand(final Throwable throwable, final Command command, AggregateState aggregateState) { + final var json = new JsonObject(); + json.put("command-type", camelToKebab(command.getClass().getSimpleName())); + json.put("command", JsonObject.mapFrom(command)); + json.put("aggregate", aggregateState.toJson()); + LOGGER.error("Command rejected {}", json.encodePrettily(), throwable); } diff --git a/es4j-core/src/main/java/io/es4j/core/exceptions/UnknownEvent.java b/es4j-core/src/main/java/io/es4j/core/exceptions/UnknownEvent.java index 06f0c0e..5a8076d 100644 --- a/es4j-core/src/main/java/io/es4j/core/exceptions/UnknownEvent.java +++ b/es4j-core/src/main/java/io/es4j/core/exceptions/UnknownEvent.java @@ -1,6 +1,7 @@ package io.es4j.core.exceptions; +import io.es4j.Event; import io.es4j.core.objects.Es4jError; public class UnknownEvent extends Es4jException { @@ -9,8 +10,8 @@ public UnknownEvent(final Es4jError es4jError) { } - public static UnknownEvent unknown(Class eventClass) { - return new UnknownEvent(new Es4jError("Event Behaviour not found", eventClass.getSimpleName() + " has not behaviour bind", 400)); + public static UnknownEvent unknown(Event event) { + return new UnknownEvent(new Es4jError("Event Behaviour not found", event.getClass().getSimpleName() + " has not behaviour bind %s".formatted(event), 400)); } } diff --git a/es4j-core/src/main/java/io/es4j/core/objects/AggregateState.java b/es4j-core/src/main/java/io/es4j/core/objects/AggregateState.java index accae8b..9005e94 100644 --- a/es4j-core/src/main/java/io/es4j/core/objects/AggregateState.java +++ b/es4j-core/src/main/java/io/es4j/core/objects/AggregateState.java @@ -38,6 +38,7 @@ public AggregateState setCurrentVersion(Long currentVersion) { return this; } + public T state() { return state; } diff --git a/es4j-core/src/main/java/io/es4j/core/objects/EventJournalFilter.java b/es4j-core/src/main/java/io/es4j/core/objects/EventJournalFilter.java index 410b935..fce5b67 100644 --- a/es4j-core/src/main/java/io/es4j/core/objects/EventJournalFilter.java +++ b/es4j-core/src/main/java/io/es4j/core/objects/EventJournalFilter.java @@ -1,13 +1,13 @@ package io.es4j.core.objects; -import io.es4j.Event; + import io.soabase.recordbuilder.core.RecordBuilder; import java.util.List; @RecordBuilder public record EventJournalFilter( - List> events, + List eventTypes, List tags ) { } diff --git a/es4j-core/src/main/java/io/es4j/core/tasks/EventLogTrimmer.java b/es4j-core/src/main/java/io/es4j/core/tasks/EventLogTrimmer.java index 16ea539..f4d12f0 100644 --- a/es4j-core/src/main/java/io/es4j/core/tasks/EventLogTrimmer.java +++ b/es4j-core/src/main/java/io/es4j/core/tasks/EventLogTrimmer.java @@ -14,7 +14,7 @@ public class EventLogTrimmer implements CronTask { // 1) marks the id idOffset in the event-journal where it is safe to chop // - first draft will determine the safety by querying the event-log for the lowest id for any given aggregateId stream where the Snapshot.class event is not present. // - second draft will could the determine whe above for each existent aggregate - // 2) add to the infrastructure an optional interface called EventStoreDump.class classes implementing this interface would handle offloading of events + // 2) add to the infrastructure an optional interface called EventStoreDump.class classes implementing this interface would handle offloading of eventTypes // - implementations would have to be idempotent // 3) adds chopping process related methods to the EventStore.class interface // - this must be called from a single actor during its entire duration diff --git a/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java b/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java index 0927622..be1c564 100644 --- a/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java +++ b/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java @@ -63,7 +63,7 @@ private static EventStream streamStatement(PollingEventProjection pollingEventPr pollingEventProjection.filter().ifPresentOrElse( filter -> eventStream.set( EventStreamBuilder.builder() - .events(filter.events()) + .eventTypes(filter.eventTypes()) .tenantId(pollingEventProjection.tenant()) .offset(offset.idOffSet()) .batchSize(1000) @@ -89,7 +89,7 @@ private List parseEvents(List events) { event.tenantId(), event.journalOffset(), event.eventVersion(), - EventParser.getEvent(event.eventClass(), event.event()) + EventParser.getEvent(event.eventType(), event.event()) ) ) .toList(); diff --git a/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java b/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java index 7740798..7f4b620 100644 --- a/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java +++ b/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java @@ -48,7 +48,7 @@ public Uni performTask() { // polling would have to be done one aggregate at the time // polling will have to be moved to a queue // a new entry must be inserted in the queue for each one of the updated streams - stateProjectionWrapper.logger().debug("Polling events"); + stateProjectionWrapper.logger().debug("Polling eventTypes"); return offsetStore.get(new OffsetKey(stateProjectionWrapper.pollingStateProjection().getClass().getName(), "default")) .flatMap(journalOffset -> { stateProjectionWrapper.logger().debug("Journal idOffset at {}", journalOffset.idOffSet()); diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/bus/Es4jService.java b/es4j-core/src/main/java/io/es4j/infrastructure/bus/Es4jService.java index e6c2c1f..7adf7ee 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/bus/Es4jService.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/bus/Es4jService.java @@ -1,18 +1,14 @@ package io.es4j.infrastructure.bus; import com.fasterxml.jackson.databind.JsonNode; -import com.github.victools.jsonschema.generator.OptionPreset; -import com.github.victools.jsonschema.generator.SchemaGenerator; -import com.github.victools.jsonschema.generator.SchemaGeneratorConfig; -import com.github.victools.jsonschema.generator.SchemaGeneratorConfigBuilder; -import com.github.victools.jsonschema.generator.SchemaVersion; + +import com.github.victools.jsonschema.generator.*; import io.es4j.Aggregate; import io.es4j.Event; import io.es4j.core.exceptions.Es4jException; import io.es4j.core.objects.*; import io.es4j.infrastructure.EventStore; import io.es4j.infrastructure.OffsetStore; -import io.es4j.infrastructure.misc.EventParser; import io.es4j.infrastructure.models.*; import io.smallrye.mutiny.Uni; import io.smallrye.mutiny.tuples.Tuple2; @@ -35,8 +31,7 @@ public class Es4jService { private final Class aClass; protected static final Logger LOGGER = LoggerFactory.getLogger(Es4jService.class); - private final Set events; - + private final Set eventTypes; private final Map commandSchemas; @@ -44,7 +39,7 @@ public Es4jService(OffsetStore offsetStore, EventStore eventStore, Class wrap.delegate().eventType()).collect(Collectors.toSet()); } public Uni register(Vertx vertx) { @@ -114,7 +109,7 @@ public Uni register(Vertx vertx) { .flatMap(av -> vertx.eventBus().consumer(availableTypes(aClass)) .handler( message -> message.reply(JsonObject.mapFrom(new AvailableTypes( - events.stream().map(Class::getName).toList(), + eventTypes, commandSchemas ) ) @@ -183,7 +178,7 @@ public Uni> fetchEvents(EventFilter ev .tags(eventFilter.tags()) .to(eventFilter.to()) .from(eventFilter.from()) - .events(figureEventClass(eventFilter.events())) + .eventTypes(eventFilter.eventTypes()) .versionFrom(eventFilter.versionFrom()) .versionTo(eventFilter.versionTo()) .aggregateIds(eventFilter.aggregateIds()) @@ -191,19 +186,7 @@ public Uni> fetchEvents(EventFilter ev ); } - private List> figureEventClass(List classNames) { - final var arrayList = new ArrayList>(); - classNames.forEach( - className -> { - try { - arrayList.add((Class) Class.forName(className)); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } - } - ); - return arrayList; - } + public Uni> offsets(OffsetFilter offsetFilter) { return offsetStore.projections(offsetFilter); diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/cache/CaffeineAggregateCache.java b/es4j-core/src/main/java/io/es4j/infrastructure/cache/CaffeineAggregateCache.java index a3fff60..1b14504 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/cache/CaffeineAggregateCache.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/cache/CaffeineAggregateCache.java @@ -19,10 +19,6 @@ public AggregateState get(AggregateKey aggregateKey) return CaffeineWrapper.get(key(aggregateKey)); } - public static AggregatePlainKey key(AggregateKey aggregateKey) { - return new AggregatePlainKey(aggregateKey.aggregateClass().getName(), aggregateKey.aggregateId(), aggregateKey.tenantId()); - } - @Override public void put(AggregateKey aggregateKey, AggregateState aggregate) { CaffeineWrapper.put(key(aggregateKey), aggregate); @@ -33,4 +29,9 @@ public Uni setup(Class aggregateClass, AggregateConfi CaffeineWrapper.setUp(configuration); return Uni.createFrom().voidItem(); } + + + public static AggregatePlainKey key(AggregateKey aggregateKey) { + return new AggregatePlainKey(aggregateKey.aggregateClass().getName(), aggregateKey.aggregateId(), aggregateKey.tenantId()); + } } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/misc/EventParser.java b/es4j-core/src/main/java/io/es4j/infrastructure/misc/EventParser.java index fd45f16..2640f87 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/misc/EventParser.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/misc/EventParser.java @@ -20,5 +20,14 @@ public static T getEvent(final String eventClazz, JsonObject e } } + public static T getEvent(final Class eventClazz, JsonObject event) { + try { + return event.mapTo(eventClazz); + } catch (Exception e) { + LOGGER.error("Unable to parse event %s error => %s".formatted(event.encode(), e.getMessage()), e); + throw new IllegalArgumentException(e); + } + } + } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/models/AggregateEventStream.java b/es4j-core/src/main/java/io/es4j/infrastructure/models/AggregateEventStream.java index ce8fe5d..6755c3e 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/models/AggregateEventStream.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/models/AggregateEventStream.java @@ -11,7 +11,7 @@ public record AggregateEventStream( String tenantId, Long eventVersionOffset, Long journalOffset, - Class startFrom, + Boolean startFromSnapshot, Integer maxSize ) { public JsonObject toJson() { @@ -20,7 +20,7 @@ public JsonObject toJson() { .put("tenant", tenantId) .put("eventVersionOffset", eventVersionOffset) .put("journalOffset", journalOffset) - .put("startFrom", startFrom) + .put("startFromSnapshot", startFromSnapshot) .put("maxSize", maxSize); } } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/models/AvailableAggregate.java b/es4j-core/src/main/java/io/es4j/infrastructure/models/AvailableAggregate.java new file mode 100644 index 0000000..0f269e1 --- /dev/null +++ b/es4j-core/src/main/java/io/es4j/infrastructure/models/AvailableAggregate.java @@ -0,0 +1,12 @@ +package io.es4j.infrastructure.models; + +import io.vertx.core.shareddata.Shareable; + +import java.io.Serializable; +import java.util.List; + +public record AvailableAggregate( + String aggregate, + List tenants +) implements Serializable, Shareable { +} diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/models/AvailableTypes.java b/es4j-core/src/main/java/io/es4j/infrastructure/models/AvailableTypes.java index 23c35e0..9c015dc 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/models/AvailableTypes.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/models/AvailableTypes.java @@ -2,11 +2,11 @@ import com.fasterxml.jackson.databind.JsonNode; -import java.util.List; import java.util.Map; +import java.util.Set; public record AvailableTypes( - List events, + Set events, Map commandSchemas ) { } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/models/Event.java b/es4j-core/src/main/java/io/es4j/infrastructure/models/Event.java index 45485fe..0504a42 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/models/Event.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/models/Event.java @@ -12,7 +12,7 @@ public record Event( Long journalOffset, String aggregateId, - String eventClass, + String eventType, Long eventVersion, JsonObject event, String tenantId, @@ -28,7 +28,7 @@ public Event(String aggregateId, String eventClass, Long eventVersion, JsonObjec public Event { Objects.requireNonNull(aggregateId, "aggregateId must not be null"); - Objects.requireNonNull(eventClass, "Event class must not be null"); + Objects.requireNonNull(eventType, "Event class must not be null"); Objects.requireNonNull(eventVersion, "Event versionTo must not be null"); if (eventVersion < 0) { throw new IllegalArgumentException("Event versionTo must be greater than 0"); diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/models/EventFilter.java b/es4j-core/src/main/java/io/es4j/infrastructure/models/EventFilter.java index 4790ce3..b2ee795 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/models/EventFilter.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/models/EventFilter.java @@ -8,7 +8,7 @@ @RecordBuilder public record EventFilter( - List events, + List eventTypes, List aggregateIds, List tags, String tenantId, diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/models/EventStream.java b/es4j-core/src/main/java/io/es4j/infrastructure/models/EventStream.java index f74af09..dd1c7b4 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/models/EventStream.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/models/EventStream.java @@ -1,6 +1,5 @@ package io.es4j.infrastructure.models; -import io.es4j.Event; import io.soabase.recordbuilder.core.RecordBuilder; import java.time.Instant; @@ -8,7 +7,7 @@ @RecordBuilder public record EventStream( - List> events, + List eventTypes, List aggregateIds, List tags, String tenantId, diff --git a/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java b/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java index cf20e4d..37c8145 100644 --- a/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java +++ b/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java @@ -7,6 +7,7 @@ import io.es4j.core.verticles.AggregateBridge; import io.es4j.infrastructure.config.Es4jConfigurationHandler; import io.es4j.infrastructure.misc.Es4jServiceLoader; +import io.es4j.infrastructure.models.AvailableAggregate; import io.es4j.task.CronTaskDeployer; import io.es4j.task.TimerTaskDeployer; import io.reactiverse.contextual.logging.ContextualData; @@ -18,6 +19,9 @@ import io.vertx.core.DeploymentOptions; import io.vertx.core.Promise; import io.vertx.core.impl.cpu.CpuCoreSensor; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; +import io.vertx.mutiny.core.eventbus.Message; import org.crac.Context; import org.crac.Core; import org.crac.Resource; @@ -29,6 +33,8 @@ import java.util.*; import java.util.concurrent.CountDownLatch; +import static io.es4j.core.CommandHandler.camelToKebab; + public class Es4jMain extends AbstractVerticle implements Resource { protected static final Logger LOGGER = LoggerFactory.getLogger(Es4jMain.class); @@ -73,9 +79,21 @@ public void start(final Promise startPromise) { this.cronTaskDeployer = new CronTaskDeployer(vertx); this.timerTaskDeployer = new TimerTaskDeployer(vertx); addEventBusInterceptors(); + vertx.eventBus().consumer("/es4j/available-aggregates", this::availableAggregates); startAggregateResources(startPromise); } + private void availableAggregates(Message tMessage) { + tMessage.reply(new JsonArray( + AGGREGATES.stream().map( + a -> new AvailableAggregate( + camelToKebab(a.aggregateClass().getSimpleName()), + a.tenants() + ) + ).toList() + )); + } + private void addEventBusInterceptors() { vertx.eventBus().addOutboundInterceptor(this::addContextualData); vertx.eventBus().addInboundInterceptor(this::addContextualData); diff --git a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/OpenApiGenerator.java b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/OpenApiGenerator.java index 4aa76d5..4ee9a1a 100644 --- a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/OpenApiGenerator.java +++ b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/OpenApiGenerator.java @@ -127,7 +127,7 @@ private List> generateJavaInterfaceWithSwagger(Map List parseInstruction(AppendInstructi return appendInstruction.events().stream() .map(event -> new EventRecord( event.aggregateId(), - event.eventClass(), + event.eventType(), event.eventVersion(), event.event(), event.commandId(), @@ -193,19 +193,16 @@ private EventRecordQuery eventJournalQuery(AggregateEventS } private static String startingOffset(AggregateEventStream aggregateEventStream) { - if (aggregateEventStream.journalOffset() != null && aggregateEventStream.journalOffset() == 0) { - return String.valueOf(0); - } else if (aggregateEventStream.startFrom() != null) { - return "(select max(id) from event_journal where event_class = '" + aggregateEventStream.startFrom().getName() + "' and aggregateId = '" + aggregateEventStream.aggregateId() + "')"; - } else { - return null; + if (aggregateEventStream.startFromSnapshot()) { + return "(select coalesce(max(id),0) from event_store where event_class = 'snapshot' and aggregate_id ilike any(#{aggregate_id}) and tenant = #{tenant})"; } + return null; } private EventRecordQuery eventJournalQuery(EventStream eventStream) { return new EventRecordQuery( eventStream.aggregateIds(), - eventStream.events() != null ? eventStream.events().stream().map(Class::getName).toList() : null, + eventStream.eventTypes(), null, eventStream.tags(), eventStream.versionFrom(), diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java index b44f96e..fbc0f2a 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java @@ -10,7 +10,6 @@ import io.es4j.sql.LiquibaseHandler; import io.es4j.sql.Repository; import io.es4j.sql.RepositoryHandler; -import io.es4j.sql.exceptions.NotFound; import io.es4j.sql.models.BaseRecord; import io.es4j.sql.models.QueryOptions; import io.smallrye.mutiny.Uni; @@ -19,7 +18,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.function.Consumer; @@ -100,7 +98,7 @@ private List parseInstruction(AppendInstructi return appendInstruction.events().stream() .map(event -> new EventRecord( event.aggregateId(), - event.eventClass(), + event.eventType(), event.eventVersion(), event.event(), event.commandId(), @@ -137,11 +135,9 @@ private EventRecordQuery eventJournalQuery(AggregateEventS } private static String startingOffset(AggregateEventStream aggregateEventStream) { - if (aggregateEventStream.journalOffset() != null && aggregateEventStream.journalOffset() == 0) { - return String.valueOf(0); - } else if (aggregateEventStream.startFrom() != null) { - return "(select max(id) from event_journal where event_class = '" + aggregateEventStream.startFrom().getName() + "' and aggregateId = '" + aggregateEventStream.aggregateId() + "')"; - } else { + if (aggregateEventStream.startFromSnapshot()) { + return "(select coalesce(max(id),0) from event_store where event_class = 'snapshot' and aggregate_id ilike any(#{aggregate_id}) and tenant = #{tenant})"; + } else { return null; } } @@ -149,7 +145,7 @@ private static String startingOffset(AggregateEventStream< private EventRecordQuery eventJournalQuery(EventStream eventStream) { return new EventRecordQuery( eventStream.aggregateIds(), - eventStream.events() != null ? eventStream.events().stream().map(Class::getName).toList() : null, + eventStream.eventTypes(), null, eventStream.tags(), null, diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/models/EventRecord.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/models/EventRecord.java index 24e46f0..0782c52 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/models/EventRecord.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/models/EventRecord.java @@ -26,7 +26,7 @@ public EventRecord(String aggregateId, String eventClass, Long eventVersion, Jso public EventRecord { Objects.requireNonNull(aggregateId, "aggregateId must not be null"); - Objects.requireNonNull(eventClass, "eventClass must not be null"); + Objects.requireNonNull(eventClass, "eventType must not be null"); Objects.requireNonNull(eventVersion, "eventVersion must not be null"); if (eventVersion < 0) { throw new IllegalArgumentException("eventVersion must be greater than 0"); diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/resources/pg-event-store.xml b/es4j-infrastructure/es4j-postgres-storage/src/main/resources/pg-event-store.xml index 6998581..b2db10a 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/resources/pg-event-store.xml +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/resources/pg-event-store.xml @@ -35,20 +35,23 @@ - - + + - + + + + diff --git a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java index 97ab80c..f0a49c0 100644 --- a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java +++ b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java @@ -95,7 +95,7 @@ private List mapArgs(Event event, String streamName) { return Arrays.asList( streamName, String.valueOf(event.eventVersion()), TENANT_ID, event.tenantId(), - EVENT_CLASS, event.eventClass(), + EVENT_CLASS, event.eventType(), COMMAND_ID, event.commandId(), EVENT, event.event().encode(), TENANT_ID, event.tenantId(), diff --git a/es4j-test/pom.xml b/es4j-test/pom.xml index 6725fad..4aee833 100644 --- a/es4j-test/pom.xml +++ b/es4j-test/pom.xml @@ -73,6 +73,11 @@ es4j-config-storage ${project.version} + + com.kjetland + mbknor-jackson-jsonschema_2.13 + 1.0.39 + io.es4j es4j-postgres-storage diff --git a/es4j-test/src/main/java/io/es4j/Es4jExtension.java b/es4j-test/src/main/java/io/es4j/Es4jExtension.java index 2606fd8..5700420 100644 --- a/es4j-test/src/main/java/io/es4j/Es4jExtension.java +++ b/es4j-test/src/main/java/io/es4j/Es4jExtension.java @@ -139,7 +139,7 @@ private void dropAggregate(GivenAggregate givenAggregate) { state.state().aggregateId(), state.state().tenant() )); - // todo drop events ? + // todo drop eventTypes ? } private void addAggregate(GivenAggregate givenAggregate) { diff --git a/es4j-test/src/test/java/io/es4j/BridgeTest.java b/es4j-test/src/test/java/io/es4j/BridgeTest.java index 431d463..66e3dc6 100644 --- a/es4j-test/src/test/java/io/es4j/BridgeTest.java +++ b/es4j-test/src/test/java/io/es4j/BridgeTest.java @@ -41,8 +41,8 @@ void test_caching() { // eventBusPoxy.eventSubscribe(fakeAggregateAggregateState -> LOGGER.info("Incoming event {}", fakeAggregateAggregateState.toJson().encodePrettily())) // .await().indefinitely(); // final var poller = new EventProjectionPoller( -// events -> { -// LOGGER.info("events {}", events); +// eventTypes -> { +// LOGGER.info("eventTypes {}", eventTypes); // return Uni.createFrom().voidItem(); // }, // new PgEventStore(), diff --git a/es4j-test/src/test/java/io/es4j/behaviours/ChangedAggregator.java b/es4j-test/src/test/java/io/es4j/behaviours/ChangedAggregator.java index a4c4c82..751386d 100644 --- a/es4j-test/src/test/java/io/es4j/behaviours/ChangedAggregator.java +++ b/es4j-test/src/test/java/io/es4j/behaviours/ChangedAggregator.java @@ -16,7 +16,10 @@ public FakeAggregate apply(FakeAggregate aggregateState, DataChanged event) { return aggregateState.replaceData(event.newData()); } - + @Override + public String eventType() { + return "data-changed"; + } } diff --git a/es4j-test/src/test/java/io/es4j/behaviours/CreateAggregator.java b/es4j-test/src/test/java/io/es4j/behaviours/CreateAggregator.java index 85383e4..039f528 100644 --- a/es4j-test/src/test/java/io/es4j/behaviours/CreateAggregator.java +++ b/es4j-test/src/test/java/io/es4j/behaviours/CreateAggregator.java @@ -16,4 +16,9 @@ public FakeAggregate apply(FakeAggregate aggregateState, DataCreated event) { event.data() ); } + + @Override + public String eventType() { + return "data-created"; + } } diff --git a/es4j-test/src/test/java/io/es4j/hashing/ConsistentHashingTest.java b/es4j-test/src/test/java/io/es4j/hashing/ConsistentHashingTest.java index 07d4b0b..a00f71d 100644 --- a/es4j-test/src/test/java/io/es4j/hashing/ConsistentHashingTest.java +++ b/es4j-test/src/test/java/io/es4j/hashing/ConsistentHashingTest.java @@ -1,5 +1,10 @@ package io.es4j.hashing; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kjetland.jackson.jsonSchema.JsonSchemaGenerator; +import io.es4j.domain.FakeAggregate; import io.es4j.infrastructure.models.AggregatePlainKey; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,6 +47,7 @@ private ArrayList createMembers(Consistent c, int numMembers) { return members; } + @Test void simplePartitionedConsistentHashing() { // Node [FIFTH_4e454162-b6cb-45c4-9a1b-89c9964ab731] received: 35765 hits diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java b/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java index 32b2231..ecbc22f 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java @@ -48,7 +48,7 @@ static void stop() { void append_ensure_uniqueness(EventStore eventStore) { eventStore.setup(FakeAggregate.class, vertx, CONFIGURATION).await().indefinitely(); eventStore.start(FakeAggregate.class, vertx, CONFIGURATION); - // Define events and append instructions + // Define eventTypes and append instructions final var aggregateId = UUID.randomUUID().toString(); final var goodAppend = createAppendInstruction(aggregateId); final var conflictingAppend = createAppendInstruction(aggregateId); @@ -68,7 +68,7 @@ void append_ensure_uniqueness(EventStore eventStore) { void append_and_fetch(EventStore eventStore) { eventStore.setup(FakeAggregate.class, vertx, CONFIGURATION).await().indefinitely(); eventStore.start(FakeAggregate.class, vertx, CONFIGURATION); - // Define events and append instructions + // Define eventTypes and append instructions final var aggregateId = UUID.randomUUID().toString(); int numberOfEvents = 100; final var goodAppend = createAppendInstruction(aggregateId, numberOfEvents); @@ -92,7 +92,7 @@ void append_and_fetch(EventStore eventStore) { void append_and_stream(EventStore eventStore) { eventStore.setup(FakeAggregate.class, vertx, CONFIGURATION).await().indefinitely(); eventStore.start(FakeAggregate.class, vertx, CONFIGURATION); - // Define events and append instructions + // Define eventTypes and append instructions final var aggregateId = UUID.randomUUID().toString(); int numberOfEvents = 100; final var goodAppend = createAppendInstruction(aggregateId, numberOfEvents); From 5ca4681cad37c60b4f3ddfeb92482e0c9a61093c Mon Sep 17 00:00:00 2001 From: reeferman Date: Wed, 12 Jul 2023 23:07:26 +0200 Subject: [PATCH 09/17] - fixes snapshotting --- .../java/io/es4j/core/CommandHandler.java | 17 ++++++------ .../core/objects/AggregateConfiguration.java | 2 +- .../es4j/hashing/ConsistentHashingTest.java | 26 +++++++++++++++++++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/es4j-core/src/main/java/io/es4j/core/CommandHandler.java b/es4j-core/src/main/java/io/es4j/core/CommandHandler.java index 09361f8..6a3d4fb 100644 --- a/es4j-core/src/main/java/io/es4j/core/CommandHandler.java +++ b/es4j-core/src/main/java/io/es4j/core/CommandHandler.java @@ -111,7 +111,7 @@ private T aggregateEvent(T aggregateState, final Event event, Integer eventSchem private AggregatorWrap findAggregator(Event event) { return aggregators.stream() - .filter(aggregator -> aggregator.eventClass().getName().equals(event.getClass().getName())) + .filter(aggregator -> aggregator.eventClass().isAssignableFrom(event.getClass())) .findFirst() .orElseThrow(() -> UnknownEvent.unknown(event)); } @@ -196,13 +196,11 @@ private void aggregateEvents(final AggregateState state, final List { - final var aggregator = findAggregator(event.eventType()); - final var parsedEvent = EventParser.getEvent(aggregator.eventClass(), event.event()); - final var isSnapshot = parsedEvent.getClass().isAssignableFrom(SnapshotEvent.class); - if (isSnapshot) { - LOGGER.debug("Aggregating snapshot {}", event.event().encodePrettily()); - applySnapshot(state, event, parsedEvent, event.schemaVersion()); + if (event.eventType().equals("snapshot")) { + applySnapshot(state, event, event.event().mapTo(SnapshotEvent.class), event.schemaVersion()); } else { + final var aggregator = findAggregator(event.eventType()); + final var parsedEvent = EventParser.getEvent(aggregator.eventClass(), event.event()); applyEvent(state, event, parsedEvent); } state @@ -285,7 +283,7 @@ public ArrayList tran private void addOptionalSnapshot(AggregateState state, Command finalCommand, ArrayList resultingEvents) { if (state.state() != null && Objects.nonNull(aggregateConfiguration.snapshotThreshold())) { - final var shouldSnapshot = aggregateConfiguration.snapshotThreshold() <= Math.floorMod(state.currentVersion(), aggregateConfiguration.snapshotThreshold()); + final var shouldSnapshot = resultingEvents.stream().anyMatch(event -> isShouldSnapshot(aggregateConfiguration.snapshotThreshold(), event.eventVersion())); if (shouldSnapshot) { final var snapshotEvent = new SnapshotEvent( JsonObject.mapFrom(state.state()).getMap(), @@ -308,6 +306,9 @@ private void addOptionalSnapshot(AggregateState state, Command finalCommand, } } + public static boolean isShouldSnapshot(int snapshotThreshold, Long currentEvent) { + return currentEvent % snapshotThreshold == 0; + } private Uni> processCommand(final AggregateState state, final C command) { checkCommandId(state, command); diff --git a/es4j-core/src/main/java/io/es4j/core/objects/AggregateConfiguration.java b/es4j-core/src/main/java/io/es4j/core/objects/AggregateConfiguration.java index d5bfe77..85c91b5 100644 --- a/es4j-core/src/main/java/io/es4j/core/objects/AggregateConfiguration.java +++ b/es4j-core/src/main/java/io/es4j/core/objects/AggregateConfiguration.java @@ -7,7 +7,7 @@ @JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) public class AggregateConfiguration { private Duration cacheTtl = Duration.ofMinutes(40); - private Integer snapshotThreshold = 500; + private Integer snapshotThreshold = 100; private Integer idempotencyThreshold = 50; public Duration ttl() { diff --git a/es4j-test/src/test/java/io/es4j/hashing/ConsistentHashingTest.java b/es4j-test/src/test/java/io/es4j/hashing/ConsistentHashingTest.java index a00f71d..d200f5f 100644 --- a/es4j-test/src/test/java/io/es4j/hashing/ConsistentHashingTest.java +++ b/es4j-test/src/test/java/io/es4j/hashing/ConsistentHashingTest.java @@ -6,6 +6,7 @@ import com.kjetland.jackson.jsonSchema.JsonSchemaGenerator; import io.es4j.domain.FakeAggregate; import io.es4j.infrastructure.models.AggregatePlainKey; +import org.junit.jupiter.api.Assertions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.es4j.infrastructure.consistenthashing.Consistent; @@ -27,6 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; +import static io.es4j.core.CommandHandler.isShouldSnapshot; import static org.junit.jupiter.api.Assertions.*; class ConsistentHashingTest { @@ -47,6 +49,30 @@ private ArrayList createMembers(Consistent c, int numMembers) { return members; } + @Test + void test() { + Assertions.assertTrue(isShouldSnapshot(100, 400L)); + Assertions.assertTrue(isShouldSnapshot(100, 100L)); + Assertions.assertTrue(isShouldSnapshot(100, 500L)); + Assertions.assertTrue(isShouldSnapshot(100, 1000L)); + Assertions.assertTrue(isShouldSnapshot(100, 200L)); + + + Assertions.assertFalse(isShouldSnapshot(100, 99L)); + Assertions.assertFalse(isShouldSnapshot(100, 1L)); + Assertions.assertFalse(isShouldSnapshot(100, 24L)); + Assertions.assertFalse(isShouldSnapshot(100, 33L)); + Assertions.assertFalse(isShouldSnapshot(100, 102L)); + Assertions.assertFalse(isShouldSnapshot(100, 201L)); + Assertions.assertFalse(isShouldSnapshot(100, 341L)); + Assertions.assertFalse(isShouldSnapshot(100, 124134L)); + Assertions.assertFalse(isShouldSnapshot(100, 1239004L)); + + + } + + + @Test void simplePartitionedConsistentHashing() { From 24e73c24684d0343153e22143dba5115bc35f8f2 Mon Sep 17 00:00:00 2001 From: reeferman Date: Mon, 17 Jul 2023 19:26:02 +0200 Subject: [PATCH 10/17] - bunch of fixes --- README.md | 2 +- .../es4j/{Bootstrap.java => Deployment.java} | 22 ++++- .../core/objects/AggregateConfiguration.java | 37 +------ .../core/tasks/StateProjectionPoller.java | 2 +- .../core/verticles/AggregateVerticle.java | 41 +++----- .../infrastructure/cache/CaffeineWrapper.java | 2 +- .../config/Es4jConfigurationHandler.java | 26 ++++- ...sinessRule.java => FileConfiguration.java} | 2 +- .../misc/Es4jServiceLoader.java | 4 +- .../io/es4j/launcher/AggregateDeployer.java | 98 ++++++++++++------- .../java/io/es4j/launcher/Es4jLauncher.java | 3 +- .../main/java/io/es4j/launcher/Es4jMain.java | 21 +--- .../main/java/io/es4j/http/HttpBridge.java | 4 +- .../java/io/es4j/http/ProjectionRoute.java | 6 +- .../java/io/es4j/task/CronTaskDeployer.java | 1 + .../java/io/es4j/task/TimerTaskDeployer.java | 3 +- es4j-stack/pom.xml | 2 +- .../ChangeBehaviourWithConfiguration.java | 4 +- .../java/io/es4j/domain/Bootstrapper.java | 6 +- 19 files changed, 150 insertions(+), 136 deletions(-) rename es4j-core/src/main/java/io/es4j/{Bootstrap.java => Deployment.java} (64%) rename es4j-core/src/main/java/io/es4j/infrastructure/config/{FileBusinessRule.java => FileConfiguration.java} (90%) diff --git a/README.md b/README.md index 8714864..663c9cf 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ In the above example, the `DepositBehaviour` processes a `DepositCommand` for th # EventBehaviour Interface -The `EventBehaviour` interface plays a vital role in event.x, managing how events apply to aggregates. +The `EventBehaviour` interface plays a vital role in es4j, managing how events apply to aggregates. ## Type Parameters diff --git a/es4j-core/src/main/java/io/es4j/Bootstrap.java b/es4j-core/src/main/java/io/es4j/Deployment.java similarity index 64% rename from es4j-core/src/main/java/io/es4j/Bootstrap.java rename to es4j-core/src/main/java/io/es4j/Deployment.java index d32d7d4..f0137b8 100644 --- a/es4j-core/src/main/java/io/es4j/Bootstrap.java +++ b/es4j-core/src/main/java/io/es4j/Deployment.java @@ -1,14 +1,19 @@ package io.es4j; +import io.es4j.core.objects.AggregateConfiguration; + +import java.time.Duration; import java.util.Collections; import java.util.List; +import static io.es4j.core.CommandHandler.camelToKebab; + /** * The Bootstrap interface is responsible for providing the framework * with the necessary information regarding the aggregate roots that * need to be taken into consideration. */ -public interface Bootstrap { +public interface Deployment { /** * Provides the class of the aggregate root that should be taken into @@ -18,13 +23,21 @@ public interface Bootstrap { */ Class aggregateClass(); + default AggregateConfiguration aggregateConfiguration() { + return new AggregateConfiguration( + Duration.ofHours(1), + 100, + 100 + ); + } + /** * Provides file configurations that may be used for additional setup. * This method provides a default implementation that returns an empty list. * * @return A list of file configurations as strings. Default is an empty list. */ - default List fileConfigurations() { + default List fileBusinessRules() { return Collections.emptyList(); } @@ -32,4 +45,9 @@ default List tenants() { return List.of("default"); } + default String infrastructureConfiguration() { + return camelToKebab(aggregateClass().getSimpleName()); + } + + } diff --git a/es4j-core/src/main/java/io/es4j/core/objects/AggregateConfiguration.java b/es4j-core/src/main/java/io/es4j/core/objects/AggregateConfiguration.java index 85c91b5..5fa6435 100644 --- a/es4j-core/src/main/java/io/es4j/core/objects/AggregateConfiguration.java +++ b/es4j-core/src/main/java/io/es4j/core/objects/AggregateConfiguration.java @@ -1,39 +1,12 @@ package io.es4j.core.objects; -import com.fasterxml.jackson.annotation.JsonAutoDetect; import java.time.Duration; -@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) -public class AggregateConfiguration { - private Duration cacheTtl = Duration.ofMinutes(40); - private Integer snapshotThreshold = 100; - private Integer idempotencyThreshold = 50; +public record AggregateConfiguration( + Duration cacheTtl, + Integer snapshotThreshold, + Integer commandIdempotencyThreshold +) { - public Duration ttl() { - return cacheTtl; - } - - public AggregateConfiguration setCacheTtl(final Duration cacheTtl) { - this.cacheTtl = cacheTtl; - return this; - } - - public Integer snapshotThreshold() { - return snapshotThreshold; - } - - public AggregateConfiguration setSnapshotThreshold(final Integer snapshotThreshold) { - this.snapshotThreshold = snapshotThreshold; - return this; - } - - public Integer idempotencyThreshold() { - return idempotencyThreshold; - } - - public AggregateConfiguration setIdempotencyThreshold(final Integer idempotencyThreshold) { - this.idempotencyThreshold = idempotencyThreshold; - return this; - } } diff --git a/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java b/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java index 7f4b620..7740798 100644 --- a/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java +++ b/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java @@ -48,7 +48,7 @@ public Uni performTask() { // polling would have to be done one aggregate at the time // polling will have to be moved to a queue // a new entry must be inserted in the queue for each one of the updated streams - stateProjectionWrapper.logger().debug("Polling eventTypes"); + stateProjectionWrapper.logger().debug("Polling events"); return offsetStore.get(new OffsetKey(stateProjectionWrapper.pollingStateProjection().getClass().getName(), "default")) .flatMap(journalOffset -> { stateProjectionWrapper.logger().debug("Journal idOffset at {}", journalOffset.idOffSet()); diff --git a/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java b/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java index a32baee..4a1e44f 100644 --- a/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java +++ b/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java @@ -13,7 +13,6 @@ import io.es4j.infrastructure.Infrastructure; import io.es4j.infrastructure.bus.AggregateBus; import io.es4j.launcher.Es4jMain; -import io.vertx.core.Promise; import io.es4j.Command; import io.es4j.Behaviour; import io.es4j.Aggregate; @@ -21,27 +20,24 @@ import io.smallrye.mutiny.Uni; import io.smallrye.mutiny.vertx.core.AbstractVerticle; import io.vertx.mutiny.core.eventbus.Message; -import org.crac.Context; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.vertx.core.json.JsonObject; import io.vertx.mutiny.core.eventbus.DeliveryContext; -import org.crac.Resource; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.*; -import java.util.concurrent.CountDownLatch; import static io.es4j.core.CommandHandler.camelToKebab; -public class AggregateVerticle extends AbstractVerticle implements Resource { +public class AggregateVerticle extends AbstractVerticle { protected static final Logger LOGGER = LoggerFactory.getLogger(AggregateVerticle.class); public static final String ACTION = "action"; private final Class aggregateClass; - private final String deploymentID; + private final String nodeDeploymentID; private AggregateConfiguration aggregateConfiguration; private CommandHandler commandHandler; private List behaviourWraps; @@ -51,18 +47,17 @@ public class AggregateVerticle extends AbstractVerticle imp public AggregateVerticle( final Class aggregateClass, - final String deploymentID + final String nodeDeploymentID ) { this.aggregateClass = aggregateClass; - this.deploymentID = deploymentID; - org.crac.Core.getGlobalContext().register(this); + this.nodeDeploymentID = nodeDeploymentID; } @Override public Uni asyncStart() { config().put("schema", camelToKebab(aggregateClass.getSimpleName())); this.aggregateConfiguration = config().getJsonObject("aggregate-configuration", new JsonObject()).mapTo(AggregateConfiguration.class); - LOGGER.info("Event.x starting {}::{}", aggregateClass.getSimpleName(), this.deploymentID); + LOGGER.info("Es4j starting aggregate {} nodeID={} verticleID={}", aggregateClass.getSimpleName(), this.nodeDeploymentID, this.localDeploymentID); this.aggregatorWraps = loadAggregators(aggregateClass); this.behaviourWraps = loadBehaviours(aggregateClass); this.infrastructure = new Infrastructure( @@ -98,13 +93,13 @@ private Uni registerAggregateBus() { cmdBehaviour -> AggregateBus.registerCommandConsumer( vertx, aggregateClass, - deploymentID, + nodeDeploymentID, message -> messageHandler(cmdBehaviour, message), cmdBehaviour.commandClass() ) ) .collect().asList() - .flatMap(avoid -> AggregateBus.waitForRegistration(deploymentID, aggregateClass)) + .flatMap(avoid -> AggregateBus.waitForRegistration(nodeDeploymentID, aggregateClass)) .replaceWithVoid(); } @@ -249,28 +244,16 @@ public static Tuple2, Class> parse @Override public Uni asyncStop() { - LOGGER.info("Stopping {} {}", aggregateClass.getSimpleName(), deploymentID); - AggregateBus.stop(vertx, aggregateClass, deploymentID); + LOGGER.info("Stopping {} {}", aggregateClass.getSimpleName(), nodeDeploymentID); + AggregateBus.stop(vertx, aggregateClass, nodeDeploymentID); return infrastructure.stop(); } + private final String localDeploymentID = UUID.randomUUID().toString(); @Override - public void beforeCheckpoint(Context context) throws Exception { - Promise p = Promise.promise(); - stop(p); - CountDownLatch latch = new CountDownLatch(1); - p.future().onComplete(event -> latch.countDown()); - latch.await(); - } - - @Override - public void afterRestore(Context context) throws Exception { - Promise p = Promise.promise(); - start(p); - CountDownLatch latch = new CountDownLatch(1); - p.future().onComplete(event -> latch.countDown()); - latch.await(); + public String deploymentID() { + return localDeploymentID; } } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/cache/CaffeineWrapper.java b/es4j-core/src/main/java/io/es4j/infrastructure/cache/CaffeineWrapper.java index 1cc6b59..e6f2431 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/cache/CaffeineWrapper.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/cache/CaffeineWrapper.java @@ -22,7 +22,7 @@ private CaffeineWrapper() { public static synchronized void setUp(AggregateConfiguration aggregateConfiguration) { if (Objects.isNull(CAFFEINE)) { CAFFEINE = Caffeine.newBuilder() - .expireAfterAccess(aggregateConfiguration.ttl()) + .expireAfterAccess(aggregateConfiguration.cacheTtl()) .initialCapacity(500) .evictionListener((key, value, reason) -> logger.info("Aggregate evicted from cache {}", new JsonObject().put("reason", reason).put("key", key).encodePrettily())) .removalListener((key, value, reason) -> logger.info("Aggregate removed from cache {}", new JsonObject().put("reason", reason).put("key", key).encodePrettily())) diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/config/Es4jConfigurationHandler.java b/es4j-core/src/main/java/io/es4j/infrastructure/config/Es4jConfigurationHandler.java index e6704f4..e1c91ee 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/config/Es4jConfigurationHandler.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/config/Es4jConfigurationHandler.java @@ -16,6 +16,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -84,7 +85,30 @@ public static void configure(Vertx vertx, String configurationName, Consumer startConfiguration(Vertx vertx, String file, Consumer configurationConsumer) { + if (Objects.nonNull(file)) { + final var promise = Promise.promise(); + configure( + vertx, + file, + newConfiguration -> { + try { + configurationConsumer.accept(newConfiguration); + if (Objects.nonNull(promise)) { + promise.complete(); + } + } catch (Exception e) { + LOGGER.error("Unable to consume file configuration {} {}", file, newConfiguration, e); + if (Objects.nonNull(promise)) { + promise.fail(e); + } + } + } + ); + return promise.future().replaceWithVoid(); + } + return Uni.createFrom().voidItem(); + } public static Uni fsConfigurations(Vertx vertx, List files) { if (!files.isEmpty()) { final var promiseMap = files.stream().map(cfg -> Map.entry(cfg, Promise.promise())) diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/config/FileBusinessRule.java b/es4j-core/src/main/java/io/es4j/infrastructure/config/FileConfiguration.java similarity index 90% rename from es4j-core/src/main/java/io/es4j/infrastructure/config/FileBusinessRule.java rename to es4j-core/src/main/java/io/es4j/infrastructure/config/FileConfiguration.java index def9e3f..803d2ac 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/config/FileBusinessRule.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/config/FileConfiguration.java @@ -3,7 +3,7 @@ import io.vertx.core.json.JsonObject; -public class FileBusinessRule { +public class FileConfiguration { public static T get(Class tClass, String fileName) { return FileConfigurationCache.get(fileName).mapTo(tClass); } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java b/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java index d1c84da..300e91b 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java @@ -72,8 +72,8 @@ public static List loadAggregators() { .toList(); } - public static List bootstrapList() { - return ServiceLoader.load(Bootstrap.class).stream() + public static List bootstrapList() { + return ServiceLoader.load(Deployment.class).stream() .map(ServiceLoader.Provider::get) .peek(aggregate -> { LOGGER.info("Bootstrapper found {}", aggregate); diff --git a/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java b/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java index 6c8f32b..e8a6cca 100644 --- a/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java +++ b/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java @@ -3,12 +3,16 @@ import io.es4j.Aggregate; +import io.es4j.Deployment; +import io.es4j.PollingStateProjection; import io.es4j.core.tasks.AggregateHeartbeat; import io.es4j.core.verticles.AggregateVerticle; import io.es4j.infrastructure.*; import io.es4j.infrastructure.cache.CaffeineAggregateCache; import io.es4j.infrastructure.config.Es4jConfigurationHandler; import io.es4j.infrastructure.misc.Es4jServiceLoader; +import io.es4j.task.CronTaskDeployer; +import io.es4j.task.TimerTaskDeployer; import io.smallrye.mutiny.Multi; import io.smallrye.mutiny.Uni; import io.vertx.core.DeploymentOptions; @@ -24,64 +28,64 @@ import io.es4j.core.objects.StateProjectionWrapper; import io.vertx.mutiny.core.Vertx; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; +import java.util.*; import java.util.function.Supplier; import static io.es4j.core.CommandHandler.camelToKebab; import static io.es4j.infrastructure.bus.AggregateBus.startChannel; -import static io.es4j.launcher.Es4jMain.*; public class AggregateDeployer { protected static final Logger LOGGER = LoggerFactory.getLogger(AggregateDeployer.class); private final Vertx vertx; - private final String deploymentID; + private final String nodeDeploymentID; + private final Deployment deploymentConfiguration; private final Class aggregateClass; - private final List files; private Infrastructure infrastructure; private List aggregateServices; + private final Set deployed = new HashSet<>(); + private CronTaskDeployer cronTaskDeployer; + private TimerTaskDeployer timerTaskDeployer; public AggregateDeployer( - final List files, final Class aggregateClass, + final Deployment deployment, final Vertx vertx, - final String deploymentID + final String nodeDeploymentID ) { - this.files = files; - this.aggregateClass = aggregateClass; + this.deploymentConfiguration = deployment; this.vertx = vertx; - this.deploymentID = deploymentID; + this.aggregateClass = aggregateClass; + this.nodeDeploymentID = nodeDeploymentID; } public void deploy(final Promise startPromise) { Es4jConfigurationHandler.configure( vertx, - camelToKebab(aggregateClass.getSimpleName()), - newConfiguration -> { - newConfiguration.put("schema", camelToKebab(aggregateClass.getSimpleName())); - LOGGER.info("--- Event.x starting {}::{} --- {}", aggregateClass.getSimpleName(), this.deploymentID, newConfiguration.encodePrettily()); + deploymentConfiguration.infrastructureConfiguration(), + infrastructureConfiguration -> { + infrastructureConfiguration.put("schema", camelToKebab(deploymentConfiguration.aggregateClass().getSimpleName())); + LOGGER.info("--- Es4j starting {}::{} --- {}", deploymentConfiguration.infrastructureConfiguration(), this.nodeDeploymentID, infrastructureConfiguration.encodePrettily()); close() - .flatMap(avoid -> infrastructure(vertx, newConfiguration) + .flatMap(avoid -> infrastructure(vertx, infrastructureConfiguration) ) .call(injector -> { addHeartBeat(); addProjections(); - final Supplier supplier = () -> new AggregateVerticle<>(aggregateClass, deploymentID); - return startChannel(vertx, aggregateClass, deploymentID) + final Supplier supplier = () -> new AggregateVerticle<>(deploymentConfiguration.aggregateClass(), nodeDeploymentID); + return startChannel(vertx, deploymentConfiguration.aggregateClass(), nodeDeploymentID) .flatMap(avoid -> vertx.deployVerticle(supplier, new DeploymentOptions() - .setConfig(newConfiguration) + .setConfig(infrastructureConfiguration) .setInstances(CpuCoreSensor.availableProcessors() * 2) ) - .replaceWithVoid() + .map(deployed::add) ) .call(avoid -> { this.aggregateServices = Es4jServiceLoader.loadAggregateServices(); - return Es4jConfigurationHandler.fsConfigurations(vertx, files) + return Es4jConfigurationHandler.fsConfigurations(vertx, deploymentConfiguration.fileBusinessRules()) .flatMap(av -> Multi.createFrom().iterable(aggregateServices) .onItem().transformToUniAndMerge( - service -> service.start(aggregateClass, vertx, newConfiguration) + service -> service.start(deploymentConfiguration.aggregateClass(), vertx, infrastructureConfiguration) ) .collect().asList() .replaceWithVoid() @@ -94,10 +98,10 @@ public void deploy(final Promise startPromise) { .with( aVoid -> { startPromise.complete(); - LOGGER.info("--- Event.x {} started ---", aggregateClass.getSimpleName()); + LOGGER.info("--- Es4j {} started ---", deploymentConfiguration.aggregateClass().getSimpleName()); } , throwable -> { - LOGGER.error("--- Event.x {} failed to start ---", aggregateClass.getSimpleName(), throwable); + LOGGER.error("--- Es4j {} failed to start ---", deploymentConfiguration.aggregateClass().getSimpleName(), throwable); startPromise.fail(throwable); } ); @@ -106,7 +110,7 @@ public void deploy(final Promise startPromise) { } private void addHeartBeat() { - HEARTBEATS.add(new AggregateHeartbeat<>(vertx, aggregateClass)); + timerTaskDeployer.deploy(new AggregateHeartbeat<>(vertx, deploymentConfiguration.aggregateClass())); } private Uni infrastructure(Vertx vertx, JsonObject configuration) { @@ -117,19 +121,21 @@ private Uni infrastructure(Vertx vertx, JsonObject configuration) { Es4jServiceLoader.loadOffsetStore() ); - return infrastructure.setup(aggregateClass, vertx, configuration); + if (Objects.isNull(cronTaskDeployer)) { + cronTaskDeployer = new CronTaskDeployer(vertx); + } + if (Objects.isNull(timerTaskDeployer)) { + timerTaskDeployer = new TimerTaskDeployer(vertx); + } + return infrastructure.setup(deploymentConfiguration.aggregateClass(), vertx, configuration); } private void addProjections() { final var aggregateProxy = new AggregateEventBusPoxy<>(vertx, aggregateClass); final var stateProjections = Es4jServiceLoader.stateProjections().stream() - .filter(cc -> Es4jServiceLoader.getFirstGenericType(cc).isAssignableFrom(aggregateClass)) - .map(cc -> new StateProjectionWrapper( - cc, - aggregateClass, - LoggerFactory.getLogger(cc.getClass()) - )) + .filter(cc -> Es4jServiceLoader.getFirstGenericType(cc).isAssignableFrom(deploymentConfiguration.aggregateClass())) + .map(cc -> gettStateProjectionWrapper(cc, aggregateClass)) .map(tStateProjectionWrapper -> new StateProjectionPoller( aggregateClass, tStateProjectionWrapper, @@ -139,7 +145,7 @@ private void addProjections() { )) .toList(); final var eventProjections = Es4jServiceLoader.pollingEventProjections().stream() - .filter(cc -> cc.aggregateClass().isAssignableFrom(aggregateClass)) + .filter(cc -> cc.aggregateClass().isAssignableFrom(deploymentConfiguration.aggregateClass())) .map(eventProjection -> new EventProjectionPoller( eventProjection, infrastructure.eventStore(), @@ -147,12 +153,34 @@ private void addProjections() { ) ) .toList(); - EVENT_PROJECTIONS.addAll(eventProjections); - STATE_PROJECTIONS.addAll(stateProjections); + eventProjections.forEach(cronTaskDeployer::deploy); + stateProjections.forEach(cronTaskDeployer::deploy); + } + + private StateProjectionWrapper gettStateProjectionWrapper(PollingStateProjection cc, Class aggregateClass) { + return new StateProjectionWrapper( + cc, + aggregateClass, + LoggerFactory.getLogger(cc.getClass()) + ); } public Uni close() { final var closeUnis = new ArrayList>(); + if (Objects.nonNull(cronTaskDeployer)) { + cronTaskDeployer.close(); + } + if (Objects.nonNull(timerTaskDeployer)) { + timerTaskDeployer.close(); + } + if (!deployed.isEmpty()) { + closeUnis.add(Multi.createFrom().iterable(deployed) + .onItem().transformToUniAndMerge(deploymentID -> vertx.undeploy(deploymentID) + .map(avoid -> deployed.remove(deploymentID))) + .collect().asList() + .replaceWithVoid() + ); + } if (infrastructure != null) { closeUnis.add(infrastructure.stop()); } diff --git a/es4j-core/src/main/java/io/es4j/launcher/Es4jLauncher.java b/es4j-core/src/main/java/io/es4j/launcher/Es4jLauncher.java index 7f2cace..2fd38a8 100644 --- a/es4j-core/src/main/java/io/es4j/launcher/Es4jLauncher.java +++ b/es4j-core/src/main/java/io/es4j/launcher/Es4jLauncher.java @@ -26,7 +26,7 @@ public static void main(String[] args) { @Override public void beforeStartingVertx(VertxOptions vertxOptions) { - logger.info("--- Starting Event.x -----"); + logger.info("--- Starting Es4j -----"); vertxOptions .setPreferNativeTransport(true) .setMetricsOptions( @@ -44,6 +44,7 @@ public void beforeStartingVertx(VertxOptions vertxOptions) { + @Override public void afterStartingVertx(Vertx vertx) { PrometheusMeterRegistry registry = (PrometheusMeterRegistry) BackendRegistries.getDefaultNow(); diff --git a/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java b/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java index 37c8145..100f06a 100644 --- a/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java +++ b/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java @@ -1,9 +1,6 @@ package io.es4j.launcher; import io.es4j.*; -import io.es4j.core.tasks.AggregateHeartbeat; -import io.es4j.core.tasks.EventProjectionPoller; -import io.es4j.core.tasks.StateProjectionPoller; import io.es4j.core.verticles.AggregateBridge; import io.es4j.infrastructure.config.Es4jConfigurationHandler; import io.es4j.infrastructure.misc.Es4jServiceLoader; @@ -38,15 +35,12 @@ public class Es4jMain extends AbstractVerticle implements Resource { protected static final Logger LOGGER = LoggerFactory.getLogger(Es4jMain.class); - public static final List AGGREGATES = Es4jServiceLoader.bootstrapList(); + public static final List AGGREGATES = Es4jServiceLoader.bootstrapList(); private CronTaskDeployer cronTaskDeployer; private TimerTaskDeployer timerTaskDeployer; - public static final List EVENT_PROJECTIONS = new ArrayList<>(); - public static final List STATE_PROJECTIONS = new ArrayList<>(); private static final List> AGGREGATE_DEPLOYERS = new ArrayList<>(); public static final Map, List>> AGGREGATE_COMMANDS = new HashMap<>(); public static final Map, List>> AGGREGATE_EVENTS = new HashMap<>(); - public static final List> HEARTBEATS = new ArrayList<>(); public Es4jMain() { Core.getGlobalContext().register(this); @@ -114,8 +108,8 @@ private void addContextualData(DeliveryContext event) { private void startAggregateResources(final Promise startPromise) { AGGREGATES.stream() .map(aClass -> new AggregateDeployer<>( - aClass.fileConfigurations(), aClass.aggregateClass(), + aClass, vertx, context.deploymentID() ) @@ -133,8 +127,6 @@ private void startAggregateResources(final Promise startPromise) { throw new IllegalStateException("Aggregates not found"); } Uni.join().all(aggregatesDeployment).andFailFast() - .invoke(avoid -> deployHeartBeat()) - .invoke(avoid -> deployProjections()) .flatMap(avoid -> deployBridges()) .subscribe() .with( @@ -150,14 +142,7 @@ private void startAggregateResources(final Promise startPromise) { ); } - private void deployProjections() { - EVENT_PROJECTIONS.forEach(cronTaskDeployer::deploy); - STATE_PROJECTIONS.forEach(cronTaskDeployer::deploy); - } - private void deployHeartBeat() { - HEARTBEATS.forEach(timerTaskDeployer::deploy); - } private Uni deployBridges() { return vertx.deployVerticle( @@ -169,7 +154,7 @@ private Uni deployBridges() { } private void handleException(Throwable throwable) { - LOGGER.error("[-- Event.x Main had to drop the following exception --]", throwable); + LOGGER.error("[-- Es4j Main had to drop the following exception --]", throwable); } @Override diff --git a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java index b41850f..533da17 100644 --- a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java +++ b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java @@ -3,7 +3,7 @@ import com.google.auto.service.AutoService; import io.es4j.Aggregate; -import io.es4j.Bootstrap; +import io.es4j.Deployment; import io.es4j.Command; import io.es4j.core.objects.DefaultFilters; import io.es4j.core.objects.Es4jError; @@ -147,7 +147,7 @@ private void aggregateWebSocket(Router router) { final var options = new SockJSHandlerOptions().setRegisterWriteHandler(true); final var bridgeOptions = new SockJSBridgeOptions(); //todo add web-socket command ingress - Es4jMain.AGGREGATES.stream().map(Bootstrap::aggregateClass).forEach( + Es4jMain.AGGREGATES.stream().map(Deployment::aggregateClass).forEach( aClass -> bridgeOptions .addInboundPermitted(permission(AggregateBus.COMMAND_BRIDGE, aClass)) .addOutboundPermitted(permission(EventbusLiveStreams.STATE_STREAM, aClass)) diff --git a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/ProjectionRoute.java b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/ProjectionRoute.java index d4f9a43..ffdaecf 100644 --- a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/ProjectionRoute.java +++ b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/ProjectionRoute.java @@ -13,7 +13,7 @@ import io.es4j.Aggregate; -import io.es4j.Bootstrap; +import io.es4j.Deployment; import io.es4j.infrastructure.models.EventFilter; import io.es4j.infrastructure.proxy.AggregateEventBusPoxy; import io.vertx.core.json.JsonArray; @@ -29,14 +29,14 @@ public class ProjectionRoute implements HttpRoute { @Override public Uni start(Vertx vertx, JsonObject configuration) { - Es4jMain.AGGREGATES.stream().map(Bootstrap::aggregateClass) + Es4jMain.AGGREGATES.stream().map(Deployment::aggregateClass) .forEach(aggregateClass -> proxies.put(aggregateClass, new AggregateEventBusPoxy<>(vertx, aggregateClass))); return Uni.createFrom().voidItem(); } @Override public void registerRoutes(Router router) { - Es4jMain.AGGREGATES.stream().map(Bootstrap::aggregateClass).forEach( + Es4jMain.AGGREGATES.stream().map(Deployment::aggregateClass).forEach( aClass -> { router.post(Es4jService.fetchEventsAddress(aClass)) .consumes(Constants.APPLICATION_JSON) diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskDeployer.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskDeployer.java index 7811183..8499e13 100644 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskDeployer.java +++ b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskDeployer.java @@ -28,6 +28,7 @@ public CronTaskDeployer( public void close() { timers.forEach((tClass, timerId) -> vertx.cancelTimer(timerId)); + timers.clear(); } public void deploy(CronTask timerTask) { diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskDeployer.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskDeployer.java index 5f11f9f..5cb53ff 100644 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskDeployer.java +++ b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskDeployer.java @@ -28,6 +28,7 @@ public TimerTaskDeployer( public void close() { timers.forEach((tClass, timerId) -> vertx.cancelTimer(timerId)); + timers.clear(); } public void deploy(TimerTask timerTask) { @@ -63,7 +64,7 @@ public static void triggerTask(TaskWrapper taskWrapper, Vertx vertx, Duration th taskWrapper.logger().info("Interrupted, backing off for {}", taskWrapper.task().configuration().interruptionBackOff()); triggerTask(taskWrapper, vertx, taskWrapper.task().configuration().interruptionBackOff()); } else if (throwable instanceof NoStackTraceThrowable noStackTraceThrowable && noStackTraceThrowable.getMessage().contains("Timed out waiting to get lock")) { - taskWrapper.logger().info("Unable to acquire lock, will back off for {}",taskWrapper.task().configuration().lockBackOff()); + taskWrapper.logger().info("Unable to acquire lock, will back off for {}", taskWrapper.task().configuration().lockBackOff()); triggerTask(taskWrapper, vertx, taskWrapper.task().configuration().lockBackOff()); } else { taskWrapper.logger().info("Error handling task, will back off for {}", taskWrapper.task().configuration().errorBackOff(), throwable); diff --git a/es4j-stack/pom.xml b/es4j-stack/pom.xml index b929007..e05ff00 100644 --- a/es4j-stack/pom.xml +++ b/es4j-stack/pom.xml @@ -27,7 +27,7 @@ - + io.es4j es4j-core diff --git a/es4j-test/src/test/java/io/es4j/behaviours/ChangeBehaviourWithConfiguration.java b/es4j-test/src/test/java/io/es4j/behaviours/ChangeBehaviourWithConfiguration.java index d845acb..cf4d9cf 100644 --- a/es4j-test/src/test/java/io/es4j/behaviours/ChangeBehaviourWithConfiguration.java +++ b/es4j-test/src/test/java/io/es4j/behaviours/ChangeBehaviourWithConfiguration.java @@ -9,7 +9,7 @@ import io.es4j.events.DataChanged; import io.es4j.commands.ChangeDataWithConfig; import io.es4j.http.OpenApiDocs; -import io.es4j.infrastructure.config.FileBusinessRule; +import io.es4j.infrastructure.config.FileConfiguration; import java.util.List; @@ -22,7 +22,7 @@ public class ChangeBehaviourWithConfiguration implements Behaviour process(final FakeAggregate state, final ChangeDataWithConfig command) { - final var dataConfiguration = FileBusinessRule.get(DataFileBusinessRule.class, "data-configuration"); + final var dataConfiguration = FileConfiguration.get(DataFileBusinessRule.class, "data-configuration"); Objects.requireNonNull(dataConfiguration.rule(), "configuration not present"); return List.of(new DataChanged(command.newData())); } diff --git a/es4j-test/src/test/java/io/es4j/domain/Bootstrapper.java b/es4j-test/src/test/java/io/es4j/domain/Bootstrapper.java index c29298b..68188bc 100644 --- a/es4j-test/src/test/java/io/es4j/domain/Bootstrapper.java +++ b/es4j-test/src/test/java/io/es4j/domain/Bootstrapper.java @@ -2,11 +2,11 @@ import com.google.auto.service.AutoService; import io.es4j.Aggregate; -import io.es4j.Bootstrap; +import io.es4j.Deployment; -@AutoService(Bootstrap.class) -public class Bootstrapper implements Bootstrap { +@AutoService(Deployment.class) +public class Bootstrapper implements Deployment { @Override public Class aggregateClass() { return FakeAggregate.class; From 4c9ccf8cc4bbd8ce5414bb5555dfd0bfe4b692b2 Mon Sep 17 00:00:00 2001 From: reeferman Date: Tue, 18 Jul 2023 11:26:32 +0200 Subject: [PATCH 11/17] - more fixes --- .../src/main/java/io/es4j/Deployment.java | 2 +- .../core/verticles/AggregateVerticle.java | 16 +++++------ .../io/es4j/infrastructure/EventStore.java | 5 ++-- .../es4j/infrastructure/Infrastructure.java | 19 ++++++------- .../io/es4j/infrastructure/OffsetStore.java | 7 ++--- .../infrastructure/SecondaryEventStore.java | 5 ++-- .../io/es4j/launcher/AggregateDeployer.java | 4 +-- .../java/io/es4j/infra/pg/PgEventStore.java | 7 ++--- .../java/io/es4j/infra/pg/PgOffsetStore.java | 9 ++++--- .../es4j/infra/pg/PgSecondaryEventStore.java | 9 ++++--- .../io/es4j/infra/redis/RedisEventStore.java | 6 ++--- .../io/es4j/infra/redis/RedisOffsetStore.java | 5 ++-- .../main/java/io/es4j/Es4jBootstrapper.java | 27 ++++++++++++------- .../src/main/java/io/es4j/Es4jExtension.java | 2 +- es4j-test/src/main/java/io/es4j/Es4jTest.java | 2 ++ .../es4j/infrastructure/EventStoreTest.java | 15 ++++++----- .../es4j/infrastructure/OffsetStoreTest.java | 6 +++-- 17 files changed, 84 insertions(+), 62 deletions(-) diff --git a/es4j-core/src/main/java/io/es4j/Deployment.java b/es4j-core/src/main/java/io/es4j/Deployment.java index f0137b8..4a1b696 100644 --- a/es4j-core/src/main/java/io/es4j/Deployment.java +++ b/es4j-core/src/main/java/io/es4j/Deployment.java @@ -46,7 +46,7 @@ default List tenants() { } default String infrastructureConfiguration() { - return camelToKebab(aggregateClass().getSimpleName()); + return "infrastructure"; } diff --git a/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java b/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java index 4a1e44f..8d7724b 100644 --- a/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java +++ b/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java @@ -1,22 +1,19 @@ package io.es4j.core.verticles; +import io.es4j.*; +import io.es4j.Event; import io.es4j.core.objects.*; import io.es4j.infrastructure.bus.Es4jService; import io.es4j.infrastructure.misc.Es4jServiceLoader; import io.reactiverse.contextual.logging.ContextualData; import io.smallrye.mutiny.Multi; import io.smallrye.mutiny.tuples.Tuple2; -import io.es4j.Event; import io.es4j.core.CommandHandler; import io.es4j.core.exceptions.Es4jException; import io.es4j.infrastructure.Infrastructure; import io.es4j.infrastructure.bus.AggregateBus; import io.es4j.launcher.Es4jMain; -import io.es4j.Command; -import io.es4j.Behaviour; -import io.es4j.Aggregate; -import io.es4j.Aggregator; import io.smallrye.mutiny.Uni; import io.smallrye.mutiny.vertx.core.AbstractVerticle; import io.vertx.mutiny.core.eventbus.Message; @@ -38,7 +35,7 @@ public class AggregateVerticle extends AbstractVerticle { public static final String ACTION = "action"; private final Class aggregateClass; private final String nodeDeploymentID; - private AggregateConfiguration aggregateConfiguration; + private final Deployment deployment; private CommandHandler commandHandler; private List behaviourWraps; private List aggregatorWraps; @@ -46,9 +43,11 @@ public class AggregateVerticle extends AbstractVerticle { private Es4jService es4jService; public AggregateVerticle( + final Deployment deployment, final Class aggregateClass, final String nodeDeploymentID ) { + this.deployment= deployment; this.aggregateClass = aggregateClass; this.nodeDeploymentID = nodeDeploymentID; } @@ -56,7 +55,6 @@ public AggregateVerticle( @Override public Uni asyncStart() { config().put("schema", camelToKebab(aggregateClass.getSimpleName())); - this.aggregateConfiguration = config().getJsonObject("aggregate-configuration", new JsonObject()).mapTo(AggregateConfiguration.class); LOGGER.info("Es4j starting aggregate {} nodeID={} verticleID={}", aggregateClass.getSimpleName(), this.nodeDeploymentID, this.localDeploymentID); this.aggregatorWraps = loadAggregators(aggregateClass); this.behaviourWraps = loadBehaviours(aggregateClass); @@ -66,7 +64,7 @@ public Uni asyncStart() { Optional.empty(), Es4jServiceLoader.loadOffsetStore() ); - infrastructure.start(aggregateClass, vertx, config()); + infrastructure.start(deployment, vertx, config()); vertx.eventBus().addInboundInterceptor(this::addContextualData); this.commandHandler = new CommandHandler<>( vertx, @@ -74,7 +72,7 @@ public Uni asyncStart() { aggregatorWraps, behaviourWraps, infrastructure, - aggregateConfiguration + deployment.aggregateConfiguration() ); this.es4jService = new Es4jService( infrastructure.offsetStore(), diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/EventStore.java b/es4j-core/src/main/java/io/es4j/infrastructure/EventStore.java index 197000d..0457bd0 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/EventStore.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/EventStore.java @@ -1,6 +1,7 @@ package io.es4j.infrastructure; import io.es4j.Aggregate; +import io.es4j.Deployment; import io.es4j.infrastructure.models.*; import io.smallrye.mutiny.Uni; import io.vertx.core.json.JsonObject; @@ -22,8 +23,8 @@ public interface EventStore { Uni startStream(StartStream appendInstruction); Uni stop(); - void start(Class aggregateClass, Vertx vertx, JsonObject configuration); - Uni setup(Class aggregateClass, Vertx vertx, JsonObject configuration); + void start(Deployment deployment, Vertx vertx, JsonObject configuration); + Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration); Uni trim(PruneEventStream trim); } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/Infrastructure.java b/es4j-core/src/main/java/io/es4j/infrastructure/Infrastructure.java index 5d9a27f..cbfd33d 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/Infrastructure.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/Infrastructure.java @@ -1,6 +1,7 @@ package io.es4j.infrastructure; import io.es4j.Aggregate; +import io.es4j.Deployment; import io.es4j.core.objects.AggregateConfiguration; import io.smallrye.mutiny.Uni; import io.vertx.core.json.JsonObject; @@ -25,19 +26,19 @@ public Uni stop() { return Uni.join().all(list).andFailFast().replaceWithVoid(); } - public Uni setup(Class aggregateClass, Vertx vertx, JsonObject configuration) { + public Uni setup(Deployment deployment, Vertx vertx, JsonObject infrastructureConfiguration) { final var list = new ArrayList>(); - cache.ifPresent(cache -> list.add(cache.setup(aggregateClass, configuration.getJsonObject("aggregate-configuration", new JsonObject()).mapTo(AggregateConfiguration.class)))); - secondaryEventStore.ifPresent(secondaryEventStore -> list.add(secondaryEventStore.setup(aggregateClass, vertx, configuration))); - list.add(eventStore.setup(aggregateClass, vertx, configuration)); - list.add(offsetStore.setup(aggregateClass, vertx, configuration)); + cache.ifPresent(cache -> list.add(cache.setup(deployment.aggregateClass(), deployment.aggregateConfiguration()))); + secondaryEventStore.ifPresent(secondaryEventStore -> list.add(secondaryEventStore.setup(deployment, vertx, infrastructureConfiguration))); + list.add(eventStore.setup(deployment, vertx, infrastructureConfiguration)); + list.add(offsetStore.setup(deployment, vertx, infrastructureConfiguration)); return Uni.join().all(list).andFailFast().replaceWithVoid(); } - public void start(Class aggregateClass, Vertx vertx, JsonObject configuration) { - eventStore.start(aggregateClass, vertx, configuration); - offsetStore.start(aggregateClass, vertx, configuration); - secondaryEventStore.ifPresent(ses -> ses.start(aggregateClass, vertx, configuration)); + public void start(Deployment deployment, Vertx vertx, JsonObject configuration) { + eventStore.start(deployment, vertx, configuration); + offsetStore.start(deployment, vertx, configuration); + secondaryEventStore.ifPresent(ses -> ses.start(deployment, vertx, configuration)); } } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/OffsetStore.java b/es4j-core/src/main/java/io/es4j/infrastructure/OffsetStore.java index 2116f23..619b1e8 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/OffsetStore.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/OffsetStore.java @@ -1,6 +1,7 @@ package io.es4j.infrastructure; -import io.es4j.Aggregate; + +import io.es4j.Deployment; import io.es4j.core.objects.Offset; import io.es4j.infrastructure.models.OffsetFilter; import io.smallrye.mutiny.Uni; @@ -18,6 +19,6 @@ public interface OffsetStore { Uni> projections(OffsetFilter offsetFilter); Uni stop(); - void start(Class aggregateClass, Vertx vertx, JsonObject configuration); - Uni setup(Class aggregateClass, Vertx vertx, JsonObject configuration); + void start(Deployment deployment, Vertx vertx, JsonObject configuration); + Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration); } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/SecondaryEventStore.java b/es4j-core/src/main/java/io/es4j/infrastructure/SecondaryEventStore.java index c50da5a..9b23deb 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/SecondaryEventStore.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/SecondaryEventStore.java @@ -1,6 +1,7 @@ package io.es4j.infrastructure; import io.es4j.Aggregate; +import io.es4j.Deployment; import io.es4j.infrastructure.models.AggregateEventStream; import io.es4j.infrastructure.models.AppendInstruction; import io.es4j.infrastructure.models.Event; @@ -23,9 +24,9 @@ public interface SecondaryEventStore { Uni stop(); - void start(Class aggregateClass, Vertx vertx, JsonObject configuration); + void start(Deployment deployment, Vertx vertx, JsonObject configuration); - Uni setup(Class aggregateClass, Vertx vertx, JsonObject configuration); + Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration); } diff --git a/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java b/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java index e8a6cca..f80fa47 100644 --- a/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java +++ b/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java @@ -72,7 +72,7 @@ public void deploy(final Promise startPromise) { .call(injector -> { addHeartBeat(); addProjections(); - final Supplier supplier = () -> new AggregateVerticle<>(deploymentConfiguration.aggregateClass(), nodeDeploymentID); + final Supplier supplier = () -> new AggregateVerticle<>(deploymentConfiguration, aggregateClass, nodeDeploymentID); return startChannel(vertx, deploymentConfiguration.aggregateClass(), nodeDeploymentID) .flatMap(avoid -> vertx.deployVerticle(supplier, new DeploymentOptions() .setConfig(infrastructureConfiguration) @@ -127,7 +127,7 @@ private Uni infrastructure(Vertx vertx, JsonObject configuration) { if (Objects.isNull(timerTaskDeployer)) { timerTaskDeployer = new TimerTaskDeployer(vertx); } - return infrastructure.setup(deploymentConfiguration.aggregateClass(), vertx, configuration); + return infrastructure.setup(deploymentConfiguration, vertx, configuration); } diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java index 510d904..ebf7e2d 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java @@ -2,6 +2,7 @@ import com.google.auto.service.AutoService; import io.es4j.Aggregate; +import io.es4j.Deployment; import io.es4j.infra.pg.models.EventRecordKey; import io.es4j.infra.pg.models.EventRecordQuery; import io.es4j.infrastructure.models.*; @@ -36,7 +37,7 @@ public class PgEventStore implements EventStore { @Override - public void start(Class aggregateClass, Vertx vertx, JsonObject configuration) { + public void start(Deployment deployment, Vertx vertx, JsonObject configuration) { this.eventJournal = new Repository<>(EventStoreMapper.INSTANCE, RepositoryHandler.leasePool(configuration, vertx)); } @@ -135,8 +136,8 @@ public Uni stop() { @Override - public Uni setup(Class aggregateClass, Vertx vertx, JsonObject configuration) { - final var schema = camelToKebab(aggregateClass.getSimpleName()); + public Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration) { + final var schema = camelToKebab(deployment.aggregateClass().getSimpleName()); LOGGER.debug("Migrating postgres schema {} configuration {}", schema, configuration); configuration.put("schema", schema); return LiquibaseHandler.liquibaseString( diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgOffsetStore.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgOffsetStore.java index b11d3c5..450469e 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgOffsetStore.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgOffsetStore.java @@ -2,6 +2,7 @@ import com.google.auto.service.AutoService; import io.es4j.Aggregate; +import io.es4j.Deployment; import io.es4j.core.objects.OffsetBuilder; import io.es4j.core.objects.OffsetKey; import io.es4j.infra.pg.mappers.JournalOffsetMapper; @@ -44,7 +45,7 @@ public Uni stop() { } @Override - public void start(Class aggregateClass, Vertx vertx, JsonObject config) { + public void start(Deployment deployment, Vertx vertx, JsonObject config) { this.repository = new Repository<>(JournalOffsetMapper.INSTANCE, RepositoryHandler.leasePool(config, vertx)); } @@ -121,15 +122,15 @@ private static Offset getJournalOffset(EventJournalOffSet offset) { } @Override - public Uni setup(Class aggregateClass, Vertx vertx, JsonObject configuration) { - final var schema = camelToKebab(aggregateClass.getSimpleName()); + public Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration) { + final var schema = camelToKebab(deployment.aggregateClass().getSimpleName()); LOGGER.debug("Migrating postgres schema {} configuration {}", schema, configuration); configuration.put("schema", schema); return LiquibaseHandler.liquibaseString( vertx, configuration, "pg-offset-store.xml", - Map.of("schema", camelToKebab(aggregateClass.getSimpleName())) + Map.of("schema", schema) ); } diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java index fbc0f2a..43b7c97 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java @@ -1,6 +1,7 @@ package io.es4j.infra.pg; import io.es4j.Aggregate; +import io.es4j.Deployment; import io.es4j.infra.pg.mappers.EventStoreMapper; import io.es4j.infra.pg.models.EventRecord; import io.es4j.infra.pg.models.EventRecordKey; @@ -79,18 +80,18 @@ public Uni stop() { } @Override - public void start(Class aggregateClass, Vertx vertx, JsonObject configuration) { + public void start(Deployment deployment, Vertx vertx, JsonObject configuration) { this.eventJournal = new Repository<>(EventStoreMapper.INSTANCE, RepositoryHandler.leasePool(configuration, vertx)); } @Override - public Uni setup(Class aggregateClass, Vertx vertx, JsonObject configuration) { - LOGGER.debug("Migrating database for {} with configuration {}", aggregateClass.getSimpleName(), configuration); + public Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration) { + LOGGER.debug("Migrating database for {} with configuration {}", deployment.aggregateClass().getSimpleName(), configuration); return LiquibaseHandler.liquibaseString( eventJournal.repositoryHandler(), "pg-event-store.xml", - Map.of("schema", camelToKebab(aggregateClass.getSimpleName())) + Map.of("schema", camelToKebab(deployment.aggregateClass().getSimpleName())) ); } diff --git a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java index f0a49c0..0673116 100644 --- a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java +++ b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java @@ -2,6 +2,7 @@ import com.google.auto.service.AutoService; import io.es4j.Aggregate; +import io.es4j.Deployment; import io.es4j.core.objects.ErrorSource; import io.es4j.core.objects.Es4jErrorBuilder; import io.es4j.infrastructure.EventStore; @@ -162,8 +163,7 @@ public Uni stop() { } @Override - public void start(Class aggregateClass, Vertx vertx, JsonObject configuration) { - this.aggregateClass = aggregateClass; + public void start(Deployment deployment, Vertx vertx, JsonObject configuration) { this.redisClient = Redis.createClient(vertx, new RedisOptions() .setMaxPoolSize(CpuCoreSensor.availableProcessors()) @@ -183,7 +183,7 @@ public void start(Class aggregateClass, Vertx vertx, JsonOb } @Override - public Uni setup(Class aggregateClass, Vertx vertx, JsonObject configuration) { + public Uni setup(Deployment aggregateClass, Vertx vertx, JsonObject configuration) { return null; } diff --git a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisOffsetStore.java b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisOffsetStore.java index 1c87c6b..b579de6 100644 --- a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisOffsetStore.java +++ b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisOffsetStore.java @@ -2,6 +2,7 @@ import com.google.auto.service.AutoService; import io.es4j.Aggregate; +import io.es4j.Deployment; import io.es4j.core.objects.Offset; import io.es4j.core.objects.OffsetKey; import io.es4j.infrastructure.OffsetStore; @@ -41,12 +42,12 @@ public Uni stop() { } @Override - public void start(Class aggregateClass, Vertx vertx, JsonObject configuration) { + public void start(Deployment aggregateClass, Vertx vertx, JsonObject configuration) { } @Override - public Uni setup(Class aggregateClass, Vertx vertx, JsonObject configuration) { + public Uni setup(Deployment aggregateClass, Vertx vertx, JsonObject configuration) { return null; } diff --git a/es4j-test/src/main/java/io/es4j/Es4jBootstrapper.java b/es4j-test/src/main/java/io/es4j/Es4jBootstrapper.java index 966bc04..47669b0 100644 --- a/es4j-test/src/main/java/io/es4j/Es4jBootstrapper.java +++ b/es4j-test/src/main/java/io/es4j/Es4jBootstrapper.java @@ -26,6 +26,7 @@ class Es4jBootstrapper { + private final String infraConfig; public AggregateEventBusPoxy eventBusPoxy; public AggregateHttpClient httpClient; private static Network network = Network.newNetwork(); @@ -49,22 +50,30 @@ class Es4jBootstrapper { public OffsetStore offsetStore; public Es4jBootstrapper( - Class aggregateClass + Class aggregateClass, + String infraConfig ) { vertx = Vertx.vertx(); + this.infraConfig = infraConfig; this.aggregateClass = aggregateClass; } public void bootstrap() { - config = configuration(aggregateClass).put("schema", camelToKebab(aggregateClass.getSimpleName())); + final var deployment = new Deployment() { + @Override + public Class aggregateClass() { + return aggregateClass; + } + }; + config = configuration().put("schema", camelToKebab(aggregateClass.getSimpleName())); if (Boolean.TRUE.equals(infrastructure())) { + this.eventStore = Es4jServiceLoader.loadEventStore(); deployPgContainer(); vertx.deployVerticle(Es4jMain::new, new DeploymentOptions().setInstances(1).setConfig(config)).await().indefinitely(); this.cache = new CaffeineAggregateCache(); - this.eventStore = Es4jServiceLoader.loadEventStore(); - eventStore.start(aggregateClass, vertx, config); + eventStore.start(deployment, vertx, config); this.offsetStore = Es4jServiceLoader.loadOffsetStore(); - offsetStore.start(aggregateClass, vertx, config); + offsetStore.start(deployment, vertx, config); } this.httpClient = new AggregateHttpClient<>( WebClient.create(vertx, new WebClientOptions() @@ -97,8 +106,8 @@ public Boolean infrastructure() { } - public JsonObject configuration(Class aggregateClass) { - return vertx.fileSystem().readFileBlocking(camelToKebab(aggregateClass.getSimpleName()) + ".json").toJsonObject(); + public JsonObject configuration() { + return vertx.fileSystem().readFileBlocking(infraConfig + ".json").toJsonObject(); } private void deployRedisContainer() { @@ -109,7 +118,7 @@ private void deployRedisContainer() { redis.start(); config.put("redisHost", redis.getHost()); config.put("redisPort", redis.getFirstMappedPort()); - vertx.fileSystem().writeFileBlocking(camelToKebab(aggregateClass.getSimpleName()) + ".json", Buffer.newInstance(config.toBuffer())); + vertx.fileSystem().writeFileBlocking(infraConfig + ".json", Buffer.newInstance(config.toBuffer())); LOGGER.debug("Configuration after container bootstrap {}", config); } @@ -124,7 +133,7 @@ public void deployPgContainer() { .put(Constants.PG_PASSWORD, postgreSQLContainer.getPassword()) .put(Constants.PG_DATABASE, postgreSQLContainer.getDatabaseName()) .put(Constants.JDBC_URL, postgreSQLContainer.getJdbcUrl()); - vertx.fileSystem().writeFileBlocking(camelToKebab(aggregateClass.getSimpleName()) + ".json", Buffer.newInstance(config.toBuffer())); + vertx.fileSystem().writeFileBlocking(infraConfig + ".json", Buffer.newInstance(config.toBuffer())); LOGGER.debug("Configuration after container bootstrap {}", config); } diff --git a/es4j-test/src/main/java/io/es4j/Es4jExtension.java b/es4j-test/src/main/java/io/es4j/Es4jExtension.java index 5700420..75876d8 100644 --- a/es4j-test/src/main/java/io/es4j/Es4jExtension.java +++ b/es4j-test/src/main/java/io/es4j/Es4jExtension.java @@ -34,7 +34,7 @@ public void beforeAll(ExtensionContext extensionContext) { extensionContext.getTestClass().ifPresent( testClass -> { Es4jTest annotation = testClass.getAnnotation(Es4jTest.class); - es4jBootstrapper = new Es4jBootstrapper<>(annotation.aggregate()) + es4jBootstrapper = new Es4jBootstrapper<>(annotation.aggregate(), annotation.infraConfig()) .setPostgres(annotation.infrastructure()) .setRemoteHost(annotation.host()) .setRemotePort(annotation.port()); diff --git a/es4j-test/src/main/java/io/es4j/Es4jTest.java b/es4j-test/src/main/java/io/es4j/Es4jTest.java index fe61cf8..a8c1262 100644 --- a/es4j-test/src/main/java/io/es4j/Es4jTest.java +++ b/es4j-test/src/main/java/io/es4j/Es4jTest.java @@ -21,6 +21,8 @@ boolean secondaryEventStore() default false; + String infraConfig() default "infrastructure"; + String host() default "localhost"; int port() default 8080; diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java b/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java index ecbc22f..567cc09 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java @@ -1,5 +1,7 @@ package io.es4j.infrastructure; + +import io.es4j.Deployment; import io.es4j.domain.FakeAggregate; import io.es4j.infra.pg.PgEventStore; import io.es4j.infrastructure.models.AggregateEventStreamBuilder; @@ -27,6 +29,7 @@ class EventStoreTest { public static final String TENANT_ID = "default"; + public static final Deployment DEPLOYMENT = () -> FakeAggregate.class; private static PostgreSQLContainer POSTGRES_CONTAINER; private static final Vertx vertx = Vertx.vertx(); private static final JsonObject CONFIGURATION = new JsonObject(); @@ -46,8 +49,8 @@ static void stop() { @ParameterizedTest @MethodSource("eventStores") void append_ensure_uniqueness(EventStore eventStore) { - eventStore.setup(FakeAggregate.class, vertx, CONFIGURATION).await().indefinitely(); - eventStore.start(FakeAggregate.class, vertx, CONFIGURATION); + eventStore.setup(DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); + eventStore.start(DEPLOYMENT, vertx, CONFIGURATION); // Define eventTypes and append instructions final var aggregateId = UUID.randomUUID().toString(); final var goodAppend = createAppendInstruction(aggregateId); @@ -66,8 +69,8 @@ void append_ensure_uniqueness(EventStore eventStore) { @ParameterizedTest @MethodSource("eventStores") void append_and_fetch(EventStore eventStore) { - eventStore.setup(FakeAggregate.class, vertx, CONFIGURATION).await().indefinitely(); - eventStore.start(FakeAggregate.class, vertx, CONFIGURATION); + eventStore.setup(DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); + eventStore.start(DEPLOYMENT, vertx, CONFIGURATION); // Define eventTypes and append instructions final var aggregateId = UUID.randomUUID().toString(); int numberOfEvents = 100; @@ -90,8 +93,8 @@ void append_and_fetch(EventStore eventStore) { @ParameterizedTest @MethodSource("eventStores") void append_and_stream(EventStore eventStore) { - eventStore.setup(FakeAggregate.class, vertx, CONFIGURATION).await().indefinitely(); - eventStore.start(FakeAggregate.class, vertx, CONFIGURATION); + eventStore.setup(DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); + eventStore.start(DEPLOYMENT, vertx, CONFIGURATION); // Define eventTypes and append instructions final var aggregateId = UUID.randomUUID().toString(); int numberOfEvents = 100; diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/OffsetStoreTest.java b/es4j-test/src/test/java/io/es4j/infrastructure/OffsetStoreTest.java index a9238d3..762933a 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/OffsetStoreTest.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/OffsetStoreTest.java @@ -1,5 +1,6 @@ package io.es4j.infrastructure; +import io.es4j.Deployment; import io.es4j.core.objects.Offset; import io.es4j.core.objects.OffsetBuilder; import io.es4j.core.objects.OffsetKeyBuilder; @@ -23,6 +24,7 @@ class OffsetStoreTest { public static final String TENANT_ID = "default"; private static PostgreSQLContainer POSTGRES_CONTAINER; + private static final Deployment DEPLOYMENT = () -> FakeAggregate.class; private static final Vertx vertx = Vertx.vertx(); private static final JsonObject CONFIGURATION = new JsonObject(); @@ -41,8 +43,8 @@ static void stop() { @ParameterizedTest @MethodSource("offsetStores") void test_offset_store(OffsetStore offsetStore) { - offsetStore.setup(FakeAggregate.class, vertx, CONFIGURATION).await().indefinitely(); - offsetStore.start(FakeAggregate.class, vertx, CONFIGURATION); + offsetStore.setup(DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); + offsetStore.start(DEPLOYMENT, vertx, CONFIGURATION); final var name = "fake-consumer"; final var offset = Assertions.assertDoesNotThrow( () -> offsetStore.put(createOffset(name)).await().indefinitely() From 92ad1ec9f66f154e3906f5761b46eeea64d1790c Mon Sep 17 00:00:00 2001 From: reeferman Date: Tue, 18 Jul 2023 16:04:37 +0200 Subject: [PATCH 12/17] - more fixes --- .../es4j/core/verticles/AggregateBridge.java | 7 +++++ .../core/verticles/AggregateVerticle.java | 3 +-- .../core/verticles/TaskProcessorVerticle.java | 14 +++++----- .../io/es4j/launcher/AggregateDeployer.java | 10 ++++--- .../main/java/io/es4j/launcher/Es4jMain.java | 19 +++++++------- .../pom.xml | 2 +- .../messagebroker}/MessageProcessor.java | 4 +-- .../messagebroker}/MessageProducer.java | 9 ++++--- .../ProcessorTransactionProvider.java | 13 ++++++++++ .../messagebroker/QueueSubscriber.java | 15 +++++++++++ .../messagebroker/TopicSubscriber.java | 15 +++++++++++ .../exceptions/ConsumerException.java | 2 +- .../exceptions/MessageException.java | 2 +- .../messagebroker}/exceptions/QueueError.java | 2 +- .../exceptions/QueueException.java | 2 +- .../messagebroker}/models/Message.java | 2 +- .../messagebroker}/models/MessageID.java | 2 +- .../models/MessageProcessorManager.java | 12 ++++----- .../models/MessageProcessorWrapper.java | 4 +-- .../messagebroker}/models/MessageState.java | 0 .../models/QueueConfiguration.java | 2 +- .../models/QueueImplementation.java | 2 +- .../models/QueueTransaction.java | 2 +- .../messagebroker}/models/RawMessage.java | 2 +- .../models/TransactionProvider.java | 2 +- .../postgres/PgMessageProducer.java | 20 +++++++------- .../postgres/PgProcessorTransaction.java} | 20 +++++++------- .../postgres/PgQueueSubscriber.java} | 26 +++++++++---------- .../messagebroker}/postgres/PgRefresher.java | 4 +-- .../postgres/mappers/DeadLetterMapper.java | 10 +++---- .../postgres/mappers/MessageQueueMapper.java | 10 +++---- .../mappers/MessageTransactionMapper.java | 8 +++--- .../postgres/mappers/PgQueueLiquibase.java | 12 ++++----- .../postgres/models/DeadLetterKey.java | 2 +- .../postgres/models/DeadLetterRecord.java | 0 .../postgres/models/MessageRecord.java | 6 ++--- .../postgres/models/MessageRecordID.java | 2 +- .../postgres/models/MessageRecordQuery.java | 4 +-- .../postgres/models/MessageTransaction.java | 2 +- .../postgres/models/MessageTransactionID.java | 2 +- .../models/MessageTransactionQuery.java | 2 +- .../src/main/resources/message-broker.xml} | 0 .../es4j/queue/QueueTransactionManager.java | 13 ---------- .../java/io/es4j/queue/TaskSubscriber.java | 15 ----------- .../io/es4j/infra/redis/RedisEventStore.java | 5 +++- es4j-infrastructure/es4j-task/pom.xml | 2 +- es4j-infrastructure/pom.xml | 2 +- .../taskqueue/MockProcessor.java | 2 +- .../taskqueue/PgTaskQueueTest.java | 18 ++++++------- 49 files changed, 183 insertions(+), 153 deletions(-) rename es4j-infrastructure/{es4j-queue => es4j-postgres-message-broker}/pom.xml (95%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/MessageProcessor.java (86%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/MessageProducer.java (55%) create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/ProcessorTransactionProvider.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/QueueSubscriber.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/TopicSubscriber.java rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/exceptions/ConsumerException.java (91%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/exceptions/MessageException.java (71%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/exceptions/QueueError.java (60%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/exceptions/QueueException.java (94%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/models/Message.java (90%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/models/MessageID.java (71%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/models/MessageProcessorManager.java (91%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/models/MessageProcessorWrapper.java (85%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/models/MessageState.java (100%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/models/QueueConfiguration.java (98%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/models/QueueImplementation.java (55%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/models/QueueTransaction.java (72%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/models/RawMessage.java (92%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/models/TransactionProvider.java (51%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/PgMessageProducer.java (84%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue/postgres/PgQueueTransaction.java => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgProcessorTransaction.java} (58%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue/postgres/PgTaskSubscriber.java => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgQueueSubscriber.java} (91%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/PgRefresher.java (97%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/mappers/DeadLetterMapper.java (92%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/mappers/MessageQueueMapper.java (92%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/mappers/MessageTransactionMapper.java (86%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/mappers/PgQueueLiquibase.java (80%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/models/DeadLetterKey.java (77%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/models/DeadLetterRecord.java (100%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/models/MessageRecord.java (94%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/models/MessageRecordID.java (77%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/models/MessageRecordQuery.java (80%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/models/MessageTransaction.java (88%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/models/MessageTransactionID.java (78%) rename es4j-infrastructure/{es4j-queue/src/main/java/io/es4j/queue => es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker}/postgres/models/MessageTransactionQuery.java (82%) rename es4j-infrastructure/{es4j-queue/src/main/resources/task-queue.xml => es4j-postgres-message-broker/src/main/resources/message-broker.xml} (100%) delete mode 100644 es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/QueueTransactionManager.java delete mode 100644 es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/TaskSubscriber.java diff --git a/es4j-core/src/main/java/io/es4j/core/verticles/AggregateBridge.java b/es4j-core/src/main/java/io/es4j/core/verticles/AggregateBridge.java index f75fda6..a2e4778 100644 --- a/es4j-core/src/main/java/io/es4j/core/verticles/AggregateBridge.java +++ b/es4j-core/src/main/java/io/es4j/core/verticles/AggregateBridge.java @@ -15,6 +15,7 @@ import io.es4j.infrastructure.Bridge; import java.util.List; +import java.util.UUID; import java.util.concurrent.CountDownLatch; public class AggregateBridge extends AbstractVerticle implements Resource { @@ -65,4 +66,10 @@ public Uni asyncStop() { .replaceWithVoid(); } + private final String DEPLOYMENT_ID = UUID.randomUUID().toString(); + + @Override + public String deploymentID() { + return DEPLOYMENT_ID; + } } diff --git a/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java b/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java index 8d7724b..6ee3e34 100644 --- a/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java +++ b/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java @@ -102,7 +102,6 @@ private Uni registerAggregateBus() { } private void messageHandler(BehaviourWrap cmdBehaviour, Message message) { - // todo add command name to contextual data commandHandler.process(parseCommand(cmdBehaviour.commandClass(), message)) .subscribe() .with( @@ -120,7 +119,7 @@ private void messageHandler(BehaviourWr private static Command parseCommand(Class cmdClass, Message message) { try { - LOGGER.debug("Incoming command {} {}", cmdClass.getName(), message.body().encodePrettily()); + LOGGER.debug("Parsing command {} {}", cmdClass.getName(), message.body().encodePrettily()); return message.body().mapTo(cmdClass); } catch (Exception e) { message.fail(400, JsonObject.mapFrom( diff --git a/es4j-core/src/main/java/io/es4j/core/verticles/TaskProcessorVerticle.java b/es4j-core/src/main/java/io/es4j/core/verticles/TaskProcessorVerticle.java index dd454ef..781374b 100644 --- a/es4j-core/src/main/java/io/es4j/core/verticles/TaskProcessorVerticle.java +++ b/es4j-core/src/main/java/io/es4j/core/verticles/TaskProcessorVerticle.java @@ -14,13 +14,13 @@ import io.es4j.queue.MessageProcessor; import io.es4j.queue.TaskSubscriber; import io.es4j.queue.QueueTransactionManager; -import io.es4j.queue.exceptions.QueueError; -import io.es4j.queue.exceptions.MessageException; -import io.es4j.queue.models.MessageProcessorManager; -import io.es4j.queue.models.MessageProcessorWrapper; -import io.es4j.queue.models.QueueConfiguration; -import io.es4j.queue.postgres.PgTaskSubscriber; -import io.es4j.queue.postgres.PgQueueTransaction; +import io.es4j.infrastructure.messagebroker.exceptions.QueueError; +import io.es4j.infrastructure.messagebroker.exceptions.MessageException; +import io.es4j.infrastructure.messagebroker.models.MessageProcessorManager; +import io.es4j.infrastructure.messagebroker.models.MessageProcessorWrapper; +import io.es4j.infrastructure.messagebroker.models.QueueConfiguration; +import io.es4j.infrastructure.messagebroker.postgres.PgTaskSubscriber; +import io.es4j.infrastructure.messagebroker.postgres.PgQueueTransaction; import io.vertx.mutiny.core.Vertx; import java.util.*; diff --git a/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java b/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java index f80fa47..e3b004a 100644 --- a/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java +++ b/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java @@ -127,7 +127,8 @@ private Uni infrastructure(Vertx vertx, JsonObject configuration) { if (Objects.isNull(timerTaskDeployer)) { timerTaskDeployer = new TimerTaskDeployer(vertx); } - return infrastructure.setup(deploymentConfiguration, vertx, configuration); + return infrastructure.setup(deploymentConfiguration, vertx, configuration) + .invoke(avoid -> infrastructure.start(deploymentConfiguration, vertx, configuration)); } @@ -173,11 +174,14 @@ public Uni close() { if (Objects.nonNull(timerTaskDeployer)) { timerTaskDeployer.close(); } + if (!aggregateServices.isEmpty()) { + closeUnis.addAll(aggregateServices.stream().map(AggregateServices::stop).toList()); + } if (!deployed.isEmpty()) { closeUnis.add(Multi.createFrom().iterable(deployed) - .onItem().transformToUniAndMerge(deploymentID -> vertx.undeploy(deploymentID) - .map(avoid -> deployed.remove(deploymentID))) + .onItem().transformToUniAndMerge(vertx::undeploy) .collect().asList() + .invoke(avoid -> deployed.clear()) .replaceWithVoid() ); } diff --git a/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java b/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java index 100f06a..d30b74b 100644 --- a/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java +++ b/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java @@ -5,8 +5,6 @@ import io.es4j.infrastructure.config.Es4jConfigurationHandler; import io.es4j.infrastructure.misc.Es4jServiceLoader; import io.es4j.infrastructure.models.AvailableAggregate; -import io.es4j.task.CronTaskDeployer; -import io.es4j.task.TimerTaskDeployer; import io.reactiverse.contextual.logging.ContextualData; import io.smallrye.mutiny.Multi; import io.smallrye.mutiny.Uni; @@ -36,11 +34,10 @@ public class Es4jMain extends AbstractVerticle implements Resource { protected static final Logger LOGGER = LoggerFactory.getLogger(Es4jMain.class); public static final List AGGREGATES = Es4jServiceLoader.bootstrapList(); - private CronTaskDeployer cronTaskDeployer; - private TimerTaskDeployer timerTaskDeployer; private static final List> AGGREGATE_DEPLOYERS = new ArrayList<>(); public static final Map, List>> AGGREGATE_COMMANDS = new HashMap<>(); public static final Map, List>> AGGREGATE_EVENTS = new HashMap<>(); + public static final Set bridges = new HashSet<>(); public Es4jMain() { Core.getGlobalContext().register(this); @@ -68,10 +65,8 @@ public void afterRestore(Context context) throws Exception { @Override public void start(final Promise startPromise) { LOGGER.info(" ---- Starting {}::{} ---- ", this.getClass().getName(), context.deploymentID()); - Infrastructure.setDroppedExceptionHandler(throwable -> LOGGER.error("[-- [Event.x] had to drop the following exception --]", throwable)); + Infrastructure.setDroppedExceptionHandler(throwable -> LOGGER.error("[-- [Es4j] had to drop the following exception --]", throwable)); vertx.exceptionHandler(this::handleException); - this.cronTaskDeployer = new CronTaskDeployer(vertx); - this.timerTaskDeployer = new TimerTaskDeployer(vertx); addEventBusInterceptors(); vertx.eventBus().consumer("/es4j/available-aggregates", this::availableAggregates); startAggregateResources(startPromise); @@ -150,6 +145,7 @@ private Uni deployBridges() { new DeploymentOptions() .setInstances(CpuCoreSensor.availableProcessors() * 2) ) + .map(bridges::add) .replaceWithVoid(); } @@ -168,8 +164,13 @@ public void stop(final Promise stopPromise) { private Uni close() { Es4jConfigurationHandler.close(); - timerTaskDeployer.close(); - cronTaskDeployer.close(); + final var unis = new ArrayList>(); + if (!bridges.isEmpty()) { + unis.add(Multi.createFrom().iterable(bridges) + .onItem().transformToUniAndMerge(vertx::undeploy) + .collect().asList() + .replaceWithVoid()); + } return Multi.createFrom().iterable(AGGREGATE_DEPLOYERS) .onItem().transformToUniAndMerge(AggregateDeployer::close) .collect().asList() diff --git a/es4j-infrastructure/es4j-queue/pom.xml b/es4j-infrastructure/es4j-postgres-message-broker/pom.xml similarity index 95% rename from es4j-infrastructure/es4j-queue/pom.xml rename to es4j-infrastructure/es4j-postgres-message-broker/pom.xml index 07ace40..ac9d992 100644 --- a/es4j-infrastructure/es4j-queue/pom.xml +++ b/es4j-infrastructure/es4j-postgres-message-broker/pom.xml @@ -9,7 +9,7 @@ 4.0.0 - es4j-queue + es4j-postgres-message-broker diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/MessageProcessor.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProcessor.java similarity index 86% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/MessageProcessor.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProcessor.java index dc05b45..23e6111 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/MessageProcessor.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProcessor.java @@ -1,6 +1,6 @@ -package io.es4j.queue; +package io.es4j.infrastructure.messagebroker; -import io.es4j.queue.models.QueueTransaction; +import io.es4j.infrastructure.messagebroker.models.QueueTransaction; import io.smallrye.mutiny.Uni; import java.util.List; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/MessageProducer.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProducer.java similarity index 55% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/MessageProducer.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProducer.java index 62b94b3..8256394 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/MessageProducer.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProducer.java @@ -1,8 +1,9 @@ -package io.es4j.queue; +package io.es4j.infrastructure.messagebroker; -import io.es4j.queue.models.Message; -import io.es4j.queue.models.MessageID; -import io.es4j.queue.models.QueueTransaction; + +import io.es4j.infrastructure.messagebroker.models.Message; +import io.es4j.infrastructure.messagebroker.models.MessageID; +import io.es4j.infrastructure.messagebroker.models.QueueTransaction; import io.smallrye.mutiny.Uni; import java.util.List; diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/ProcessorTransactionProvider.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/ProcessorTransactionProvider.java new file mode 100644 index 0000000..a55602c --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/ProcessorTransactionProvider.java @@ -0,0 +1,13 @@ +package io.es4j.infrastructure.messagebroker; + + +import io.es4j.infrastructure.messagebroker.models.Message; +import io.es4j.infrastructure.messagebroker.models.QueueTransaction; +import io.smallrye.mutiny.Uni; + +import java.util.function.BiFunction; + +public interface ProcessorTransactionProvider { + + Uni transaction(Message message, BiFunction, QueueTransaction, Uni> function); +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/QueueSubscriber.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/QueueSubscriber.java new file mode 100644 index 0000000..0456e20 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/QueueSubscriber.java @@ -0,0 +1,15 @@ +package io.es4j.infrastructure.messagebroker; + + +import io.es4j.infrastructure.messagebroker.models.MessageProcessorManager; +import io.smallrye.mutiny.Uni; + +public interface QueueSubscriber { + + Uni unsubscribe(); + + Uni subscribe(MessageProcessorManager messageProcessorManager); + + + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/TopicSubscriber.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/TopicSubscriber.java new file mode 100644 index 0000000..5805211 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/TopicSubscriber.java @@ -0,0 +1,15 @@ +package io.es4j.infrastructure.messagebroker; + + +import io.es4j.infrastructure.messagebroker.models.MessageProcessorManager; +import io.smallrye.mutiny.Uni; + +public interface TopicSubscriber { + + Uni unsubscribe(); + + Uni subscribe(MessageProcessorManager messageProcessorManager); + + + +} diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/ConsumerException.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/ConsumerException.java similarity index 91% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/ConsumerException.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/ConsumerException.java index ab0cafb..db5c706 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/ConsumerException.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/ConsumerException.java @@ -1,4 +1,4 @@ -package io.es4j.queue.exceptions; +package io.es4j.infrastructure.messagebroker.exceptions; public class ConsumerException extends QueueException { diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/MessageException.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/MessageException.java similarity index 71% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/MessageException.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/MessageException.java index 685c83d..4cab098 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/MessageException.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/MessageException.java @@ -1,4 +1,4 @@ -package io.es4j.queue.exceptions; +package io.es4j.infrastructure.messagebroker.exceptions; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/QueueError.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueError.java similarity index 60% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/QueueError.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueError.java index 56c76ba..a3192f0 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/QueueError.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueError.java @@ -1,4 +1,4 @@ -package io.es4j.queue.exceptions; +package io.es4j.infrastructure.messagebroker.exceptions; public record QueueError( String cause, diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/QueueException.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueException.java similarity index 94% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/QueueException.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueException.java index 9c00ee9..f4b4fd1 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/exceptions/QueueException.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueException.java @@ -1,4 +1,4 @@ -package io.es4j.queue.exceptions; +package io.es4j.infrastructure.messagebroker.exceptions; import io.vertx.core.json.JsonObject; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/Message.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/Message.java similarity index 90% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/Message.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/Message.java index 96cd6da..540ee84 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/Message.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/Message.java @@ -1,4 +1,4 @@ -package io.es4j.queue.models; +package io.es4j.infrastructure.messagebroker.models; import io.soabase.recordbuilder.core.RecordBuilder; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/MessageID.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageID.java similarity index 71% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/MessageID.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageID.java index 584abdd..d00427c 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/MessageID.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageID.java @@ -1,4 +1,4 @@ -package io.es4j.queue.models; +package io.es4j.infrastructure.messagebroker.models; import io.soabase.recordbuilder.core.RecordBuilder; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/MessageProcessorManager.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorManager.java similarity index 91% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/MessageProcessorManager.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorManager.java index b6bdb7b..cb33a87 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/MessageProcessorManager.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorManager.java @@ -1,14 +1,14 @@ -package io.es4j.queue.models; +package io.es4j.infrastructure.messagebroker.models; -import io.es4j.queue.exceptions.ConsumerException; +import io.es4j.infrastructure.messagebroker.exceptions.ConsumerException; import io.es4j.sql.exceptions.IntegrityContraintViolation; import io.smallrye.mutiny.Uni; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import io.vertx.core.json.JsonObject; import io.vertx.mutiny.core.Vertx; -import io.es4j.queue.MessageProcessor; -import io.es4j.queue.QueueTransactionManager; +import io.es4j.infrastructure.messagebroker.MessageProcessor; +import io.es4j.infrastructure.messagebroker.ProcessorTransactionProvider; import java.util.List; @@ -16,14 +16,14 @@ public record MessageProcessorManager( QueueConfiguration queueConfiguration, List processorWrappers, - QueueTransactionManager queueTransactionManager, + ProcessorTransactionProvider processorTransactionProvider, Vertx vertx ) { private static final Logger LOGGER = LoggerFactory.getLogger(MessageProcessorManager.class); public Uni processMessage(RawMessage rawMessage) { final var processor = resolveProcessor(rawMessage); final var parsedMessage = parseMessage(rawMessage); - return queueTransactionManager.transaction( + return processorTransactionProvider.transaction( parsedMessage, (msg, taskTransaction) -> { try { if (processor.blockingProcessor()) { diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/MessageProcessorWrapper.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorWrapper.java similarity index 85% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/MessageProcessorWrapper.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorWrapper.java index d7b7afc..e6e43b8 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/MessageProcessorWrapper.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorWrapper.java @@ -1,6 +1,6 @@ -package io.es4j.queue.models; +package io.es4j.infrastructure.messagebroker.models; -import io.es4j.queue.MessageProcessor; +import io.es4j.infrastructure.messagebroker.MessageProcessor; import java.util.List; import java.util.Map; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/MessageState.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageState.java similarity index 100% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/MessageState.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageState.java diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/QueueConfiguration.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueConfiguration.java similarity index 98% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/QueueConfiguration.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueConfiguration.java index 2a0a920..1df349f 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/QueueConfiguration.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueConfiguration.java @@ -1,4 +1,4 @@ -package io.es4j.queue.models; +package io.es4j.infrastructure.messagebroker.models; import com.fasterxml.jackson.annotation.JsonAutoDetect; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/QueueImplementation.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueImplementation.java similarity index 55% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/QueueImplementation.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueImplementation.java index 03a6b31..8972b82 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/QueueImplementation.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueImplementation.java @@ -1,4 +1,4 @@ -package io.es4j.queue.models; +package io.es4j.infrastructure.messagebroker.models; public enum QueueImplementation { diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/QueueTransaction.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueTransaction.java similarity index 72% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/QueueTransaction.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueTransaction.java index daf8ccb..1cd96e8 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/QueueTransaction.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueTransaction.java @@ -1,4 +1,4 @@ -package io.es4j.queue.models; +package io.es4j.infrastructure.messagebroker.models; public record QueueTransaction( Object connection diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/RawMessage.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/RawMessage.java similarity index 92% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/RawMessage.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/RawMessage.java index a5c7923..e9e88d9 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/RawMessage.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/RawMessage.java @@ -1,4 +1,4 @@ -package io.es4j.queue.models; +package io.es4j.infrastructure.messagebroker.models; import io.soabase.recordbuilder.core.RecordBuilder; import io.vertx.core.json.JsonObject; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/TransactionProvider.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/TransactionProvider.java similarity index 51% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/TransactionProvider.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/TransactionProvider.java index c447b2e..9abe054 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/models/TransactionProvider.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/TransactionProvider.java @@ -1,4 +1,4 @@ -package io.es4j.queue.models; +package io.es4j.infrastructure.messagebroker.models; public enum TransactionProvider { diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgMessageProducer.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgMessageProducer.java similarity index 84% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgMessageProducer.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgMessageProducer.java index 6817554..1dc05dc 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgMessageProducer.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgMessageProducer.java @@ -1,20 +1,20 @@ -package io.es4j.queue.postgres; +package io.es4j.infrastructure.messagebroker.postgres; -import io.es4j.queue.models.Message; -import io.es4j.queue.models.MessageID; -import io.es4j.queue.models.MessageState; -import io.es4j.queue.models.QueueTransaction; -import io.es4j.queue.postgres.mappers.MessageQueueMapper; -import io.es4j.queue.postgres.models.MessageRecord; -import io.es4j.queue.postgres.models.MessageRecordID; -import io.es4j.queue.postgres.models.MessageRecordQuery; +import io.es4j.infrastructure.messagebroker.models.Message; +import io.es4j.infrastructure.messagebroker.models.MessageID; +import io.es4j.infrastructure.messagebroker.models.MessageState; +import io.es4j.infrastructure.messagebroker.models.QueueTransaction; +import io.es4j.infrastructure.messagebroker.postgres.mappers.MessageQueueMapper; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecord; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordID; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordQuery; import io.es4j.sql.Repository; import io.es4j.sql.RepositoryHandler; import io.es4j.sql.models.BaseRecord; import io.vertx.mutiny.sqlclient.SqlConnection; import io.smallrye.mutiny.Uni; import io.vertx.core.json.JsonObject; -import io.es4j.queue.MessageProducer; +import io.es4j.infrastructure.messagebroker.MessageProducer; import java.util.List; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgQueueTransaction.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgProcessorTransaction.java similarity index 58% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgQueueTransaction.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgProcessorTransaction.java index bfeb425..fc95c89 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgQueueTransaction.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgProcessorTransaction.java @@ -1,26 +1,26 @@ -package io.es4j.queue.postgres; +package io.es4j.infrastructure.messagebroker.postgres; -import io.es4j.queue.postgres.mappers.MessageTransactionMapper; -import io.es4j.queue.postgres.models.MessageTransaction; -import io.es4j.queue.postgres.models.MessageTransactionID; -import io.es4j.queue.postgres.models.MessageTransactionQuery; +import io.es4j.infrastructure.messagebroker.postgres.mappers.MessageTransactionMapper; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransaction; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransactionID; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransactionQuery; import io.es4j.sql.Repository; import io.es4j.sql.RepositoryHandler; import io.es4j.sql.models.BaseRecord; import io.smallrye.mutiny.Uni; -import io.es4j.queue.QueueTransactionManager; -import io.es4j.queue.models.Message; -import io.es4j.queue.models.QueueTransaction; +import io.es4j.infrastructure.messagebroker.ProcessorTransactionProvider; +import io.es4j.infrastructure.messagebroker.models.Message; +import io.es4j.infrastructure.messagebroker.models.QueueTransaction; import io.vertx.core.json.JsonObject; import io.vertx.mutiny.core.Vertx; import java.util.function.BiFunction; -public class PgQueueTransaction implements QueueTransactionManager { +public class PgProcessorTransaction implements ProcessorTransactionProvider { private final Repository transactionStore; - public PgQueueTransaction(Vertx vertx, JsonObject configuration) { + public PgProcessorTransaction(Vertx vertx, JsonObject configuration) { this.transactionStore = new Repository<>(MessageTransactionMapper.INSTANCE, RepositoryHandler.leasePool(configuration, vertx)); } diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgTaskSubscriber.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgQueueSubscriber.java similarity index 91% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgTaskSubscriber.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgQueueSubscriber.java index eed703f..8e1d8e4 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgTaskSubscriber.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgQueueSubscriber.java @@ -1,10 +1,10 @@ -package io.es4j.queue.postgres; +package io.es4j.infrastructure.messagebroker.postgres; -import io.es4j.queue.postgres.mappers.DeadLetterMapper; -import io.es4j.queue.postgres.mappers.MessageQueueMapper; -import io.es4j.queue.postgres.mappers.PgQueueLiquibase; -import io.es4j.queue.postgres.models.*; +import io.es4j.infrastructure.messagebroker.postgres.mappers.DeadLetterMapper; +import io.es4j.infrastructure.messagebroker.postgres.mappers.MessageQueueMapper; +import io.es4j.infrastructure.messagebroker.postgres.mappers.PgQueueLiquibase; +import io.es4j.infrastructure.messagebroker.postgres.models.*; import io.es4j.sql.Repository; import io.es4j.sql.RepositoryHandler; import io.es4j.sql.exceptions.NotFound; @@ -12,10 +12,10 @@ import io.es4j.sql.models.QueryOptions; import io.smallrye.mutiny.Multi; import io.smallrye.mutiny.subscription.FixedDemandPacer; -import io.es4j.queue.models.MessageState; -import io.es4j.queue.models.QueueConfiguration; -import io.es4j.queue.models.RawMessage; -import io.es4j.queue.models.MessageProcessorManager; +import io.es4j.infrastructure.messagebroker.models.MessageState; +import io.es4j.infrastructure.messagebroker.models.QueueConfiguration; +import io.es4j.infrastructure.messagebroker.models.RawMessage; +import io.es4j.infrastructure.messagebroker.models.MessageProcessorManager; import io.smallrye.mutiny.Uni; import io.vertx.core.impl.NoStackTraceThrowable; @@ -23,7 +23,7 @@ import io.vertx.mutiny.core.Vertx; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.es4j.queue.TaskSubscriber; +import io.es4j.infrastructure.messagebroker.QueueSubscriber; import java.time.Duration; import java.util.*; @@ -31,15 +31,15 @@ import static java.util.stream.Collectors.groupingBy; -public class PgTaskSubscriber implements TaskSubscriber { +public class PgQueueSubscriber implements QueueSubscriber { public static final AtomicBoolean LIQUIBASE_DEPLOYED = new AtomicBoolean(false); private final Repository messageQueue; private final Repository deadLetterQueue; private final io.vertx.mutiny.pgclient.pubsub.PgSubscriber pgSubscriber; - private static final Logger LOGGER = LoggerFactory.getLogger(PgTaskSubscriber.class); + private static final Logger LOGGER = LoggerFactory.getLogger(PgQueueSubscriber.class); - public PgTaskSubscriber(Vertx vertx, JsonObject configuration) { + public PgQueueSubscriber(Vertx vertx, JsonObject configuration) { final var repositoryHandler = RepositoryHandler.leasePool(configuration, vertx); this.messageQueue = new Repository<>(MessageQueueMapper.INSTANCE, repositoryHandler); this.deadLetterQueue = new Repository<>(DeadLetterMapper.INSTANCE, repositoryHandler); diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgRefresher.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgRefresher.java similarity index 97% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgRefresher.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgRefresher.java index 7481fc7..57429ba 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/PgRefresher.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgRefresher.java @@ -1,9 +1,9 @@ -package io.es4j.queue.postgres; +package io.es4j.infrastructure.messagebroker.postgres; import io.es4j.sql.RepositoryHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.es4j.queue.models.QueueConfiguration; +import io.es4j.infrastructure.messagebroker.models.QueueConfiguration; import java.util.concurrent.atomic.AtomicLong; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/DeadLetterMapper.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/DeadLetterMapper.java similarity index 92% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/DeadLetterMapper.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/DeadLetterMapper.java index 857267b..b230f81 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/DeadLetterMapper.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/DeadLetterMapper.java @@ -1,14 +1,14 @@ -package io.es4j.queue.postgres.mappers; +package io.es4j.infrastructure.messagebroker.postgres.mappers; import io.es4j.sql.RecordMapper; import io.es4j.sql.generator.filters.QueryBuilder; import io.es4j.sql.models.QueryFilter; import io.es4j.sql.models.QueryFilters; -import io.es4j.queue.models.MessageState; -import io.es4j.queue.postgres.models.DeadLetterKey; -import io.es4j.queue.postgres.models.DeadLetterRecord; -import io.es4j.queue.postgres.models.MessageRecordQuery; +import io.es4j.infrastructure.messagebroker.models.MessageState; +import io.es4j.infrastructure.messagebroker.postgres.models.DeadLetterKey; +import io.es4j.infrastructure.messagebroker.postgres.models.DeadLetterRecord; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordQuery; import io.vertx.sqlclient.Row; import java.time.Instant; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/MessageQueueMapper.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageQueueMapper.java similarity index 92% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/MessageQueueMapper.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageQueueMapper.java index 09c4c25..8f8401c 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/MessageQueueMapper.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageQueueMapper.java @@ -1,14 +1,14 @@ -package io.es4j.queue.postgres.mappers; +package io.es4j.infrastructure.messagebroker.postgres.mappers; import io.es4j.sql.RecordMapper; import io.es4j.sql.generator.filters.QueryBuilder; import io.es4j.sql.models.QueryFilter; import io.es4j.sql.models.QueryFilters; -import io.es4j.queue.models.MessageState; -import io.es4j.queue.postgres.models.MessageRecord; -import io.es4j.queue.postgres.models.MessageRecordID; -import io.es4j.queue.postgres.models.MessageRecordQuery; +import io.es4j.infrastructure.messagebroker.models.MessageState; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecord; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordID; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordQuery; import io.vertx.sqlclient.Row; import java.time.Instant; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/MessageTransactionMapper.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageTransactionMapper.java similarity index 86% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/MessageTransactionMapper.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageTransactionMapper.java index 01c20cb..9b8d3ed 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/MessageTransactionMapper.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageTransactionMapper.java @@ -1,11 +1,11 @@ -package io.es4j.queue.postgres.mappers; +package io.es4j.infrastructure.messagebroker.postgres.mappers; import io.es4j.sql.RecordMapper; import io.es4j.sql.generator.filters.QueryBuilder; import io.es4j.sql.models.QueryFilters; -import io.es4j.queue.postgres.models.MessageTransaction; -import io.es4j.queue.postgres.models.MessageTransactionID; -import io.es4j.queue.postgres.models.MessageTransactionQuery; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransaction; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransactionID; +import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransactionQuery; import io.vertx.sqlclient.Row; import java.util.Map; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/PgQueueLiquibase.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/PgQueueLiquibase.java similarity index 80% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/PgQueueLiquibase.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/PgQueueLiquibase.java index cbb9a34..8414bd1 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/mappers/PgQueueLiquibase.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/PgQueueLiquibase.java @@ -1,13 +1,13 @@ -package io.es4j.queue.postgres.mappers; +package io.es4j.infrastructure.messagebroker.postgres.mappers; import io.es4j.sql.LiquibaseHandler; import io.es4j.sql.RepositoryHandler; import io.es4j.sql.misc.Constants; import io.es4j.sql.misc.EnvVars; -import io.es4j.queue.models.QueueConfiguration; -import io.es4j.queue.postgres.PgRefresher; +import io.es4j.infrastructure.messagebroker.models.QueueConfiguration; +import io.es4j.infrastructure.messagebroker.postgres.PgRefresher; import io.smallrye.mutiny.Uni; -import io.es4j.queue.postgres.PgTaskSubscriber; +import io.es4j.infrastructure.messagebroker.postgres.PgQueueSubscriber; import java.util.Map; @@ -17,7 +17,7 @@ private PgQueueLiquibase(){} public static Uni bootstrapQueue(final RepositoryHandler repositoryHandler, QueueConfiguration configuration) { - if (PgTaskSubscriber.LIQUIBASE_DEPLOYED.compareAndSet(false, true)) { + if (PgQueueSubscriber.LIQUIBASE_DEPLOYED.compareAndSet(false, true)) { return liquibase(repositoryHandler) .invoke(avoid -> bootstrapQueueRefresher(repositoryHandler, configuration)); } @@ -27,7 +27,7 @@ public static Uni bootstrapQueue(final RepositoryHandler repositoryHandler private static Uni liquibase(RepositoryHandler repositoryHandler) { return LiquibaseHandler.liquibaseString( repositoryHandler, - "task-queue.xml", + "message-broker.xml", Map.of( "schema", repositoryHandler.configuration().getString(Constants.SCHEMA, EnvVars.SCHEMA) ) diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/DeadLetterKey.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterKey.java similarity index 77% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/DeadLetterKey.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterKey.java index 871baa7..c8ccb7c 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/DeadLetterKey.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterKey.java @@ -1,4 +1,4 @@ -package io.es4j.queue.postgres.models; +package io.es4j.infrastructure.messagebroker.postgres.models; import io.es4j.sql.models.RepositoryRecordKey; import io.soabase.recordbuilder.core.RecordBuilder; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/DeadLetterRecord.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterRecord.java similarity index 100% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/DeadLetterRecord.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterRecord.java diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageRecord.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecord.java similarity index 94% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageRecord.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecord.java index 9b7f645..311c8ee 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageRecord.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecord.java @@ -1,10 +1,10 @@ -package io.es4j.queue.postgres.models; +package io.es4j.infrastructure.messagebroker.postgres.models; import io.es4j.sql.models.BaseRecord; import io.es4j.sql.models.RepositoryRecord; import io.soabase.recordbuilder.core.RecordBuilder; -import io.es4j.queue.models.MessageState; -import io.es4j.queue.models.RawMessage; +import io.es4j.infrastructure.messagebroker.models.MessageState; +import io.es4j.infrastructure.messagebroker.models.RawMessage; import io.vertx.core.json.JsonObject; import java.time.Instant; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageRecordID.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordID.java similarity index 77% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageRecordID.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordID.java index c7327db..9ef6ca2 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageRecordID.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordID.java @@ -1,4 +1,4 @@ -package io.es4j.queue.postgres.models; +package io.es4j.infrastructure.messagebroker.postgres.models; import io.es4j.sql.models.RepositoryRecordKey; import io.soabase.recordbuilder.core.RecordBuilder; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageRecordQuery.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordQuery.java similarity index 80% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageRecordQuery.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordQuery.java index 0650c26..04235e0 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageRecordQuery.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordQuery.java @@ -1,9 +1,9 @@ -package io.es4j.queue.postgres.models; +package io.es4j.infrastructure.messagebroker.postgres.models; import io.es4j.sql.models.Query; import io.es4j.sql.models.QueryOptions; import io.soabase.recordbuilder.core.RecordBuilder; -import io.es4j.queue.models.MessageState; +import io.es4j.infrastructure.messagebroker.models.MessageState; import java.time.Instant; import java.util.List; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageTransaction.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransaction.java similarity index 88% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageTransaction.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransaction.java index e704891..d7ef9cd 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageTransaction.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransaction.java @@ -1,4 +1,4 @@ -package io.es4j.queue.postgres.models; +package io.es4j.infrastructure.messagebroker.postgres.models; import io.es4j.sql.models.BaseRecord; import io.es4j.sql.models.RepositoryRecord; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageTransactionID.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionID.java similarity index 78% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageTransactionID.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionID.java index 6086510..f6b8619 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageTransactionID.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionID.java @@ -1,4 +1,4 @@ -package io.es4j.queue.postgres.models; +package io.es4j.infrastructure.messagebroker.postgres.models; import io.es4j.sql.models.RepositoryRecordKey; import io.soabase.recordbuilder.core.RecordBuilder; diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageTransactionQuery.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionQuery.java similarity index 82% rename from es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageTransactionQuery.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionQuery.java index 225a6cb..283911f 100644 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/postgres/models/MessageTransactionQuery.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionQuery.java @@ -1,4 +1,4 @@ -package io.es4j.queue.postgres.models; +package io.es4j.infrastructure.messagebroker.postgres.models; import io.es4j.sql.models.Query; diff --git a/es4j-infrastructure/es4j-queue/src/main/resources/task-queue.xml b/es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/message-broker.xml similarity index 100% rename from es4j-infrastructure/es4j-queue/src/main/resources/task-queue.xml rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/message-broker.xml diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/QueueTransactionManager.java b/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/QueueTransactionManager.java deleted file mode 100644 index fbcd7e7..0000000 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/QueueTransactionManager.java +++ /dev/null @@ -1,13 +0,0 @@ -package io.es4j.queue; - - -import io.es4j.queue.models.Message; -import io.es4j.queue.models.QueueTransaction; -import io.smallrye.mutiny.Uni; - -import java.util.function.BiFunction; - -public interface QueueTransactionManager { - - Uni transaction(Message message, BiFunction, QueueTransaction, Uni> function); -} diff --git a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/TaskSubscriber.java b/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/TaskSubscriber.java deleted file mode 100644 index 4bc3cc5..0000000 --- a/es4j-infrastructure/es4j-queue/src/main/java/io/es4j/queue/TaskSubscriber.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.es4j.queue; - - -import io.es4j.queue.models.MessageProcessorManager; -import io.smallrye.mutiny.Uni; - -public interface TaskSubscriber { - - Uni unsubscribe(); - - Uni subscribe(MessageProcessorManager messageProcessorManager); - - - -} diff --git a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java index 0673116..7a8c871 100644 --- a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java +++ b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java @@ -25,6 +25,8 @@ import java.util.*; import java.util.function.Consumer; +import static io.es4j.core.CommandHandler.camelToKebab; + @AutoService(EventStore.class) public class RedisEventStore implements EventStore { @@ -164,12 +166,13 @@ public Uni stop() { @Override public void start(Deployment deployment, Vertx vertx, JsonObject configuration) { + this.aggregateClass = deployment.aggregateClass(); this.redisClient = Redis.createClient(vertx, new RedisOptions() .setMaxPoolSize(CpuCoreSensor.availableProcessors()) .setMaxWaitingHandlers(CpuCoreSensor.availableProcessors() * 4) .setPassword(configuration.getString("redisPassword")) - .setPoolName(aggregateClass.getSimpleName()) + .setPoolName(camelToKebab(deployment.aggregateClass().getSimpleName())) .setConnectionString("redis://:%s@%s:%s/%s".formatted( configuration.getString("redisPassword"), configuration.getString("redisHost"), diff --git a/es4j-infrastructure/es4j-task/pom.xml b/es4j-infrastructure/es4j-task/pom.xml index cfaea25..25530ba 100644 --- a/es4j-infrastructure/es4j-task/pom.xml +++ b/es4j-infrastructure/es4j-task/pom.xml @@ -43,7 +43,7 @@ io.es4j - es4j-queue + es4j-postgres-queue ${project.version} diff --git a/es4j-infrastructure/pom.xml b/es4j-infrastructure/pom.xml index a6fc4d3..c8c59fc 100644 --- a/es4j-infrastructure/pom.xml +++ b/es4j-infrastructure/pom.xml @@ -16,7 +16,7 @@ es4j-sql es4j-task - es4j-queue + es4j-postgres-message-broker es4j-http-bridge es4j-config-storage es4j-postgres-storage diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java b/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java index 133fd79..64dc892 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java @@ -2,7 +2,7 @@ import io.smallrye.mutiny.Uni; import io.es4j.queue.MessageProcessor; -import io.es4j.queue.models.QueueTransaction; +import io.es4j.infrastructure.messagebroker.models.QueueTransaction; public class MockProcessor implements MessageProcessor { diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/PgTaskQueueTest.java b/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/PgTaskQueueTest.java index 79d48c8..aa9389b 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/PgTaskQueueTest.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/PgTaskQueueTest.java @@ -5,15 +5,15 @@ //import io.es4j.sql.Repository; //import io.es4j.sql.exceptions.NotFound; //import io.es4j.InfrastructureBootstrap; -//import io.es4j.queue.models.QueueTransaction; -//import io.es4j.queue.postgres.PgMessageProducer; -//import io.es4j.queue.models.Message; -//import io.es4j.queue.postgres.mappers.DeadLetterMapper; -//import io.es4j.queue.postgres.mappers.MessageQueueMapper; -//import io.es4j.queue.postgres.mappers.MessageTransactionMapper; -//import io.es4j.queue.postgres.models.DeadLetterKey; -//import io.es4j.queue.postgres.models.MessageRecordID; -//import io.es4j.queue.postgres.models.MessageTransactionID; +//import io.es4j.infrastructure.messagebroker.models.QueueTransaction; +//import io.es4j.infrastructure.messagebroker.postgres.PgMessageProducer; +//import io.es4j.infrastructure.messagebroker.models.Message; +//import io.es4j.infrastructure.messagebroker.postgres.mappers.DeadLetterMapper; +//import io.es4j.infrastructure.messagebroker.postgres.mappers.MessageQueueMapper; +//import io.es4j.infrastructure.messagebroker.postgres.mappers.MessageTransactionMapper; +//import io.es4j.infrastructure.messagebroker.postgres.models.DeadLetterKey; +//import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordID; +//import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransactionID; //import org.jetbrains.annotations.NotNull; //import org.junit.jupiter.api.AfterAll; //import org.junit.jupiter.api.BeforeAll; From 517b8bbd4bf570f2fa0d2f1fbdb4920522e9e453 Mon Sep 17 00:00:00 2001 From: reeferman Date: Tue, 18 Jul 2023 16:07:57 +0200 Subject: [PATCH 13/17] - more fixes --- .../infrastructure/messagebroker/models/MessageState.java | 2 +- .../messagebroker/postgres/models/DeadLetterRecord.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageState.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageState.java index 8d1b556..4ab1545 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageState.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageState.java @@ -1,4 +1,4 @@ -package io.es4j.queue.models; +package io.es4j.infrastructure.messagebroker.models; public enum MessageState { CREATED, PROCESSING, SCHEDULED, EXPIRED, RETRY, RETRIES_EXHAUSTED, RECOVERY, PROCESSED, FATAL_FAILURE diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterRecord.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterRecord.java index 026c42f..f3c8f1f 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterRecord.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterRecord.java @@ -1,10 +1,11 @@ -package io.es4j.queue.postgres.models; +package io.es4j.infrastructure.messagebroker.postgres.models; +import io.es4j.infrastructure.messagebroker.models.MessageState; import io.es4j.sql.models.BaseRecord; import io.es4j.sql.models.RepositoryRecord; import io.soabase.recordbuilder.core.RecordBuilder; import io.vertx.core.json.JsonObject; -import io.es4j.queue.models.MessageState; + import java.time.Instant; import java.util.Map; From 6cc255651a8a5a897b8fae950d952f1bf7938d02 Mon Sep 17 00:00:00 2001 From: reeferman Date: Thu, 20 Jul 2023 15:26:02 +0200 Subject: [PATCH 14/17] - more fixes --- ...ntProjection.java => AsyncProjection.java} | 11 +- ...rojection.java => AsyncStateTransfer.java} | 12 +- .../{Deployment.java => Es4jDeployment.java} | 4 +- ...EventStream.java => InlineProjection.java} | 2 +- ...teStream.java => InlineStateTransfer.java} | 2 +- .../es4j/core/objects/EventJournalFilter.java | 1 + .../core/objects/StateProjectionWrapper.java | 6 +- .../core/projections/EventStreamListener.java | 10 +- .../core/tasks/EventProjectionPoller.java | 26 ++- .../core/tasks/StateProjectionPoller.java | 4 +- .../core/verticles/AggregateVerticle.java | 10 +- .../core/verticles/TaskProcessorVerticle.java | 171 ------------------ .../io/es4j/infrastructure/EventStore.java | 6 +- .../es4j/infrastructure/Infrastructure.java | 22 +-- .../io/es4j/infrastructure/OffsetStore.java | 6 +- .../infrastructure/SecondaryEventStore.java | 7 +- .../misc/Es4jServiceLoader.java | 20 +- .../io/es4j/launcher/AggregateDeployer.java | 40 ++-- .../main/java/io/es4j/launcher/Es4jMain.java | 2 +- .../main/java/io/es4j/http/HttpBridge.java | 4 +- .../java/io/es4j/http/ProjectionRoute.java | 6 +- .../java/io/es4j/infra/pg/PgEventStore.java | 8 +- .../java/io/es4j/infra/pg/PgOffsetStore.java | 9 +- .../es4j/infra/pg/PgSecondaryEventStore.java | 10 +- .../io/es4j/infra/redis/RedisEventStore.java | 10 +- .../io/es4j/infra/redis/RedisOffsetStore.java | 7 +- es4j-infrastructure/es4j-task/pom.xml | 2 +- es4j-infrastructure/pom.xml | 21 ++- .../main/java/io/es4j/Es4jBootstrapper.java | 2 +- .../java/io/es4j/domain/Bootstrapper.java | 6 +- .../es4j/infrastructure/EventStoreTest.java | 16 +- .../es4j/infrastructure/OffsetStoreTest.java | 8 +- .../taskqueue/MockDeadPayloadProcessor.java | 48 ++--- .../taskqueue/MockProcessor.java | 27 ++- 34 files changed, 174 insertions(+), 372 deletions(-) rename es4j-core/src/main/java/io/es4j/{PollingEventProjection.java => AsyncProjection.java} (88%) rename es4j-core/src/main/java/io/es4j/{PollingStateProjection.java => AsyncStateTransfer.java} (81%) rename es4j-core/src/main/java/io/es4j/{Deployment.java => Es4jDeployment.java} (93%) rename es4j-core/src/main/java/io/es4j/{LiveEventStream.java => InlineProjection.java} (95%) rename es4j-core/src/main/java/io/es4j/{LiveStateStream.java => InlineStateTransfer.java} (80%) delete mode 100644 es4j-core/src/main/java/io/es4j/core/verticles/TaskProcessorVerticle.java diff --git a/es4j-core/src/main/java/io/es4j/PollingEventProjection.java b/es4j-core/src/main/java/io/es4j/AsyncProjection.java similarity index 88% rename from es4j-core/src/main/java/io/es4j/PollingEventProjection.java rename to es4j-core/src/main/java/io/es4j/AsyncProjection.java index b890a16..25e840b 100644 --- a/es4j-core/src/main/java/io/es4j/PollingEventProjection.java +++ b/es4j-core/src/main/java/io/es4j/AsyncProjection.java @@ -17,7 +17,7 @@ * The PollingEventProjection interface defines the structure for event projections * that use a polling strategy. */ -public interface PollingEventProjection { +public interface AsyncProjection { /** * Apply the list of AggregateEvent objects to the projection. @@ -36,15 +36,6 @@ default Optional filter() { return Optional.empty(); } - /** - * Retrieve the tenant associated with the projection. - * - * @return a String representing the tenant. Defaults to "default". - */ - default String tenant() { - return "default"; - } - /** * Defines the polling policy for the projection. * diff --git a/es4j-core/src/main/java/io/es4j/PollingStateProjection.java b/es4j-core/src/main/java/io/es4j/AsyncStateTransfer.java similarity index 81% rename from es4j-core/src/main/java/io/es4j/PollingStateProjection.java rename to es4j-core/src/main/java/io/es4j/AsyncStateTransfer.java index eac75f6..794343c 100644 --- a/es4j-core/src/main/java/io/es4j/PollingStateProjection.java +++ b/es4j-core/src/main/java/io/es4j/AsyncStateTransfer.java @@ -19,7 +19,7 @@ * * @param the type of the aggregate */ -public interface PollingStateProjection { +public interface AsyncStateTransfer { /** * Updates the current state of an aggregate. @@ -29,16 +29,6 @@ public interface PollingStateProjection { */ Uni update(AggregateState currentState); - /** - * Returns the tenant for this polling state projection. - * - *

The default implementation returns "default", but this can be overridden by subclasses.

- * - * @return a string representing the tenant - */ - default String tenant() { - return "default"; - } /** * Returns the polling policy for this polling state projection. diff --git a/es4j-core/src/main/java/io/es4j/Deployment.java b/es4j-core/src/main/java/io/es4j/Es4jDeployment.java similarity index 93% rename from es4j-core/src/main/java/io/es4j/Deployment.java rename to es4j-core/src/main/java/io/es4j/Es4jDeployment.java index 4a1b696..3ee3a0d 100644 --- a/es4j-core/src/main/java/io/es4j/Deployment.java +++ b/es4j-core/src/main/java/io/es4j/Es4jDeployment.java @@ -6,14 +6,12 @@ import java.util.Collections; import java.util.List; -import static io.es4j.core.CommandHandler.camelToKebab; - /** * The Bootstrap interface is responsible for providing the framework * with the necessary information regarding the aggregate roots that * need to be taken into consideration. */ -public interface Deployment { +public interface Es4jDeployment { /** * Provides the class of the aggregate root that should be taken into diff --git a/es4j-core/src/main/java/io/es4j/LiveEventStream.java b/es4j-core/src/main/java/io/es4j/InlineProjection.java similarity index 95% rename from es4j-core/src/main/java/io/es4j/LiveEventStream.java rename to es4j-core/src/main/java/io/es4j/InlineProjection.java index a3e209e..3f85fe9 100644 --- a/es4j-core/src/main/java/io/es4j/LiveEventStream.java +++ b/es4j-core/src/main/java/io/es4j/InlineProjection.java @@ -8,7 +8,7 @@ * within the framework. Implementors of this interface will be called in a best-effort * manner for emitted eventTypes. */ -public interface LiveEventStream { +public interface InlineProjection { /** * Apply an Event to the live stream. This method will be called in a best-effort diff --git a/es4j-core/src/main/java/io/es4j/LiveStateStream.java b/es4j-core/src/main/java/io/es4j/InlineStateTransfer.java similarity index 80% rename from es4j-core/src/main/java/io/es4j/LiveStateStream.java rename to es4j-core/src/main/java/io/es4j/InlineStateTransfer.java index 12d682b..6071b4c 100644 --- a/es4j-core/src/main/java/io/es4j/LiveStateStream.java +++ b/es4j-core/src/main/java/io/es4j/InlineStateTransfer.java @@ -4,7 +4,7 @@ import io.smallrye.mutiny.Uni; // create javadoc for this interface -public interface LiveStateStream { +public interface InlineStateTransfer { Uni update(AggregateState currentState); diff --git a/es4j-core/src/main/java/io/es4j/core/objects/EventJournalFilter.java b/es4j-core/src/main/java/io/es4j/core/objects/EventJournalFilter.java index fce5b67..7ccb572 100644 --- a/es4j-core/src/main/java/io/es4j/core/objects/EventJournalFilter.java +++ b/es4j-core/src/main/java/io/es4j/core/objects/EventJournalFilter.java @@ -7,6 +7,7 @@ @RecordBuilder public record EventJournalFilter( + String tenant, List eventTypes, List tags ) { diff --git a/es4j-core/src/main/java/io/es4j/core/objects/StateProjectionWrapper.java b/es4j-core/src/main/java/io/es4j/core/objects/StateProjectionWrapper.java index 544cb3e..f77a9ab 100644 --- a/es4j-core/src/main/java/io/es4j/core/objects/StateProjectionWrapper.java +++ b/es4j-core/src/main/java/io/es4j/core/objects/StateProjectionWrapper.java @@ -1,16 +1,16 @@ package io.es4j.core.objects; import io.es4j.Aggregate; -import io.es4j.PollingStateProjection; +import io.es4j.AsyncStateTransfer; import io.smallrye.mutiny.Uni; import org.slf4j.Logger; public record StateProjectionWrapper( - PollingStateProjection pollingStateProjection, + AsyncStateTransfer asyncStateTransfer, Class entityAggregateClass, Logger logger ) { public Uni update(AggregateState state) { - return pollingStateProjection.update(state); + return asyncStateTransfer.update(state); } } diff --git a/es4j-core/src/main/java/io/es4j/core/projections/EventStreamListener.java b/es4j-core/src/main/java/io/es4j/core/projections/EventStreamListener.java index 294cade..09f385c 100644 --- a/es4j-core/src/main/java/io/es4j/core/projections/EventStreamListener.java +++ b/es4j-core/src/main/java/io/es4j/core/projections/EventStreamListener.java @@ -1,7 +1,7 @@ package io.es4j.core.projections; import io.es4j.Aggregate; -import io.es4j.LiveEventStream; +import io.es4j.InlineProjection; import io.smallrye.mutiny.Uni; import io.vertx.mutiny.core.Vertx; import org.slf4j.Logger; @@ -13,15 +13,15 @@ public class EventStreamListener { private final Vertx vertx; private final Class aggregateClass; - private final List liveEventStreamConsumer; + private final List inlineProjectionConsumer; protected static final Logger LOGGER = LoggerFactory.getLogger(EventStreamListener.class); - public EventStreamListener(Vertx vertx, Class aggregateClass, List liveEventStreamConsumer) { + public EventStreamListener(Vertx vertx, Class aggregateClass, List inlineProjectionConsumer) { this.vertx = vertx; this.aggregateClass = aggregateClass; - this.liveEventStreamConsumer = liveEventStreamConsumer; + this.inlineProjectionConsumer = inlineProjectionConsumer; } @@ -41,7 +41,7 @@ public Uni start() { // .replaceWithVoid(); } - public static void handle(Throwable throwable, LiveEventStream consumer) { + public static void handle(Throwable throwable, InlineProjection consumer) { LOGGER.error("Error in live event stream {}::{}", consumer.getClass().getName(), consumer.tenant(), throwable); } diff --git a/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java b/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java index be1c564..206f40f 100644 --- a/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java +++ b/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java @@ -14,7 +14,7 @@ import io.es4j.task.CronTaskConfigurationBuilder; import io.es4j.task.LockLevel; import io.smallrye.mutiny.Uni; -import io.es4j.PollingEventProjection; +import io.es4j.AsyncProjection; import io.es4j.infrastructure.misc.EventParser; import io.es4j.infrastructure.models.EventStream; @@ -26,18 +26,17 @@ public class EventProjectionPoller implements CronTask { - private static final Logger logger = LoggerFactory.getLogger(EventProjectionPoller.class); - private final PollingEventProjection pollingEventProjection; + private final AsyncProjection asyncProjection; private final EventStore eventStore; private final OffsetStore offsetStore; public EventProjectionPoller( - PollingEventProjection pollingEventProjections, + AsyncProjection asyncProjections, EventStore eventStore, OffsetStore offsetStore ) { - this.pollingEventProjection = pollingEventProjections; + this.asyncProjection = asyncProjections; this.eventStore = eventStore; this.offsetStore = offsetStore; } @@ -45,26 +44,26 @@ public EventProjectionPoller( @Override public Uni performTask() { return offsetStore.get(getOffset()) - .flatMap(journalOffset -> eventStore.fetch(streamStatement(pollingEventProjection, journalOffset)) - .flatMap(events -> pollingEventProjection.apply(parseEvents(events)) + .flatMap(journalOffset -> eventStore.fetch(streamStatement(asyncProjection, journalOffset)) + .flatMap(events -> asyncProjection.apply(parseEvents(events)) .flatMap(avoid -> offsetStore.put(journalOffset.updateOffset(events))) ) ) - .onFailure().invoke(throwable -> logger.error("Unable to update projection {}", pollingEventProjection.getClass().getName(), throwable)) + .onFailure().invoke(throwable -> logger.error("Unable to update projection {}", asyncProjection.getClass().getName(), throwable)) .replaceWithVoid(); } private OffsetKey getOffset() { - return new OffsetKey(pollingEventProjection.getClass().getName(), pollingEventProjection.tenant()); + return new OffsetKey(asyncProjection.getClass().getName(), "default"); } - private static EventStream streamStatement(PollingEventProjection pollingEventProjection, Offset offset) { + private static EventStream streamStatement(AsyncProjection asyncProjection, Offset offset) { AtomicReference eventStream = new AtomicReference<>(); - pollingEventProjection.filter().ifPresentOrElse( + asyncProjection.filter().ifPresentOrElse( filter -> eventStream.set( EventStreamBuilder.builder() .eventTypes(filter.eventTypes()) - .tenantId(pollingEventProjection.tenant()) + .tenantId(filter.tenant()) .offset(offset.idOffSet()) .batchSize(1000) .tags(filter.tags()) @@ -72,7 +71,6 @@ private static EventStream streamStatement(PollingEventProjection pollingEventPr ), () -> eventStream.set( EventStreamBuilder.builder() - .tenantId(pollingEventProjection.tenant()) .offset(offset.idOffSet()) .batchSize(1000) .build() @@ -100,7 +98,7 @@ public CronTaskConfiguration configuration() { return CronTaskConfigurationBuilder.builder() .knownInterruptions(List.of(NotFound.class)) .lockLevel(LockLevel.CLUSTER_WIDE) - .cron(pollingEventProjection.pollingPolicy()) + .cron(asyncProjection.pollingPolicy()) .build(); } diff --git a/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java b/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java index 7740798..3aa5dee 100644 --- a/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java +++ b/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java @@ -49,7 +49,7 @@ public Uni performTask() { // polling will have to be moved to a queue // a new entry must be inserted in the queue for each one of the updated streams stateProjectionWrapper.logger().debug("Polling events"); - return offsetStore.get(new OffsetKey(stateProjectionWrapper.pollingStateProjection().getClass().getName(), "default")) + return offsetStore.get(new OffsetKey(stateProjectionWrapper.asyncStateTransfer().getClass().getName(), "default")) .flatMap(journalOffset -> { stateProjectionWrapper.logger().debug("Journal idOffset at {}", journalOffset.idOffSet()); return eventStore.fetch(EventStreamBuilder.builder() @@ -87,7 +87,7 @@ public CronTaskConfiguration configuration() { return CronTaskConfigurationBuilder.builder() .knownInterruptions(List.of(NotFound.class)) .lockLevel(LockLevel.CLUSTER_WIDE) - .cron(stateProjectionWrapper.pollingStateProjection().pollingPolicy()) + .cron(stateProjectionWrapper.asyncStateTransfer().pollingPolicy()) .build(); } diff --git a/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java b/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java index 6ee3e34..06ac80f 100644 --- a/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java +++ b/es4j-core/src/main/java/io/es4j/core/verticles/AggregateVerticle.java @@ -35,7 +35,7 @@ public class AggregateVerticle extends AbstractVerticle { public static final String ACTION = "action"; private final Class aggregateClass; private final String nodeDeploymentID; - private final Deployment deployment; + private final Es4jDeployment es4jDeployment; private CommandHandler commandHandler; private List behaviourWraps; private List aggregatorWraps; @@ -43,11 +43,11 @@ public class AggregateVerticle extends AbstractVerticle { private Es4jService es4jService; public AggregateVerticle( - final Deployment deployment, + final Es4jDeployment es4jDeployment, final Class aggregateClass, final String nodeDeploymentID ) { - this.deployment= deployment; + this.es4jDeployment = es4jDeployment; this.aggregateClass = aggregateClass; this.nodeDeploymentID = nodeDeploymentID; } @@ -64,7 +64,7 @@ public Uni asyncStart() { Optional.empty(), Es4jServiceLoader.loadOffsetStore() ); - infrastructure.start(deployment, vertx, config()); + infrastructure.start(es4jDeployment, vertx, config()); vertx.eventBus().addInboundInterceptor(this::addContextualData); this.commandHandler = new CommandHandler<>( vertx, @@ -72,7 +72,7 @@ public Uni asyncStart() { aggregatorWraps, behaviourWraps, infrastructure, - deployment.aggregateConfiguration() + es4jDeployment.aggregateConfiguration() ); this.es4jService = new Es4jService( infrastructure.offsetStore(), diff --git a/es4j-core/src/main/java/io/es4j/core/verticles/TaskProcessorVerticle.java b/es4j-core/src/main/java/io/es4j/core/verticles/TaskProcessorVerticle.java deleted file mode 100644 index 781374b..0000000 --- a/es4j-core/src/main/java/io/es4j/core/verticles/TaskProcessorVerticle.java +++ /dev/null @@ -1,171 +0,0 @@ -package io.es4j.core.verticles; - - -import io.smallrye.mutiny.Uni; -import io.smallrye.mutiny.vertx.core.AbstractVerticle; -import io.vertx.core.Promise; - -import org.crac.Context; -import org.crac.Resource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import io.vertx.core.json.JsonObject; -import io.es4j.infrastructure.misc.Es4jServiceLoader; -import io.es4j.queue.MessageProcessor; -import io.es4j.queue.TaskSubscriber; -import io.es4j.queue.QueueTransactionManager; -import io.es4j.infrastructure.messagebroker.exceptions.QueueError; -import io.es4j.infrastructure.messagebroker.exceptions.MessageException; -import io.es4j.infrastructure.messagebroker.models.MessageProcessorManager; -import io.es4j.infrastructure.messagebroker.models.MessageProcessorWrapper; -import io.es4j.infrastructure.messagebroker.models.QueueConfiguration; -import io.es4j.infrastructure.messagebroker.postgres.PgTaskSubscriber; -import io.es4j.infrastructure.messagebroker.postgres.PgQueueTransaction; -import io.vertx.mutiny.core.Vertx; - -import java.util.*; -import java.util.concurrent.CountDownLatch; - -import static java.util.stream.Collectors.groupingBy; - -public class TaskProcessorVerticle extends AbstractVerticle implements Resource { - private final List messageProcessors; - - @Override - public void beforeCheckpoint(Context context) throws Exception { - Promise p = Promise.promise(); - stop(p); - CountDownLatch latch = new CountDownLatch(1); - p.future().onComplete(event -> latch.countDown()); - latch.await(); - } - - @Override - public void afterRestore(Context context) throws Exception { - Promise p = Promise.promise(); - start(p); - CountDownLatch latch = new CountDownLatch(1); - p.future().onComplete(event -> latch.countDown()); - latch.await(); - } - - private static final Logger LOGGER = LoggerFactory.getLogger(TaskProcessorVerticle.class); - private TaskSubscriber subscriber; - private QueueConfiguration taskConfiguration; - - public TaskProcessorVerticle() { - this.messageProcessors = ServiceLoader.load(MessageProcessor.class).stream().map(ServiceLoader.Provider::get).toList(); - ; - org.crac.Core.getGlobalContext().register(this); - } - - - public static Uni deploy( - final Vertx vertx, - final JsonObject newConfiguration - ) { - return Uni.createFrom().voidItem(); - } - - @Override - public Uni asyncStop() { - return subscriber.unsubscribe(); - } - - @Override - public Uni asyncStart() { - this.taskConfiguration = config().getJsonObject("task-queue", new JsonObject()).mapTo(QueueConfiguration.class); - QueueTransactionManager queueTransactionManager = getTransactionManager(); - this.subscriber = getSubscriber(); - return subscriber.subscribe(new MessageProcessorManager( - taskConfiguration, - bootstrapProcessors(this.deploymentID()), - queueTransactionManager, - vertx - ) - ) - .replaceWithVoid(); - } - - private QueueTransactionManager getTransactionManager() { - return switch (taskConfiguration.transactionManagerImplementation()) { - case VERTX_PG_CLIENT -> new PgQueueTransaction(vertx, config()); - }; - } - - private TaskSubscriber getSubscriber() { - return switch (taskConfiguration.queueImplementation()) { - case PG_QUEUE -> new PgTaskSubscriber(vertx, config()); - case RABBITMQ -> - throw new MessageException(new QueueError("queue type not supported", "rabbit task queue is not yet implemented", 999)); - case SOLACE -> - throw new MessageException(new QueueError("queue type not supported", "solace task queue is not yet implemented", 999)); - }; - } - - - public List bootstrapProcessors(String deploymentId) { - final var queueMap = new HashMap, List>(); - messageProcessors.forEach( - impl -> { - final var tClass = Es4jServiceLoader.getFirstGenericType(impl); - if (queueMap.containsKey(tClass)) { - queueMap.get(tClass).add(impl); - } else { - final var param = new ArrayList(); - param.add(impl); - queueMap.put(tClass, param); - } - } - ); - return queueMap.entrySet().stream() - .map(entry -> { - final var tClass = entry.getKey(); - validateProcessors(entry.getValue(), tClass); - final var defaultProcessor = entry.getValue().stream().filter(p -> p.tenants() == null).findFirst().orElseThrow(); - final var customProcessors = entry.getValue().stream() - .filter(p -> p.tenants() != null) - .collect(groupingBy(MessageProcessor::tenants)); - final var queueWrapper = new MessageProcessorWrapper( - deploymentId, - defaultProcessor, - customProcessors, - tClass - ); - logQueueConfiguration(queueWrapper, taskConfiguration); - return queueWrapper; - } - ) - .toList(); - } - - private static void validateProcessors(List queues, Class tClass) { - if (queues.stream().filter( - p -> p.tenants() == null - ).toList().size() > 1) { - throw new IllegalStateException("More than one default implementation for -> " + tClass); - } - queues.stream() - .filter(p -> p.tenants() != null) - .collect(groupingBy(MessageProcessor::tenants)) - .forEach((key, value) -> { - if (value.size() > 1) { - throw new IllegalStateException("More than one custom implementation for tenant " + key + " queue -> " + tClass); - } - } - ); - } - - public static void logQueueConfiguration(final MessageProcessorWrapper messageProcessorWrapper, QueueConfiguration queueConfiguration) { - final var customProcessors = new JsonObject(); - messageProcessorWrapper.customProcessors() - .forEach((key, value) -> key.forEach(k -> customProcessors.put(k, value.getClass().getName()))); - final var json = new JsonObject() - .put("defaultProcessor", messageProcessorWrapper.defaultProcessor().getClass().getName()) - .put("customProcessors", customProcessors.encodePrettily()) - .put("payloadClass", messageProcessorWrapper.payloadClass().getName()) - .put("configuration", JsonObject.mapFrom(queueConfiguration).encodePrettily()); - LOGGER.info("Queue configuration {} ", json.encodePrettily()); - } - -} diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/EventStore.java b/es4j-core/src/main/java/io/es4j/infrastructure/EventStore.java index 0457bd0..bcfbd3f 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/EventStore.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/EventStore.java @@ -1,7 +1,7 @@ package io.es4j.infrastructure; import io.es4j.Aggregate; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; import io.es4j.infrastructure.models.*; import io.smallrye.mutiny.Uni; import io.vertx.core.json.JsonObject; @@ -23,8 +23,8 @@ public interface EventStore { Uni startStream(StartStream appendInstruction); Uni stop(); - void start(Deployment deployment, Vertx vertx, JsonObject configuration); - Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration); + void start(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration); + Uni setup(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration); Uni trim(PruneEventStream trim); } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/Infrastructure.java b/es4j-core/src/main/java/io/es4j/infrastructure/Infrastructure.java index cbfd33d..7602fa9 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/Infrastructure.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/Infrastructure.java @@ -1,8 +1,6 @@ package io.es4j.infrastructure; -import io.es4j.Aggregate; -import io.es4j.Deployment; -import io.es4j.core.objects.AggregateConfiguration; +import io.es4j.Es4jDeployment; import io.smallrye.mutiny.Uni; import io.vertx.core.json.JsonObject; import io.vertx.mutiny.core.Vertx; @@ -26,19 +24,19 @@ public Uni stop() { return Uni.join().all(list).andFailFast().replaceWithVoid(); } - public Uni setup(Deployment deployment, Vertx vertx, JsonObject infrastructureConfiguration) { + public Uni setup(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject infrastructureConfiguration) { final var list = new ArrayList>(); - cache.ifPresent(cache -> list.add(cache.setup(deployment.aggregateClass(), deployment.aggregateConfiguration()))); - secondaryEventStore.ifPresent(secondaryEventStore -> list.add(secondaryEventStore.setup(deployment, vertx, infrastructureConfiguration))); - list.add(eventStore.setup(deployment, vertx, infrastructureConfiguration)); - list.add(offsetStore.setup(deployment, vertx, infrastructureConfiguration)); + cache.ifPresent(cache -> list.add(cache.setup(es4jDeployment.aggregateClass(), es4jDeployment.aggregateConfiguration()))); + secondaryEventStore.ifPresent(secondaryEventStore -> list.add(secondaryEventStore.setup(es4jDeployment, vertx, infrastructureConfiguration))); + list.add(eventStore.setup(es4jDeployment, vertx, infrastructureConfiguration)); + list.add(offsetStore.setup(es4jDeployment, vertx, infrastructureConfiguration)); return Uni.join().all(list).andFailFast().replaceWithVoid(); } - public void start(Deployment deployment, Vertx vertx, JsonObject configuration) { - eventStore.start(deployment, vertx, configuration); - offsetStore.start(deployment, vertx, configuration); - secondaryEventStore.ifPresent(ses -> ses.start(deployment, vertx, configuration)); + public void start(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration) { + eventStore.start(es4jDeployment, vertx, configuration); + offsetStore.start(es4jDeployment, vertx, configuration); + secondaryEventStore.ifPresent(ses -> ses.start(es4jDeployment, vertx, configuration)); } } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/OffsetStore.java b/es4j-core/src/main/java/io/es4j/infrastructure/OffsetStore.java index 619b1e8..439be83 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/OffsetStore.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/OffsetStore.java @@ -1,7 +1,7 @@ package io.es4j.infrastructure; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; import io.es4j.core.objects.Offset; import io.es4j.infrastructure.models.OffsetFilter; import io.smallrye.mutiny.Uni; @@ -19,6 +19,6 @@ public interface OffsetStore { Uni> projections(OffsetFilter offsetFilter); Uni stop(); - void start(Deployment deployment, Vertx vertx, JsonObject configuration); - Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration); + void start(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration); + Uni setup(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration); } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/SecondaryEventStore.java b/es4j-core/src/main/java/io/es4j/infrastructure/SecondaryEventStore.java index 9b23deb..8df98a7 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/SecondaryEventStore.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/SecondaryEventStore.java @@ -1,8 +1,7 @@ package io.es4j.infrastructure; import io.es4j.Aggregate; -import io.es4j.Deployment; -import io.es4j.infrastructure.models.AggregateEventStream; +import io.es4j.Es4jDeployment; import io.es4j.infrastructure.models.AppendInstruction; import io.es4j.infrastructure.models.Event; import io.es4j.infrastructure.models.EventStream; @@ -24,9 +23,9 @@ public interface SecondaryEventStore { Uni stop(); - void start(Deployment deployment, Vertx vertx, JsonObject configuration); + void start(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration); - Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration); + Uni setup(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration); } diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java b/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java index 300e91b..32c453b 100644 --- a/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java +++ b/es4j-core/src/main/java/io/es4j/infrastructure/misc/Es4jServiceLoader.java @@ -35,26 +35,26 @@ public static EventStore loadEventStore() { .orElseThrow(() -> new IllegalStateException("EventStore not found")); } - public static List stateProjections() { - return ServiceLoader.load(PollingStateProjection.class).stream() + public static List stateProjections() { + return ServiceLoader.load(AsyncStateTransfer.class).stream() .map(ServiceLoader.Provider::get) .toList(); } - public static List pollingEventProjections() { - return ServiceLoader.load(PollingEventProjection.class).stream() + public static List pollingEventProjections() { + return ServiceLoader.load(AsyncProjection.class).stream() .map(ServiceLoader.Provider::get) .toList(); } - public static List liveEventProjections() { - return ServiceLoader.load(LiveEventStream.class).stream() + public static List liveEventProjections() { + return ServiceLoader.load(InlineProjection.class).stream() .map(ServiceLoader.Provider::get) .toList(); } - public static List liveStateProjections() { - return ServiceLoader.load(LiveStateStream.class).stream() + public static List liveStateProjections() { + return ServiceLoader.load(InlineStateTransfer.class).stream() .map(ServiceLoader.Provider::get) .toList(); } @@ -72,8 +72,8 @@ public static List loadAggregators() { .toList(); } - public static List bootstrapList() { - return ServiceLoader.load(Deployment.class).stream() + public static List bootstrapList() { + return ServiceLoader.load(Es4jDeployment.class).stream() .map(ServiceLoader.Provider::get) .peek(aggregate -> { LOGGER.info("Bootstrapper found {}", aggregate); diff --git a/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java b/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java index e3b004a..5591cdb 100644 --- a/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java +++ b/es4j-core/src/main/java/io/es4j/launcher/AggregateDeployer.java @@ -3,8 +3,8 @@ import io.es4j.Aggregate; -import io.es4j.Deployment; -import io.es4j.PollingStateProjection; +import io.es4j.Es4jDeployment; +import io.es4j.AsyncStateTransfer; import io.es4j.core.tasks.AggregateHeartbeat; import io.es4j.core.verticles.AggregateVerticle; import io.es4j.infrastructure.*; @@ -39,7 +39,7 @@ public class AggregateDeployer { protected static final Logger LOGGER = LoggerFactory.getLogger(AggregateDeployer.class); private final Vertx vertx; private final String nodeDeploymentID; - private final Deployment deploymentConfiguration; + private final Es4jDeployment es4jDeploymentConfiguration; private final Class aggregateClass; private Infrastructure infrastructure; private List aggregateServices; @@ -49,11 +49,11 @@ public class AggregateDeployer { public AggregateDeployer( final Class aggregateClass, - final Deployment deployment, + final Es4jDeployment es4jDeployment, final Vertx vertx, final String nodeDeploymentID ) { - this.deploymentConfiguration = deployment; + this.es4jDeploymentConfiguration = es4jDeployment; this.vertx = vertx; this.aggregateClass = aggregateClass; this.nodeDeploymentID = nodeDeploymentID; @@ -62,18 +62,18 @@ public AggregateDeployer( public void deploy(final Promise startPromise) { Es4jConfigurationHandler.configure( vertx, - deploymentConfiguration.infrastructureConfiguration(), + es4jDeploymentConfiguration.infrastructureConfiguration(), infrastructureConfiguration -> { - infrastructureConfiguration.put("schema", camelToKebab(deploymentConfiguration.aggregateClass().getSimpleName())); - LOGGER.info("--- Es4j starting {}::{} --- {}", deploymentConfiguration.infrastructureConfiguration(), this.nodeDeploymentID, infrastructureConfiguration.encodePrettily()); + infrastructureConfiguration.put("schema", camelToKebab(es4jDeploymentConfiguration.aggregateClass().getSimpleName())); + LOGGER.info("--- Es4j starting {}::{} --- {}", es4jDeploymentConfiguration.infrastructureConfiguration(), this.nodeDeploymentID, infrastructureConfiguration.encodePrettily()); close() .flatMap(avoid -> infrastructure(vertx, infrastructureConfiguration) ) .call(injector -> { addHeartBeat(); addProjections(); - final Supplier supplier = () -> new AggregateVerticle<>(deploymentConfiguration, aggregateClass, nodeDeploymentID); - return startChannel(vertx, deploymentConfiguration.aggregateClass(), nodeDeploymentID) + final Supplier supplier = () -> new AggregateVerticle<>(es4jDeploymentConfiguration, aggregateClass, nodeDeploymentID); + return startChannel(vertx, es4jDeploymentConfiguration.aggregateClass(), nodeDeploymentID) .flatMap(avoid -> vertx.deployVerticle(supplier, new DeploymentOptions() .setConfig(infrastructureConfiguration) .setInstances(CpuCoreSensor.availableProcessors() * 2) @@ -82,10 +82,10 @@ public void deploy(final Promise startPromise) { ) .call(avoid -> { this.aggregateServices = Es4jServiceLoader.loadAggregateServices(); - return Es4jConfigurationHandler.fsConfigurations(vertx, deploymentConfiguration.fileBusinessRules()) + return Es4jConfigurationHandler.fsConfigurations(vertx, es4jDeploymentConfiguration.fileBusinessRules()) .flatMap(av -> Multi.createFrom().iterable(aggregateServices) .onItem().transformToUniAndMerge( - service -> service.start(deploymentConfiguration.aggregateClass(), vertx, infrastructureConfiguration) + service -> service.start(es4jDeploymentConfiguration.aggregateClass(), vertx, infrastructureConfiguration) ) .collect().asList() .replaceWithVoid() @@ -98,10 +98,10 @@ public void deploy(final Promise startPromise) { .with( aVoid -> { startPromise.complete(); - LOGGER.info("--- Es4j {} started ---", deploymentConfiguration.aggregateClass().getSimpleName()); + LOGGER.info("--- Es4j {} started ---", es4jDeploymentConfiguration.aggregateClass().getSimpleName()); } , throwable -> { - LOGGER.error("--- Es4j {} failed to start ---", deploymentConfiguration.aggregateClass().getSimpleName(), throwable); + LOGGER.error("--- Es4j {} failed to start ---", es4jDeploymentConfiguration.aggregateClass().getSimpleName(), throwable); startPromise.fail(throwable); } ); @@ -110,7 +110,7 @@ public void deploy(final Promise startPromise) { } private void addHeartBeat() { - timerTaskDeployer.deploy(new AggregateHeartbeat<>(vertx, deploymentConfiguration.aggregateClass())); + timerTaskDeployer.deploy(new AggregateHeartbeat<>(vertx, es4jDeploymentConfiguration.aggregateClass())); } private Uni infrastructure(Vertx vertx, JsonObject configuration) { @@ -127,15 +127,15 @@ private Uni infrastructure(Vertx vertx, JsonObject configuration) { if (Objects.isNull(timerTaskDeployer)) { timerTaskDeployer = new TimerTaskDeployer(vertx); } - return infrastructure.setup(deploymentConfiguration, vertx, configuration) - .invoke(avoid -> infrastructure.start(deploymentConfiguration, vertx, configuration)); + return infrastructure.setup(es4jDeploymentConfiguration, vertx, configuration) + .invoke(avoid -> infrastructure.start(es4jDeploymentConfiguration, vertx, configuration)); } private void addProjections() { final var aggregateProxy = new AggregateEventBusPoxy<>(vertx, aggregateClass); final var stateProjections = Es4jServiceLoader.stateProjections().stream() - .filter(cc -> Es4jServiceLoader.getFirstGenericType(cc).isAssignableFrom(deploymentConfiguration.aggregateClass())) + .filter(cc -> Es4jServiceLoader.getFirstGenericType(cc).isAssignableFrom(es4jDeploymentConfiguration.aggregateClass())) .map(cc -> gettStateProjectionWrapper(cc, aggregateClass)) .map(tStateProjectionWrapper -> new StateProjectionPoller( aggregateClass, @@ -146,7 +146,7 @@ private void addProjections() { )) .toList(); final var eventProjections = Es4jServiceLoader.pollingEventProjections().stream() - .filter(cc -> cc.aggregateClass().isAssignableFrom(deploymentConfiguration.aggregateClass())) + .filter(cc -> cc.aggregateClass().isAssignableFrom(es4jDeploymentConfiguration.aggregateClass())) .map(eventProjection -> new EventProjectionPoller( eventProjection, infrastructure.eventStore(), @@ -158,7 +158,7 @@ private void addProjections() { stateProjections.forEach(cronTaskDeployer::deploy); } - private StateProjectionWrapper gettStateProjectionWrapper(PollingStateProjection cc, Class aggregateClass) { + private StateProjectionWrapper gettStateProjectionWrapper(AsyncStateTransfer cc, Class aggregateClass) { return new StateProjectionWrapper( cc, aggregateClass, diff --git a/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java b/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java index d30b74b..82ae7a8 100644 --- a/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java +++ b/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java @@ -33,7 +33,7 @@ public class Es4jMain extends AbstractVerticle implements Resource { protected static final Logger LOGGER = LoggerFactory.getLogger(Es4jMain.class); - public static final List AGGREGATES = Es4jServiceLoader.bootstrapList(); + public static final List AGGREGATES = Es4jServiceLoader.bootstrapList(); private static final List> AGGREGATE_DEPLOYERS = new ArrayList<>(); public static final Map, List>> AGGREGATE_COMMANDS = new HashMap<>(); public static final Map, List>> AGGREGATE_EVENTS = new HashMap<>(); diff --git a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java index 533da17..264dc6e 100644 --- a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java +++ b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/HttpBridge.java @@ -3,7 +3,7 @@ import com.google.auto.service.AutoService; import io.es4j.Aggregate; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; import io.es4j.Command; import io.es4j.core.objects.DefaultFilters; import io.es4j.core.objects.Es4jError; @@ -147,7 +147,7 @@ private void aggregateWebSocket(Router router) { final var options = new SockJSHandlerOptions().setRegisterWriteHandler(true); final var bridgeOptions = new SockJSBridgeOptions(); //todo add web-socket command ingress - Es4jMain.AGGREGATES.stream().map(Deployment::aggregateClass).forEach( + Es4jMain.AGGREGATES.stream().map(Es4jDeployment::aggregateClass).forEach( aClass -> bridgeOptions .addInboundPermitted(permission(AggregateBus.COMMAND_BRIDGE, aClass)) .addOutboundPermitted(permission(EventbusLiveStreams.STATE_STREAM, aClass)) diff --git a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/ProjectionRoute.java b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/ProjectionRoute.java index ffdaecf..9a39c67 100644 --- a/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/ProjectionRoute.java +++ b/es4j-infrastructure/es4j-http-bridge/src/main/java/io/es4j/http/ProjectionRoute.java @@ -13,7 +13,7 @@ import io.es4j.Aggregate; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; import io.es4j.infrastructure.models.EventFilter; import io.es4j.infrastructure.proxy.AggregateEventBusPoxy; import io.vertx.core.json.JsonArray; @@ -29,14 +29,14 @@ public class ProjectionRoute implements HttpRoute { @Override public Uni start(Vertx vertx, JsonObject configuration) { - Es4jMain.AGGREGATES.stream().map(Deployment::aggregateClass) + Es4jMain.AGGREGATES.stream().map(Es4jDeployment::aggregateClass) .forEach(aggregateClass -> proxies.put(aggregateClass, new AggregateEventBusPoxy<>(vertx, aggregateClass))); return Uni.createFrom().voidItem(); } @Override public void registerRoutes(Router router) { - Es4jMain.AGGREGATES.stream().map(Deployment::aggregateClass).forEach( + Es4jMain.AGGREGATES.stream().map(Es4jDeployment::aggregateClass).forEach( aClass -> { router.post(Es4jService.fetchEventsAddress(aClass)) .consumes(Constants.APPLICATION_JSON) diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java index ebf7e2d..e352e41 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java @@ -2,7 +2,7 @@ import com.google.auto.service.AutoService; import io.es4j.Aggregate; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; import io.es4j.infra.pg.models.EventRecordKey; import io.es4j.infra.pg.models.EventRecordQuery; import io.es4j.infrastructure.models.*; @@ -37,7 +37,7 @@ public class PgEventStore implements EventStore { @Override - public void start(Deployment deployment, Vertx vertx, JsonObject configuration) { + public void start(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration) { this.eventJournal = new Repository<>(EventStoreMapper.INSTANCE, RepositoryHandler.leasePool(configuration, vertx)); } @@ -136,8 +136,8 @@ public Uni stop() { @Override - public Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration) { - final var schema = camelToKebab(deployment.aggregateClass().getSimpleName()); + public Uni setup(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration) { + final var schema = camelToKebab(es4jDeployment.aggregateClass().getSimpleName()); LOGGER.debug("Migrating postgres schema {} configuration {}", schema, configuration); configuration.put("schema", schema); return LiquibaseHandler.liquibaseString( diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgOffsetStore.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgOffsetStore.java index 450469e..d2f45e0 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgOffsetStore.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgOffsetStore.java @@ -1,8 +1,7 @@ package io.es4j.infra.pg; import com.google.auto.service.AutoService; -import io.es4j.Aggregate; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; import io.es4j.core.objects.OffsetBuilder; import io.es4j.core.objects.OffsetKey; import io.es4j.infra.pg.mappers.JournalOffsetMapper; @@ -45,7 +44,7 @@ public Uni stop() { } @Override - public void start(Deployment deployment, Vertx vertx, JsonObject config) { + public void start(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject config) { this.repository = new Repository<>(JournalOffsetMapper.INSTANCE, RepositoryHandler.leasePool(config, vertx)); } @@ -122,8 +121,8 @@ private static Offset getJournalOffset(EventJournalOffSet offset) { } @Override - public Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration) { - final var schema = camelToKebab(deployment.aggregateClass().getSimpleName()); + public Uni setup(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration) { + final var schema = camelToKebab(es4jDeployment.aggregateClass().getSimpleName()); LOGGER.debug("Migrating postgres schema {} configuration {}", schema, configuration); configuration.put("schema", schema); return LiquibaseHandler.liquibaseString( diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java index 43b7c97..c823d30 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgSecondaryEventStore.java @@ -1,7 +1,7 @@ package io.es4j.infra.pg; import io.es4j.Aggregate; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; import io.es4j.infra.pg.mappers.EventStoreMapper; import io.es4j.infra.pg.models.EventRecord; import io.es4j.infra.pg.models.EventRecordKey; @@ -80,18 +80,18 @@ public Uni stop() { } @Override - public void start(Deployment deployment, Vertx vertx, JsonObject configuration) { + public void start(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration) { this.eventJournal = new Repository<>(EventStoreMapper.INSTANCE, RepositoryHandler.leasePool(configuration, vertx)); } @Override - public Uni setup(Deployment deployment, Vertx vertx, JsonObject configuration) { - LOGGER.debug("Migrating database for {} with configuration {}", deployment.aggregateClass().getSimpleName(), configuration); + public Uni setup(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration) { + LOGGER.debug("Migrating database for {} with configuration {}", es4jDeployment.aggregateClass().getSimpleName(), configuration); return LiquibaseHandler.liquibaseString( eventJournal.repositoryHandler(), "pg-event-store.xml", - Map.of("schema", camelToKebab(deployment.aggregateClass().getSimpleName())) + Map.of("schema", camelToKebab(es4jDeployment.aggregateClass().getSimpleName())) ); } diff --git a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java index 7a8c871..3913be5 100644 --- a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java +++ b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java @@ -2,7 +2,7 @@ import com.google.auto.service.AutoService; import io.es4j.Aggregate; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; import io.es4j.core.objects.ErrorSource; import io.es4j.core.objects.Es4jErrorBuilder; import io.es4j.infrastructure.EventStore; @@ -165,14 +165,14 @@ public Uni stop() { } @Override - public void start(Deployment deployment, Vertx vertx, JsonObject configuration) { - this.aggregateClass = deployment.aggregateClass(); + public void start(Es4jDeployment es4jDeployment, Vertx vertx, JsonObject configuration) { + this.aggregateClass = es4jDeployment.aggregateClass(); this.redisClient = Redis.createClient(vertx, new RedisOptions() .setMaxPoolSize(CpuCoreSensor.availableProcessors()) .setMaxWaitingHandlers(CpuCoreSensor.availableProcessors() * 4) .setPassword(configuration.getString("redisPassword")) - .setPoolName(camelToKebab(deployment.aggregateClass().getSimpleName())) + .setPoolName(camelToKebab(es4jDeployment.aggregateClass().getSimpleName())) .setConnectionString("redis://:%s@%s:%s/%s".formatted( configuration.getString("redisPassword"), configuration.getString("redisHost"), @@ -186,7 +186,7 @@ public void start(Deployment deployment, Vertx vertx, JsonObject configuration) } @Override - public Uni setup(Deployment aggregateClass, Vertx vertx, JsonObject configuration) { + public Uni setup(Es4jDeployment aggregateClass, Vertx vertx, JsonObject configuration) { return null; } diff --git a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisOffsetStore.java b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisOffsetStore.java index b579de6..b5a266c 100644 --- a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisOffsetStore.java +++ b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisOffsetStore.java @@ -1,8 +1,7 @@ package io.es4j.infra.redis; import com.google.auto.service.AutoService; -import io.es4j.Aggregate; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; import io.es4j.core.objects.Offset; import io.es4j.core.objects.OffsetKey; import io.es4j.infrastructure.OffsetStore; @@ -42,12 +41,12 @@ public Uni stop() { } @Override - public void start(Deployment aggregateClass, Vertx vertx, JsonObject configuration) { + public void start(Es4jDeployment aggregateClass, Vertx vertx, JsonObject configuration) { } @Override - public Uni setup(Deployment aggregateClass, Vertx vertx, JsonObject configuration) { + public Uni setup(Es4jDeployment aggregateClass, Vertx vertx, JsonObject configuration) { return null; } diff --git a/es4j-infrastructure/es4j-task/pom.xml b/es4j-infrastructure/es4j-task/pom.xml index 25530ba..b995dfd 100644 --- a/es4j-infrastructure/es4j-task/pom.xml +++ b/es4j-infrastructure/es4j-task/pom.xml @@ -43,7 +43,7 @@ io.es4j - es4j-postgres-queue + es4j-postgres-message-broker ${project.version} diff --git a/es4j-infrastructure/pom.xml b/es4j-infrastructure/pom.xml index c8c59fc..53afe33 100644 --- a/es4j-infrastructure/pom.xml +++ b/es4j-infrastructure/pom.xml @@ -25,16 +25,17 @@ - - - io.es4j - es4j-dependencies - ${project.version} - pom - import - - - + + + io.es4j + es4j-dependencies + ${project.version} + pom + import + + + + 17 17 diff --git a/es4j-test/src/main/java/io/es4j/Es4jBootstrapper.java b/es4j-test/src/main/java/io/es4j/Es4jBootstrapper.java index 47669b0..06586aa 100644 --- a/es4j-test/src/main/java/io/es4j/Es4jBootstrapper.java +++ b/es4j-test/src/main/java/io/es4j/Es4jBootstrapper.java @@ -59,7 +59,7 @@ public Es4jBootstrapper( } public void bootstrap() { - final var deployment = new Deployment() { + final var deployment = new Es4jDeployment() { @Override public Class aggregateClass() { return aggregateClass; diff --git a/es4j-test/src/test/java/io/es4j/domain/Bootstrapper.java b/es4j-test/src/test/java/io/es4j/domain/Bootstrapper.java index 68188bc..1b41a93 100644 --- a/es4j-test/src/test/java/io/es4j/domain/Bootstrapper.java +++ b/es4j-test/src/test/java/io/es4j/domain/Bootstrapper.java @@ -2,11 +2,11 @@ import com.google.auto.service.AutoService; import io.es4j.Aggregate; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; -@AutoService(Deployment.class) -public class Bootstrapper implements Deployment { +@AutoService(Es4jDeployment.class) +public class Bootstrapper implements Es4jDeployment { @Override public Class aggregateClass() { return FakeAggregate.class; diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java b/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java index 567cc09..ed7e556 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/EventStoreTest.java @@ -1,7 +1,7 @@ package io.es4j.infrastructure; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; import io.es4j.domain.FakeAggregate; import io.es4j.infra.pg.PgEventStore; import io.es4j.infrastructure.models.AggregateEventStreamBuilder; @@ -29,7 +29,7 @@ class EventStoreTest { public static final String TENANT_ID = "default"; - public static final Deployment DEPLOYMENT = () -> FakeAggregate.class; + public static final Es4jDeployment ES_4_J_DEPLOYMENT = () -> FakeAggregate.class; private static PostgreSQLContainer POSTGRES_CONTAINER; private static final Vertx vertx = Vertx.vertx(); private static final JsonObject CONFIGURATION = new JsonObject(); @@ -49,8 +49,8 @@ static void stop() { @ParameterizedTest @MethodSource("eventStores") void append_ensure_uniqueness(EventStore eventStore) { - eventStore.setup(DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); - eventStore.start(DEPLOYMENT, vertx, CONFIGURATION); + eventStore.setup(ES_4_J_DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); + eventStore.start(ES_4_J_DEPLOYMENT, vertx, CONFIGURATION); // Define eventTypes and append instructions final var aggregateId = UUID.randomUUID().toString(); final var goodAppend = createAppendInstruction(aggregateId); @@ -69,8 +69,8 @@ void append_ensure_uniqueness(EventStore eventStore) { @ParameterizedTest @MethodSource("eventStores") void append_and_fetch(EventStore eventStore) { - eventStore.setup(DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); - eventStore.start(DEPLOYMENT, vertx, CONFIGURATION); + eventStore.setup(ES_4_J_DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); + eventStore.start(ES_4_J_DEPLOYMENT, vertx, CONFIGURATION); // Define eventTypes and append instructions final var aggregateId = UUID.randomUUID().toString(); int numberOfEvents = 100; @@ -93,8 +93,8 @@ void append_and_fetch(EventStore eventStore) { @ParameterizedTest @MethodSource("eventStores") void append_and_stream(EventStore eventStore) { - eventStore.setup(DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); - eventStore.start(DEPLOYMENT, vertx, CONFIGURATION); + eventStore.setup(ES_4_J_DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); + eventStore.start(ES_4_J_DEPLOYMENT, vertx, CONFIGURATION); // Define eventTypes and append instructions final var aggregateId = UUID.randomUUID().toString(); int numberOfEvents = 100; diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/OffsetStoreTest.java b/es4j-test/src/test/java/io/es4j/infrastructure/OffsetStoreTest.java index 762933a..d0175b0 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/OffsetStoreTest.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/OffsetStoreTest.java @@ -1,6 +1,6 @@ package io.es4j.infrastructure; -import io.es4j.Deployment; +import io.es4j.Es4jDeployment; import io.es4j.core.objects.Offset; import io.es4j.core.objects.OffsetBuilder; import io.es4j.core.objects.OffsetKeyBuilder; @@ -24,7 +24,7 @@ class OffsetStoreTest { public static final String TENANT_ID = "default"; private static PostgreSQLContainer POSTGRES_CONTAINER; - private static final Deployment DEPLOYMENT = () -> FakeAggregate.class; + private static final Es4jDeployment ES_4_J_DEPLOYMENT = () -> FakeAggregate.class; private static final Vertx vertx = Vertx.vertx(); private static final JsonObject CONFIGURATION = new JsonObject(); @@ -43,8 +43,8 @@ static void stop() { @ParameterizedTest @MethodSource("offsetStores") void test_offset_store(OffsetStore offsetStore) { - offsetStore.setup(DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); - offsetStore.start(DEPLOYMENT, vertx, CONFIGURATION); + offsetStore.setup(ES_4_J_DEPLOYMENT, vertx, CONFIGURATION).await().indefinitely(); + offsetStore.start(ES_4_J_DEPLOYMENT, vertx, CONFIGURATION); final var name = "fake-consumer"; final var offset = Assertions.assertDoesNotThrow( () -> offsetStore.put(createOffset(name)).await().indefinitely() diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockDeadPayloadProcessor.java b/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockDeadPayloadProcessor.java index e4d03ed..1f367af 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockDeadPayloadProcessor.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockDeadPayloadProcessor.java @@ -1,24 +1,24 @@ -package io.es4j.infrastructure.taskqueue; - -import io.smallrye.mutiny.Uni; -import io.es4j.queue.MessageProcessor; -import io.es4j.queue.models.QueueTransaction; - -import java.util.List; - -public class MockDeadPayloadProcessor implements MessageProcessor { - @Override - public Uni process(MockDeadPayload payload, QueueTransaction queueTransaction) { - if (payload.fatal()) { - return Uni.createFrom().failure(new FatalException("Fatal failure !")); - } - return Uni.createFrom().failure(new RuntimeException("Mocking failure !")); - } - - @Override - public List> fatalExceptions() { - return List.of(FatalException.class); - } - - -} +//package io.es4j.infrastructure.taskqueue; +// +//import io.smallrye.mutiny.Uni; +//import io.es4j.queue.MessageProcessor; +//import io.es4j.queue.models.QueueTransaction; +// +//import java.util.List; +// +//public class MockDeadPayloadProcessor implements MessageProcessor { +// @Override +// public Uni process(MockDeadPayload payload, QueueTransaction queueTransaction) { +// if (payload.fatal()) { +// return Uni.createFrom().failure(new FatalException("Fatal failure !")); +// } +// return Uni.createFrom().failure(new RuntimeException("Mocking failure !")); +// } +// +// @Override +// public List> fatalExceptions() { +// return List.of(FatalException.class); +// } +// +// +//} diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java b/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java index 64dc892..4b9b5a4 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java @@ -1,14 +1,13 @@ -package io.es4j.infrastructure.taskqueue; - -import io.smallrye.mutiny.Uni; -import io.es4j.queue.MessageProcessor; -import io.es4j.infrastructure.messagebroker.models.QueueTransaction; - - -public class MockProcessor implements MessageProcessor { - - @Override - public Uni process(MockPayload payload, QueueTransaction queueTransaction) { - return Uni.createFrom().voidItem(); - } -} +//package io.es4j.infrastructure.taskqueue; +// +//import io.smallrye.mutiny.Uni; +// +// +// +//public class MockProcessor implements MessageProcessor { +// +// @Override +// public Uni process(MockPayload payload, QueueTransaction queueTransaction) { +// return Uni.createFrom().voidItem(); +// } +//} From 3afd4c92ed46390c0fc67d75b927776dfc4518db Mon Sep 17 00:00:00 2001 From: reeferman Date: Sat, 22 Jul 2023 22:10:54 +0200 Subject: [PATCH 15/17] - postgres broker tests --- .../java/io/es4j/core/CommandHandler.java | 5 +- .../es4j/core/tasks/AggregateHeartbeat.java | 4 +- .../core/tasks/EventProjectionPoller.java | 2 - .../core/tasks/StateProjectionPoller.java | 2 - .../models/ConcurrentAppend.java | 8 + .../models/EventStoreExeception.java | 7 + .../main/java/io/es4j/launcher/Es4jMain.java | 2 +- .../es4j-config-storage/pom.xml | 2 +- .../io/es4j/config/ConfigurationFilter.java | 0 .../io/es4j/config/ConfigurationRoute.java | 0 .../io/es4j/config/DatabaseConfiguration.java | 0 .../config/DatabaseConfigurationCache.java | 0 .../config/DatabaseConfigurationFetcher.java | 0 .../DatabaseConfigurationInternalService.java | 0 .../config/DatabaseConfigurationService.java | 0 .../java/io/es4j/config/TarGzipHandler.java | 0 .../config/exceptions/ConfigStoreError.java | 0 .../exceptions/ConfigStoreException.java | 0 .../io/es4j/config/orm/ConfigurationKey.java | 0 .../es4j/config/orm/ConfigurationQuery.java | 0 .../es4j/config/orm/ConfigurationRecord.java | 0 .../config/orm/ConfigurationRecordMapper.java | 0 .../src/main/resources/config.xml | 0 es4j-extensions/es4j-notifications/pom.xml | 20 ++ es4j-extensions/pom.xml | 28 ++ .../es4j-postgres-message-broker/pom.xml | 5 + .../messagebroker/MessageProducer.java | 15 -- .../ProcessorTransactionProvider.java | 13 - .../messagebroker/QueueSubscriber.java | 15 -- .../messagebroker/TopicSubscriber.java | 15 -- .../exceptions/ConsumerException.java | 26 -- .../exceptions/MessageException.java | 11 - .../messagebroker/exceptions/QueueError.java | 8 - .../exceptions/QueueException.java | 39 --- .../messagebroker/models/Message.java | 26 -- .../models/MessageProcessorManager.java | 108 -------- .../messagebroker/models/MessageState.java | 5 - .../models/QueueConfiguration.java | 163 ------------ .../models/QueueImplementation.java | 6 - .../models/QueueTransaction.java | 12 - .../messagebroker/models/RawMessage.java | 36 --- .../models/TransactionProvider.java | 6 - .../postgres/PgMessageProducer.java | 131 ---------- .../postgres/PgProcessorTransaction.java | 43 --- .../postgres/PgQueueSubscriber.java | 231 ---------------- .../messagebroker/postgres/PgRefresher.java | 110 -------- .../postgres/mappers/DeadLetterMapper.java | 140 ---------- .../postgres/mappers/MessageQueueMapper.java | 139 ---------- .../postgres/mappers/PgQueueLiquibase.java | 66 ----- .../postgres/models/DeadLetterRecord.java | 96 ------- .../postgres/models/MessageTransaction.java | 18 -- .../pgbroker/ConsumerTransactionProvider.java | 18 ++ .../infrastructure/pgbroker/PgBroker.java | 95 +++++++ .../pgbroker/PgBrokerVerticle.java | 139 ++++++++++ .../QueueConsumer.java} | 19 +- .../pgbroker/TopicConsumer.java | 42 +++ .../exceptions/ConsumerExeception.java | 15 ++ .../pgbroker/exceptions/DuplicateMessage.java | 10 + .../exceptions/InterruptMessageStream.java | 6 + .../exceptions/InvalidProcessorException.java | 4 + .../exceptions/MessageParsingException.java | 4 + .../exceptions/PartitionNotFound.java | 4 + .../exceptions/PartitionTakenException.java | 4 + .../exceptions/ProducerExeception.java | 15 ++ .../exceptions/QueueEmptyException.java | 4 + .../pgbroker/mappers/DeadLetterMapper.java | 164 ++++++++++++ .../pgbroker/mappers/MessageQueueMapper.java | 155 +++++++++++ .../mappers/MessageTransactionMapper.java | 11 +- .../mappers/QueuePartitionMapper.java | 73 ++++++ .../ConcurrentPollingSession.java | 126 +++++++++ .../messagebroker/MessageProcessor.java | 232 ++++++++++++++++ .../messagebroker/PartitionHashRing.java | 56 ++++ .../PartitionPollingSession.java | 247 ++++++++++++++++++ .../pgbroker/messagebroker/PgChannel.java | 82 ++++++ .../messagebroker/SessionManager.java | 164 ++++++++++++ .../messagebroker/SessionRefresher.java | 102 ++++++++ .../pgbroker/models/ConsumerManager.java | 116 ++++++++ .../pgbroker/models/ConsumerTransaction.java | 10 + .../models/ConsumerWrap.java} | 14 +- .../models/DeadLetterKey.java | 4 +- .../pgbroker/models/DeadLetterRecord.java | 34 +++ .../pgbroker/models/Message.java | 38 +++ .../models/MessageID.java | 3 +- .../pgbroker/models/MessagePartition.java | 24 ++ .../models/MessageRecord.java | 79 ++---- .../models/MessageRecordID.java | 3 +- .../models/MessageRecordQuery.java | 8 +- .../pgbroker/models/MessageState.java | 5 + .../pgbroker/models/MessageTransaction.java | 27 ++ .../models/MessageTransactionID.java | 3 +- .../models/MessageTransactionQuery.java | 6 +- .../pgbroker/models/PartitionKey.java | 11 + .../pgbroker/models/PartitionQuery.java | 15 ++ .../models/PgBrokerConfiguration.java | 45 ++++ .../pgbroker/models/RawMessage.java | 54 ++++ .../vertx/VertxConsumerTransaction.java | 37 +++ .../pgbroker/vertx/VertxMessageProducer.java | 169 ++++++++++++ .../src/main/resources/queue.xml | 173 ++++++++++++ .../{message-broker.xml => topic.xml} | 90 ++++--- .../java/io/es4j/infra/pg/PgEventStore.java | 10 +- .../io/es4j/infra/redis/RedisEventStore.java | 3 +- es4j-infrastructure/es4j-sql/pom.xml | 6 - .../main/java/io/es4j/sql/RecordMapper.java | 147 ++++++----- .../java/io/es4j/sql/models/QueryFilters.java | 9 +- es4j-infrastructure/es4j-task/pom.xml | 10 +- .../src/main/java/io/es4j/task/CronTask.java | 6 +- .../io/es4j/task/CronTaskConfiguration.java | 3 +- .../java/io/es4j/task/CronTaskDeployer.java | 4 +- .../main/java/io/es4j/task/CronTaskKey.java | 8 - .../java/io/es4j/task/CronTaskMapper.java | 92 ------- .../main/java/io/es4j/task/CronTaskQuery.java | 20 -- .../java/io/es4j/task/CronTaskRecord.java | 25 -- .../src/main/java/io/es4j/task/TimerTask.java | 6 +- .../io/es4j/task/TimerTaskConfiguration.java | 4 +- .../java/io/es4j/task/TimerTaskDeployer.java | 15 +- .../es4j-task/src/main/resources/task.xml | 31 --- es4j-infrastructure/pom.xml | 1 - es4j-test/pom.xml | 5 + .../messagebroker/DeadPayloadConsumer.java | 26 ++ .../FatalException.java | 2 +- .../MockDeadPayload.java | 2 +- .../MockPayload.java | 2 +- .../messagebroker/MockPayloadConsumer.java | 16 ++ .../messagebroker/PgTaskQueueTest.java | 218 ++++++++++++++++ .../RecoveryTestPayload.java | 2 +- .../es4j/infrastructure/sql/SqlBootstrap.java | 4 +- .../taskqueue/MockDeadPayloadProcessor.java | 24 -- .../taskqueue/MockProcessor.java | 13 - .../taskqueue/PgTaskQueueTest.java | 136 ---------- es4j-test/src/test/resources/logback-test.xml | 2 +- pom.xml | 115 ++++---- 131 files changed, 3160 insertions(+), 2129 deletions(-) create mode 100644 es4j-core/src/main/java/io/es4j/infrastructure/models/ConcurrentAppend.java create mode 100644 es4j-core/src/main/java/io/es4j/infrastructure/models/EventStoreExeception.java rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/pom.xml (97%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/ConfigurationFilter.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/ConfigurationRoute.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfiguration.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationCache.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationFetcher.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationInternalService.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationService.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/TarGzipHandler.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/exceptions/ConfigStoreError.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/exceptions/ConfigStoreException.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationKey.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationQuery.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationRecord.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationRecordMapper.java (100%) rename {es4j-infrastructure => es4j-extensions}/es4j-config-storage/src/main/resources/config.xml (100%) create mode 100644 es4j-extensions/es4j-notifications/pom.xml create mode 100644 es4j-extensions/pom.xml delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProducer.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/ProcessorTransactionProvider.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/QueueSubscriber.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/TopicSubscriber.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/ConsumerException.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/MessageException.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueError.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueException.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/Message.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorManager.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageState.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueConfiguration.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueImplementation.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueTransaction.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/RawMessage.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/TransactionProvider.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgMessageProducer.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgProcessorTransaction.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgQueueSubscriber.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgRefresher.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/DeadLetterMapper.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageQueueMapper.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/PgQueueLiquibase.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterRecord.java delete mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransaction.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/ConsumerTransactionProvider.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/PgBroker.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/PgBrokerVerticle.java rename es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/{messagebroker/MessageProcessor.java => pgbroker/QueueConsumer.java} (54%) create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/TopicConsumer.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/ConsumerExeception.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/DuplicateMessage.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/InterruptMessageStream.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/InvalidProcessorException.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/MessageParsingException.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/PartitionNotFound.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/PartitionTakenException.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/ProducerExeception.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/QueueEmptyException.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/DeadLetterMapper.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/MessageQueueMapper.java rename es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/{messagebroker/postgres => pgbroker}/mappers/MessageTransactionMapper.java (83%) create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/QueuePartitionMapper.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/ConcurrentPollingSession.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/MessageProcessor.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PartitionHashRing.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PartitionPollingSession.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PgChannel.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/SessionManager.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/SessionRefresher.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/ConsumerManager.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/ConsumerTransaction.java rename es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/{messagebroker/models/MessageProcessorWrapper.java => pgbroker/models/ConsumerWrap.java} (58%) rename es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/{messagebroker/postgres => pgbroker}/models/DeadLetterKey.java (77%) create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/DeadLetterRecord.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/Message.java rename es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/{messagebroker => pgbroker}/models/MessageID.java (70%) create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessagePartition.java rename es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/{messagebroker/postgres => pgbroker}/models/MessageRecord.java (54%) rename es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/{messagebroker/postgres => pgbroker}/models/MessageRecordID.java (77%) rename es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/{messagebroker/postgres => pgbroker}/models/MessageRecordQuery.java (80%) create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageState.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageTransaction.java rename es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/{messagebroker/postgres => pgbroker}/models/MessageTransactionID.java (78%) rename es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/{messagebroker/postgres => pgbroker}/models/MessageTransactionQuery.java (82%) create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PartitionKey.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PartitionQuery.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PgBrokerConfiguration.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/RawMessage.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/vertx/VertxConsumerTransaction.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/vertx/VertxMessageProducer.java create mode 100644 es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/queue.xml rename es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/{message-broker.xml => topic.xml} (54%) delete mode 100644 es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskKey.java delete mode 100644 es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskMapper.java delete mode 100644 es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskQuery.java delete mode 100644 es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskRecord.java delete mode 100644 es4j-infrastructure/es4j-task/src/main/resources/task.xml create mode 100644 es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/DeadPayloadConsumer.java rename es4j-test/src/test/java/io/es4j/infrastructure/{taskqueue => messagebroker}/FatalException.java (73%) rename es4j-test/src/test/java/io/es4j/infrastructure/{taskqueue => messagebroker}/MockDeadPayload.java (60%) rename es4j-test/src/test/java/io/es4j/infrastructure/{taskqueue => messagebroker}/MockPayload.java (51%) create mode 100644 es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/MockPayloadConsumer.java create mode 100644 es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/PgTaskQueueTest.java rename es4j-test/src/test/java/io/es4j/infrastructure/{taskqueue => messagebroker}/RecoveryTestPayload.java (53%) delete mode 100644 es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockDeadPayloadProcessor.java delete mode 100644 es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java delete mode 100644 es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/PgTaskQueueTest.java diff --git a/es4j-core/src/main/java/io/es4j/core/CommandHandler.java b/es4j-core/src/main/java/io/es4j/core/CommandHandler.java index 6a3d4fb..8f7ad3a 100644 --- a/es4j-core/src/main/java/io/es4j/core/CommandHandler.java +++ b/es4j-core/src/main/java/io/es4j/core/CommandHandler.java @@ -6,7 +6,6 @@ import io.es4j.core.objects.*; import io.es4j.infrastructure.Infrastructure; import io.es4j.infrastructure.models.*; -import io.es4j.sql.exceptions.Conflict; import io.vertx.core.eventbus.DeliveryOptions; import io.vertx.core.json.JsonArray; import io.vertx.core.tracing.TracingPolicy; @@ -75,10 +74,10 @@ public Uni process(Command command) { private Uni replayAndAppend(Command command) { return replayAggregateAndCache(command.aggregateId(), command.tenant()) .flatMap(aggregateState -> processCommand(aggregateState, command) - .onFailure(Conflict.class).recoverWithUni( + .onFailure(ConcurrentAppend.class).recoverWithUni( () -> playFromLastSnapshot(command.aggregateId(), command.tenant(), aggregateState) .flatMap(reconstructedState -> processCommand(reconstructedState, command)) - .onFailure(Conflict.class).retry().atMost(5) + .onFailure(ConcurrentAppend.class).retry().atMost(5) ) .onFailure().invoke(throwable -> logRejectedCommand(throwable, command, aggregateState)) ) diff --git a/es4j-core/src/main/java/io/es4j/core/tasks/AggregateHeartbeat.java b/es4j-core/src/main/java/io/es4j/core/tasks/AggregateHeartbeat.java index ec98b6e..c6ee2a0 100644 --- a/es4j-core/src/main/java/io/es4j/core/tasks/AggregateHeartbeat.java +++ b/es4j-core/src/main/java/io/es4j/core/tasks/AggregateHeartbeat.java @@ -13,6 +13,7 @@ import java.time.Duration; import java.util.List; +import java.util.Optional; public class AggregateHeartbeat implements TimerTask { @@ -42,8 +43,7 @@ public TimerTaskConfiguration configuration() { Duration.ofSeconds(5), Duration.ofMinutes(1), Duration.ofMinutes(1), - Duration.ofMinutes(1), - List.of() + Optional.empty() ); } diff --git a/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java b/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java index 206f40f..5a4bb4d 100644 --- a/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java +++ b/es4j-core/src/main/java/io/es4j/core/tasks/EventProjectionPoller.java @@ -8,7 +8,6 @@ import io.es4j.infrastructure.OffsetStore; import io.es4j.infrastructure.models.Event; import io.es4j.infrastructure.models.EventStreamBuilder; -import io.es4j.sql.exceptions.NotFound; import io.es4j.task.CronTask; import io.es4j.task.CronTaskConfiguration; import io.es4j.task.CronTaskConfigurationBuilder; @@ -96,7 +95,6 @@ private List parseEvents(List events) { @Override public CronTaskConfiguration configuration() { return CronTaskConfigurationBuilder.builder() - .knownInterruptions(List.of(NotFound.class)) .lockLevel(LockLevel.CLUSTER_WIDE) .cron(asyncProjection.pollingPolicy()) .build(); diff --git a/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java b/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java index 3aa5dee..f10f7bd 100644 --- a/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java +++ b/es4j-core/src/main/java/io/es4j/core/tasks/StateProjectionPoller.java @@ -4,7 +4,6 @@ import io.es4j.infrastructure.EventStore; import io.es4j.infrastructure.OffsetStore; import io.es4j.infrastructure.models.EventStreamBuilder; -import io.es4j.sql.exceptions.NotFound; import io.es4j.task.CronTask; import io.es4j.task.CronTaskConfiguration; import io.es4j.task.CronTaskConfigurationBuilder; @@ -85,7 +84,6 @@ public Uni performTask() { @Override public CronTaskConfiguration configuration() { return CronTaskConfigurationBuilder.builder() - .knownInterruptions(List.of(NotFound.class)) .lockLevel(LockLevel.CLUSTER_WIDE) .cron(stateProjectionWrapper.asyncStateTransfer().pollingPolicy()) .build(); diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/models/ConcurrentAppend.java b/es4j-core/src/main/java/io/es4j/infrastructure/models/ConcurrentAppend.java new file mode 100644 index 0000000..b62c827 --- /dev/null +++ b/es4j-core/src/main/java/io/es4j/infrastructure/models/ConcurrentAppend.java @@ -0,0 +1,8 @@ +package io.es4j.infrastructure.models; + +public class ConcurrentAppend extends RuntimeException { + + public ConcurrentAppend(Throwable throwable) { + super(throwable); + } +} diff --git a/es4j-core/src/main/java/io/es4j/infrastructure/models/EventStoreExeception.java b/es4j-core/src/main/java/io/es4j/infrastructure/models/EventStoreExeception.java new file mode 100644 index 0000000..00e9d16 --- /dev/null +++ b/es4j-core/src/main/java/io/es4j/infrastructure/models/EventStoreExeception.java @@ -0,0 +1,7 @@ +package io.es4j.infrastructure.models; + +public class EventStoreExeception extends RuntimeException{ + public EventStoreExeception(Throwable t) { + super(t); + } +} diff --git a/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java b/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java index 82ae7a8..03d53ce 100644 --- a/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java +++ b/es4j-core/src/main/java/io/es4j/launcher/Es4jMain.java @@ -32,7 +32,7 @@ public class Es4jMain extends AbstractVerticle implements Resource { - protected static final Logger LOGGER = LoggerFactory.getLogger(Es4jMain.class); + private static final Logger LOGGER = LoggerFactory.getLogger(Es4jMain.class); public static final List AGGREGATES = Es4jServiceLoader.bootstrapList(); private static final List> AGGREGATE_DEPLOYERS = new ArrayList<>(); public static final Map, List>> AGGREGATE_COMMANDS = new HashMap<>(); diff --git a/es4j-infrastructure/es4j-config-storage/pom.xml b/es4j-extensions/es4j-config-storage/pom.xml similarity index 97% rename from es4j-infrastructure/es4j-config-storage/pom.xml rename to es4j-extensions/es4j-config-storage/pom.xml index 2cd1e05..816500d 100644 --- a/es4j-infrastructure/es4j-config-storage/pom.xml +++ b/es4j-extensions/es4j-config-storage/pom.xml @@ -3,7 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - es4j-infrastructure + es4j-extensions io.es4j 0 diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/ConfigurationFilter.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/ConfigurationFilter.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/ConfigurationFilter.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/ConfigurationFilter.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/ConfigurationRoute.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/ConfigurationRoute.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/ConfigurationRoute.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/ConfigurationRoute.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfiguration.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfiguration.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfiguration.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfiguration.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationCache.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationCache.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationCache.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationCache.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationFetcher.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationFetcher.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationFetcher.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationFetcher.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationInternalService.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationInternalService.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationInternalService.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationInternalService.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationService.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationService.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationService.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/DatabaseConfigurationService.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/TarGzipHandler.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/TarGzipHandler.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/TarGzipHandler.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/TarGzipHandler.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/exceptions/ConfigStoreError.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/exceptions/ConfigStoreError.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/exceptions/ConfigStoreError.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/exceptions/ConfigStoreError.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/exceptions/ConfigStoreException.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/exceptions/ConfigStoreException.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/exceptions/ConfigStoreException.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/exceptions/ConfigStoreException.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationKey.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationKey.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationKey.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationKey.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationQuery.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationQuery.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationQuery.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationQuery.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationRecord.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationRecord.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationRecord.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationRecord.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationRecordMapper.java b/es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationRecordMapper.java similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationRecordMapper.java rename to es4j-extensions/es4j-config-storage/src/main/java/io/es4j/config/orm/ConfigurationRecordMapper.java diff --git a/es4j-infrastructure/es4j-config-storage/src/main/resources/config.xml b/es4j-extensions/es4j-config-storage/src/main/resources/config.xml similarity index 100% rename from es4j-infrastructure/es4j-config-storage/src/main/resources/config.xml rename to es4j-extensions/es4j-config-storage/src/main/resources/config.xml diff --git a/es4j-extensions/es4j-notifications/pom.xml b/es4j-extensions/es4j-notifications/pom.xml new file mode 100644 index 0000000..6fefecc --- /dev/null +++ b/es4j-extensions/es4j-notifications/pom.xml @@ -0,0 +1,20 @@ + + + 4.0.0 + + io.es4j + es4j-extensions + 0 + + + es4j-notifications + + + 17 + 17 + UTF-8 + + + diff --git a/es4j-extensions/pom.xml b/es4j-extensions/pom.xml new file mode 100644 index 0000000..14c472c --- /dev/null +++ b/es4j-extensions/pom.xml @@ -0,0 +1,28 @@ + + + 4.0.0 + + io.es4j + es4j + 0 + + + pom + + es4j-notifications + es4j-config-storage + + + es4j-extensions + + + + + 17 + 17 + UTF-8 + + + diff --git a/es4j-infrastructure/es4j-postgres-message-broker/pom.xml b/es4j-infrastructure/es4j-postgres-message-broker/pom.xml index ac9d992..35932ed 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/pom.xml +++ b/es4j-infrastructure/es4j-postgres-message-broker/pom.xml @@ -19,6 +19,11 @@ + + io.es4j + es4j-core + ${project.version} + io.es4j es4j-sql diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProducer.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProducer.java deleted file mode 100644 index 8256394..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProducer.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.es4j.infrastructure.messagebroker; - - -import io.es4j.infrastructure.messagebroker.models.Message; -import io.es4j.infrastructure.messagebroker.models.MessageID; -import io.es4j.infrastructure.messagebroker.models.QueueTransaction; -import io.smallrye.mutiny.Uni; - -import java.util.List; - -public interface MessageProducer { - Uni enqueue(Message message, QueueTransaction transaction); - Uni enqueue(List> entries, QueueTransaction queueTransaction); - Uni cancel(MessageID messageID); -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/ProcessorTransactionProvider.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/ProcessorTransactionProvider.java deleted file mode 100644 index a55602c..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/ProcessorTransactionProvider.java +++ /dev/null @@ -1,13 +0,0 @@ -package io.es4j.infrastructure.messagebroker; - - -import io.es4j.infrastructure.messagebroker.models.Message; -import io.es4j.infrastructure.messagebroker.models.QueueTransaction; -import io.smallrye.mutiny.Uni; - -import java.util.function.BiFunction; - -public interface ProcessorTransactionProvider { - - Uni transaction(Message message, BiFunction, QueueTransaction, Uni> function); -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/QueueSubscriber.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/QueueSubscriber.java deleted file mode 100644 index 0456e20..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/QueueSubscriber.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.es4j.infrastructure.messagebroker; - - -import io.es4j.infrastructure.messagebroker.models.MessageProcessorManager; -import io.smallrye.mutiny.Uni; - -public interface QueueSubscriber { - - Uni unsubscribe(); - - Uni subscribe(MessageProcessorManager messageProcessorManager); - - - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/TopicSubscriber.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/TopicSubscriber.java deleted file mode 100644 index 5805211..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/TopicSubscriber.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.es4j.infrastructure.messagebroker; - - -import io.es4j.infrastructure.messagebroker.models.MessageProcessorManager; -import io.smallrye.mutiny.Uni; - -public interface TopicSubscriber { - - Uni unsubscribe(); - - Uni subscribe(MessageProcessorManager messageProcessorManager); - - - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/ConsumerException.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/ConsumerException.java deleted file mode 100644 index db5c706..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/ConsumerException.java +++ /dev/null @@ -1,26 +0,0 @@ -package io.es4j.infrastructure.messagebroker.exceptions; - - -public class ConsumerException extends QueueException { - - public ConsumerException(QueueError eventxError) { - super(eventxError); - } - - public ConsumerException(Throwable throwable) { - super(throwable); - } - - public ConsumerException(QueueError eventxError, Throwable throwable) { - super(eventxError, throwable); - } - - public ConsumerException(String cause, String hint, Integer errorCode) { - super(cause, hint, errorCode); - } - - public ConsumerException(String cause, String hint, Integer errorCode, Throwable throwable) { - super(cause, hint, errorCode, throwable); - } - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/MessageException.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/MessageException.java deleted file mode 100644 index 4cab098..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/MessageException.java +++ /dev/null @@ -1,11 +0,0 @@ -package io.es4j.infrastructure.messagebroker.exceptions; - - - -public class MessageException extends QueueException { - - public MessageException(QueueError eventxError) { - super(eventxError); - } - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueError.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueError.java deleted file mode 100644 index a3192f0..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueError.java +++ /dev/null @@ -1,8 +0,0 @@ -package io.es4j.infrastructure.messagebroker.exceptions; - -public record QueueError( - String cause, - String hint, - Integer internalCode -) { -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueException.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueException.java deleted file mode 100644 index f4b4fd1..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/exceptions/QueueException.java +++ /dev/null @@ -1,39 +0,0 @@ -package io.es4j.infrastructure.messagebroker.exceptions; - -import io.vertx.core.json.JsonObject; - -public class QueueException extends RuntimeException{ - private final QueueError queueError; - - public QueueException(QueueError queueError) { - super(JsonObject.mapFrom(queueError).encodePrettily()); - this.queueError = queueError; - } - - public QueueException(Throwable throwable) { - super(throwable.getMessage(), throwable); - this.queueError = new QueueError( - throwable.getMessage(), - throwable.getLocalizedMessage(), - 500 - ); - } - public QueueException(QueueError queueError, Throwable throwable) { - super(queueError.cause(), throwable); - this.queueError = queueError; - } - - public QueueException(String cause, String hint, Integer errorCode) { - this.queueError = new QueueError(cause, hint, errorCode); - } - - public QueueException(String cause, String hint, Integer errorCode, Throwable throwable) { - super(cause, throwable); - this.queueError = new QueueError(cause, hint, errorCode); - } - - public QueueError error() { - return queueError; - } - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/Message.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/Message.java deleted file mode 100644 index 540ee84..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/Message.java +++ /dev/null @@ -1,26 +0,0 @@ -package io.es4j.infrastructure.messagebroker.models; - - -import io.soabase.recordbuilder.core.RecordBuilder; - -import java.time.Instant; - -@RecordBuilder -public record Message( - String messageId, - String tenant, - Instant scheduled, - Instant expiration, - Integer priority, - T payload -) { - - public Message { - if (priority != null && priority > 10) { - throw new IllegalArgumentException("Max priority is 10"); - } - if (messageId == null) { - throw new IllegalArgumentException("Id must not be null"); - } - } -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorManager.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorManager.java deleted file mode 100644 index cb33a87..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorManager.java +++ /dev/null @@ -1,108 +0,0 @@ -package io.es4j.infrastructure.messagebroker.models; - -import io.es4j.infrastructure.messagebroker.exceptions.ConsumerException; -import io.es4j.sql.exceptions.IntegrityContraintViolation; -import io.smallrye.mutiny.Uni; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import io.vertx.core.json.JsonObject; -import io.vertx.mutiny.core.Vertx; -import io.es4j.infrastructure.messagebroker.MessageProcessor; -import io.es4j.infrastructure.messagebroker.ProcessorTransactionProvider; - -import java.util.List; - - -public record MessageProcessorManager( - QueueConfiguration queueConfiguration, - List processorWrappers, - ProcessorTransactionProvider processorTransactionProvider, - Vertx vertx -) { - private static final Logger LOGGER = LoggerFactory.getLogger(MessageProcessorManager.class); - public Uni processMessage(RawMessage rawMessage) { - final var processor = resolveProcessor(rawMessage); - final var parsedMessage = parseMessage(rawMessage); - return processorTransactionProvider.transaction( - parsedMessage, (msg, taskTransaction) -> { - try { - if (processor.blockingProcessor()) { - return vertx.executeBlocking(process(parsedMessage, processor, taskTransaction) - .onFailure().transform(ConsumerException::new) - ); - } - return process(parsedMessage, processor, taskTransaction) - .onFailure().transform(ConsumerException::new); - } catch (Exception exception) { - throw new ConsumerException(exception); - } - } - ) - .onItemOrFailure().transform( - (avoid, throwable) -> { - if (throwable != null && !(throwable instanceof IntegrityContraintViolation)) { - return retryableFailure(queueConfiguration, rawMessage, throwable, processor); - } else { - return rawMessage.withState(MessageState.PROCESSED); - } - } - ); - } - - private Uni process(Message message, MessageProcessor processor, QueueTransaction queueTransaction) { - return processor.process(message.payload(), queueTransaction); - } - - private MessageProcessor resolveProcessor(RawMessage messageRecord) { - return processorWrappers.stream() - .filter(processor -> processor.doesMessageMatch(messageRecord)) - .findFirst() - .map(processor -> processor.resolveProcessor(messageRecord.tenant())) - .orElseThrow(); - } - - private Message parseMessage(RawMessage rawMessage) { - final Class tClass; - try { - tClass = Class.forName(rawMessage.payloadClass()); - return new Message<>( - rawMessage.id(), - rawMessage.tenant(), - rawMessage.scheduled(), - rawMessage.expiration(), - rawMessage.priority(), - rawMessage.payload().mapTo(tClass) - ); - } catch (ClassNotFoundException e) { - LOGGER.error("Unable to parse message {}" ,JsonObject.mapFrom(rawMessage), e); - throw new ConsumerException(e); - } - } - - private RawMessage retryableFailure(QueueConfiguration configuration, RawMessage messageRecord, Throwable throwable, MessageProcessor processor) { - MessageState failureState; - if (processor.fatalExceptions().stream().anyMatch(f -> f.getName().equals(throwable.getCause().getClass().getName()))) { - LOGGER.error("Fatal failure for message {} in processor {}", JsonObject.mapFrom(messageRecord).encodePrettily(), processor.getClass().getName(), throwable.getCause()); - failureState = MessageState.FATAL_FAILURE; - } else if (configuration.maxRetry() != null && messageRecord.retryCounter() + 1 > configuration.maxRetry()) { - LOGGER.error("Retries exhausted for message {} in processor {}", JsonObject.mapFrom(messageRecord).encodePrettily(), processor.getClass().getName(), throwable.getCause()); - failureState = MessageState.RETRIES_EXHAUSTED; - } else { - LOGGER.error("Failure for message {} in processor {}", processor.getClass().getName(), JsonObject.mapFrom(messageRecord).encodePrettily(), throwable.getCause()); - failureState = MessageState.RETRY; - } - return new RawMessage( - messageRecord.id(), - messageRecord.scheduled(), - messageRecord.expiration(), - messageRecord.priority(), - messageRecord.retryCounter() + 1, - failureState, - messageRecord.payloadClass(), - messageRecord.payload(), - new JsonObject().put(processor.getClass().getName(), throwable.getMessage()), - messageRecord.tenant() - ); - } - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageState.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageState.java deleted file mode 100644 index 4ab1545..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageState.java +++ /dev/null @@ -1,5 +0,0 @@ -package io.es4j.infrastructure.messagebroker.models; - -public enum MessageState { - CREATED, PROCESSING, SCHEDULED, EXPIRED, RETRY, RETRIES_EXHAUSTED, RECOVERY, PROCESSED, FATAL_FAILURE -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueConfiguration.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueConfiguration.java deleted file mode 100644 index 1df349f..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueConfiguration.java +++ /dev/null @@ -1,163 +0,0 @@ -package io.es4j.infrastructure.messagebroker.models; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; - -@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY) -public class QueueConfiguration { - - private Boolean bootstrapQueue = true; - private Boolean idempotency = false; - private Integer idempotencyNumberOfDays = 2; - private QueueImplementation queueImplementation = QueueImplementation.PG_QUEUE; - private TransactionProvider transactionProvider = TransactionProvider.VERTX_PG_CLIENT; - private Long emptyBackOffInMinutes = 5L; - private Long throttleInMs = 100L; - private Integer concurrency = null; - private Long errorBackOffInMinutes = 5L; - private Long retryInvervalInSeconds = 1L; - private Long maxProcessingTimeInMinutes = 30L; - private Long batchSize = 50L; - private Integer maxRetry = null; - private Long maintenanceEvery = 30L; - private Integer circuitBreakerMaxFailues = 10; - - public TransactionProvider transactionManagerImplementation() { - return transactionProvider; - } - - public QueueConfiguration setTransactionManagerImplementation(TransactionProvider transactionProvider) { - this.transactionProvider = transactionProvider; - return this; - } - - public Boolean idempotentProcessors() { - return idempotency; - } - - public Integer circuitBreakerMaxFailues() { - return circuitBreakerMaxFailues; - } - - public QueueConfiguration setCircuitBreakerMaxFailuer(Integer circuitBreakerMaxFailuer) { - this.circuitBreakerMaxFailues = circuitBreakerMaxFailuer; - return this; - } - - public Boolean bootstrapQueue() { - return bootstrapQueue; - } - - public QueueConfiguration setBootstrapQueue(Boolean bootstrapQueue) { - this.bootstrapQueue = bootstrapQueue; - return this; - } - - public Integer concurrency() { - return concurrency; - } - - public QueueConfiguration setConcurrency(Integer concurrency) { - this.concurrency = concurrency; - return this; - } - - public Long maintenanceEvery() { - return maintenanceEvery; - } - - public QueueConfiguration setMaintenanceEvery(Long maintenanceEvery) { - this.maintenanceEvery = maintenanceEvery; - return this; - } - - public Boolean idempotentProcess() { - return idempotency; - } - - public QueueConfiguration setIdempotency(Boolean idempotency) { - this.idempotency = idempotency; - return this; - } - - public Integer idempotencyNumberOfDays() { - return idempotencyNumberOfDays; - } - - public QueueConfiguration setIdempotencyNumberOfDays(Integer idempotencyNumberOfDays) { - this.idempotencyNumberOfDays = idempotencyNumberOfDays; - return this; - } - public QueueImplementation queueImplementation() { - return queueImplementation; - } - - public QueueConfiguration setQueueType(QueueImplementation queueImplementation) { - this.queueImplementation = queueImplementation; - return this; - } - - public Long emptyBackOffInMinutes() { - return emptyBackOffInMinutes; - } - - public QueueConfiguration setEmptyBackOffInMinutes(Long emptyBackOffInMinutes) { - this.emptyBackOffInMinutes = emptyBackOffInMinutes; - return this; - } - - public Long throttleInMs() { - return throttleInMs; - } - - public QueueConfiguration setThrottleInMs(Long throttleInMs) { - this.throttleInMs = throttleInMs; - return this; - } - - public Long errorBackOffInMinutes() { - return errorBackOffInMinutes; - } - - public QueueConfiguration setErrorBackOffInMinutes(Long errorBackOffInMinutes) { - this.errorBackOffInMinutes = errorBackOffInMinutes; - return this; - } - - public Long retryIntervalInSeconds() { - return retryInvervalInSeconds; - } - - public QueueConfiguration setRetryInvervalInSeconds(Long retryInvervalInSeconds) { - this.retryInvervalInSeconds = retryInvervalInSeconds; - return this; - } - - public Long maxProcessingTimeInMinutes() { - return maxProcessingTimeInMinutes; - } - - public QueueConfiguration setMaxProcessingTimeInMinutes(Long maxProcessingTimeInMinutes) { - this.maxProcessingTimeInMinutes = maxProcessingTimeInMinutes; - return this; - } - - public Long batchSize() { - return batchSize; - } - - public QueueConfiguration setBatchSize(Long batchSize) { - this.batchSize = batchSize; - return this; - } - - public Integer maxRetry() { - return maxRetry; - } - - public QueueConfiguration setMaxRetry(Integer maxRetry) { - this.maxRetry = maxRetry; - return this; - } - - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueImplementation.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueImplementation.java deleted file mode 100644 index 8972b82..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueImplementation.java +++ /dev/null @@ -1,6 +0,0 @@ -package io.es4j.infrastructure.messagebroker.models; - -public enum QueueImplementation { - - RABBITMQ, PG_QUEUE, SOLACE -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueTransaction.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueTransaction.java deleted file mode 100644 index 1cd96e8..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/QueueTransaction.java +++ /dev/null @@ -1,12 +0,0 @@ -package io.es4j.infrastructure.messagebroker.models; - -public record QueueTransaction( - Object connection -) { - - - public T cast(Class target) { - return target.cast(connection); - } - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/RawMessage.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/RawMessage.java deleted file mode 100644 index e9e88d9..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/RawMessage.java +++ /dev/null @@ -1,36 +0,0 @@ -package io.es4j.infrastructure.messagebroker.models; - -import io.soabase.recordbuilder.core.RecordBuilder; -import io.vertx.core.json.JsonObject; - -import java.time.Instant; - -@RecordBuilder -public record RawMessage( - String id, - Instant scheduled, - Instant expiration, - Integer priority, - Integer retryCounter, - MessageState messageState, - String payloadClass, - JsonObject payload, - JsonObject failures, - String tenant -) { - - public RawMessage withState(MessageState messageState) { - return new RawMessage( - id, - scheduled, - expiration, - priority, - retryCounter, - messageState, - payloadClass, - payload, - failures, - tenant - ); - } -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/TransactionProvider.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/TransactionProvider.java deleted file mode 100644 index 9abe054..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/TransactionProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package io.es4j.infrastructure.messagebroker.models; - -public enum TransactionProvider { - - VERTX_PG_CLIENT -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgMessageProducer.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgMessageProducer.java deleted file mode 100644 index 1dc05dc..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgMessageProducer.java +++ /dev/null @@ -1,131 +0,0 @@ -package io.es4j.infrastructure.messagebroker.postgres; - -import io.es4j.infrastructure.messagebroker.models.Message; -import io.es4j.infrastructure.messagebroker.models.MessageID; -import io.es4j.infrastructure.messagebroker.models.MessageState; -import io.es4j.infrastructure.messagebroker.models.QueueTransaction; -import io.es4j.infrastructure.messagebroker.postgres.mappers.MessageQueueMapper; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecord; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordID; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordQuery; -import io.es4j.sql.Repository; -import io.es4j.sql.RepositoryHandler; -import io.es4j.sql.models.BaseRecord; -import io.vertx.mutiny.sqlclient.SqlConnection; -import io.smallrye.mutiny.Uni; -import io.vertx.core.json.JsonObject; -import io.es4j.infrastructure.messagebroker.MessageProducer; - - -import java.util.List; -import java.util.function.Consumer; - -public class PgMessageProducer implements MessageProducer { - private final Repository queue; - - public PgMessageProducer(RepositoryHandler repositoryHandler) { - this.queue = new Repository<>(MessageQueueMapper.INSTANCE, repositoryHandler); - } - - public Uni enqueue(Message message, QueueTransaction queueTransaction) { - final var queueEntry = new MessageRecord( - message.messageId(), - message.scheduled(), - message.expiration(), - message.priority(), - 0, - MessageState.CREATED, - message.payload().getClass().getName(), - JsonObject.mapFrom(message.payload()), - null, - null, - BaseRecord.newRecord(message.tenant()) - ); - return queue.insert(queueEntry, (SqlConnection) queueTransaction.connection()).replaceWithVoid(); - } - - public Uni enqueue(List> entries) { - final var queueEntries = entries.stream().map( - message -> new MessageRecord( - message.messageId(), - message.scheduled(), - message.expiration(), - message.priority(), - 0, - MessageState.CREATED, - message.payload().getClass().getName(), - JsonObject.mapFrom(message.payload()), - null, - null, - BaseRecord.newRecord(message.tenant()) - ) - ).toList(); - return queue.insertBatch(queueEntries).replaceWithVoid(); - } - - public Uni enqueue(List> entries, QueueTransaction queueTransaction) { - final var queueEntries = entries.stream().map( - message -> new MessageRecord( - message.messageId(), - message.scheduled(), - message.expiration(), - message.priority(), - 0, - MessageState.CREATED, - message.payload().getClass().getName(), - JsonObject.mapFrom(message.payload()), - null, - null, - BaseRecord.newRecord(message.tenant()) - ) - ).toList(); - return queue.insertBatch(queueEntries, (SqlConnection) queueTransaction.connection()).replaceWithVoid(); - } - - public Uni cancel(MessageID messageID) { - return queue.deleteByKey(new MessageRecordID(messageID.id(), messageID.tenant())); - } - - public Uni> get(MessageID messageID, Class tClass) { - return queue.selectByKey(new MessageRecordID(messageID.id(), messageID.tenant())) - .map(entry -> new Message<>( - entry.id(), - entry.baseRecord().tenant(), - entry.scheduled(), - entry.expiration(), - entry.priority(), - entry.payload().mapTo(tClass) - ) - ); - } - - public Uni>> query(MessageRecordQuery query, Class tClass) { - return queue.query(query) - .map(entries -> entries.stream() - .map(entry -> new Message<>( - entry.id(), - entry.baseRecord().tenant(), - entry.scheduled(), - entry.expiration(), - entry.priority(), - entry.payload().mapTo(tClass) - )).toList() - ); - } - public Uni stream(MessageRecordQuery query, Class tClass, Consumer> messageConsumer) { - return queue.stream(entry -> { - final var msg = new Message<>( - entry.id(), - entry.baseRecord().tenant(), - entry.scheduled(), - entry.expiration(), - entry.priority(), - entry.payload().mapTo(tClass) - ); - messageConsumer.accept(msg); - }, - query - ); - } - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgProcessorTransaction.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgProcessorTransaction.java deleted file mode 100644 index fc95c89..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgProcessorTransaction.java +++ /dev/null @@ -1,43 +0,0 @@ -package io.es4j.infrastructure.messagebroker.postgres; - - -import io.es4j.infrastructure.messagebroker.postgres.mappers.MessageTransactionMapper; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransaction; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransactionID; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransactionQuery; -import io.es4j.sql.Repository; -import io.es4j.sql.RepositoryHandler; -import io.es4j.sql.models.BaseRecord; -import io.smallrye.mutiny.Uni; -import io.es4j.infrastructure.messagebroker.ProcessorTransactionProvider; -import io.es4j.infrastructure.messagebroker.models.Message; -import io.es4j.infrastructure.messagebroker.models.QueueTransaction; -import io.vertx.core.json.JsonObject; -import io.vertx.mutiny.core.Vertx; - -import java.util.function.BiFunction; - -public class PgProcessorTransaction implements ProcessorTransactionProvider { - private final Repository transactionStore; - - public PgProcessorTransaction(Vertx vertx, JsonObject configuration) { - this.transactionStore = new Repository<>(MessageTransactionMapper.INSTANCE, RepositoryHandler.leasePool(configuration, vertx)); - } - - - @Override - public Uni transaction(Message message, BiFunction, QueueTransaction, Uni> function) { - return transactionStore.transaction( - sqlConnection -> transactionStore.insert( - new MessageTransaction( - message.messageId(), - null, - message.payload().getClass().getName(), - BaseRecord.newRecord(message.tenant()) - ), - sqlConnection - ) - .flatMap(avoid -> function.apply(message, new QueueTransaction(sqlConnection))) - ); - } -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgQueueSubscriber.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgQueueSubscriber.java deleted file mode 100644 index 8e1d8e4..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgQueueSubscriber.java +++ /dev/null @@ -1,231 +0,0 @@ -package io.es4j.infrastructure.messagebroker.postgres; - - -import io.es4j.infrastructure.messagebroker.postgres.mappers.DeadLetterMapper; -import io.es4j.infrastructure.messagebroker.postgres.mappers.MessageQueueMapper; -import io.es4j.infrastructure.messagebroker.postgres.mappers.PgQueueLiquibase; -import io.es4j.infrastructure.messagebroker.postgres.models.*; -import io.es4j.sql.Repository; -import io.es4j.sql.RepositoryHandler; -import io.es4j.sql.exceptions.NotFound; -import io.es4j.sql.models.BaseRecord; -import io.es4j.sql.models.QueryOptions; -import io.smallrye.mutiny.Multi; -import io.smallrye.mutiny.subscription.FixedDemandPacer; -import io.es4j.infrastructure.messagebroker.models.MessageState; -import io.es4j.infrastructure.messagebroker.models.QueueConfiguration; -import io.es4j.infrastructure.messagebroker.models.RawMessage; -import io.es4j.infrastructure.messagebroker.models.MessageProcessorManager; -import io.smallrye.mutiny.Uni; - -import io.vertx.core.impl.NoStackTraceThrowable; -import io.vertx.core.json.JsonObject; -import io.vertx.mutiny.core.Vertx; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import io.es4j.infrastructure.messagebroker.QueueSubscriber; - -import java.time.Duration; -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; - -import static java.util.stream.Collectors.groupingBy; - -public class PgQueueSubscriber implements QueueSubscriber { - - public static final AtomicBoolean LIQUIBASE_DEPLOYED = new AtomicBoolean(false); - private final Repository messageQueue; - private final Repository deadLetterQueue; - private final io.vertx.mutiny.pgclient.pubsub.PgSubscriber pgSubscriber; - private static final Logger LOGGER = LoggerFactory.getLogger(PgQueueSubscriber.class); - - public PgQueueSubscriber(Vertx vertx, JsonObject configuration) { - final var repositoryHandler = RepositoryHandler.leasePool(configuration, vertx); - this.messageQueue = new Repository<>(MessageQueueMapper.INSTANCE, repositoryHandler); - this.deadLetterQueue = new Repository<>(DeadLetterMapper.INSTANCE, repositoryHandler); - this.pgSubscriber = io.vertx.mutiny.pgclient.pubsub.PgSubscriber.subscriber( - repositoryHandler.vertx(), - RepositoryHandler.connectionOptions(repositoryHandler.configuration()) - ); - pgSubscriber.reconnectPolicy(integer -> 0L); - } - - - @Override - public Uni unsubscribe() { - return pgSubscriber.close(); - } - - @Override - public Uni subscribe(MessageProcessorManager messageProcessorManager) { - // todo put schema in channel - final var pgChannel = pgSubscriber.channel("task_queue_ch"); - pgChannel.handler(payload -> { - pgChannel.pause(); - LOGGER.info("Message available {}", payload); - poll(messageProcessorManager, null) - .subscribe() - .with( - item -> { - LOGGER.info("Queue empty, resuming subscription"); - pgChannel.resume(); - }, - throwable -> { - if (throwable instanceof NoStackTraceThrowable illegalStateException) { - LOGGER.info(illegalStateException.getMessage()); - } else if (throwable instanceof NotFound) { - LOGGER.info("Queue empty"); - } else { - LOGGER.error("PgSubscriber dropped exception", throwable); - } - pgChannel.resume(); - } - ); - } - ) - .endHandler(() -> LOGGER.info("pg-channel subscription stopped")) - .subscribeHandler(() -> LOGGER.info("subscribed to pg-channel")) - .exceptionHandler(throwable -> LOGGER.error("Error in pg-subscription", throwable)); - return PgQueueLiquibase.bootstrapQueue(messageQueue.repositoryHandler(), messageProcessorManager.queueConfiguration()) - .flatMap(avoid -> pgSubscriber.connect()); - } - - private Multi startPacedStream(QueueConfiguration queueConfiguration, List messageRecords) { - if (queueConfiguration.concurrency() != null) { - final var pacer = new FixedDemandPacer( - queueConfiguration.concurrency(), - Duration.ofMillis(queueConfiguration.throttleInMs()) - ); - return Multi.createFrom().iterable(messageRecords) - .paceDemand().using(pacer); - } - return Multi.createFrom().iterable(messageRecords); - } - - private Uni poll(MessageProcessorManager taskManager, String verticleId) { - return pollBatch(taskManager.queueConfiguration(), verticleId) - .onItem().transformToMulti(messageRecords -> startPacedStream(taskManager.queueConfiguration(), messageRecords)) - .onItem().transformToUniAndMerge(messageRecord -> taskManager.processMessage(parseRecord(messageRecord))) - .collect().asList() - .flatMap(rawMessages -> handleResults(rawMessages.stream().map(MessageRecord::from).toList())) - .replaceWithVoid() - .flatMap(avoid -> poll(taskManager, verticleId)); - } - - private RawMessage parseRecord(MessageRecord messageRecord) { - return new RawMessage( - messageRecord.id(), - messageRecord.scheduled(), - messageRecord.expiration(), - messageRecord.priority(), - messageRecord.retryCounter(), - messageRecord.messageState(), - messageRecord.payloadClass(), - messageRecord.payload(), - messageRecord.failedProcessors(), - messageRecord.baseRecord().tenant() - ); - } - - private Uni> pollBatch(QueueConfiguration configuration, String deploymentId) { - return messageQueue.query(pollingStatement(configuration, deploymentId)).onFailure(NotFound.class) - .recoverWithUni( - () -> messageQueue.query(recoveryPollingStatement(configuration, deploymentId)) - .map(messageRecords -> messageRecords.stream().map(m -> m.withState(MessageState.RECOVERY)).toList()) - ); - } - - private String pollingStatement( - final QueueConfiguration configuration, - String deploymentId - ) { - return "update task_queue set state = 'PROCESSING', verticle_id = '" + deploymentId + "' where message_id in (" + - " select message_id from task_queue where " + - " state in ('CREATED','SCHEDULED','RETRY')" + - " and (scheduled is null or scheduled <= current_timestamp)" + - " and (expiration is null or expiration >= current_timestamp)" + - " and (retry_counter = 0 or updated + interval '" + configuration.retryIntervalInSeconds() + " seconds' <= current_timestamp)" + - " order by priority for update skip locked limit " + configuration.batchSize() + - " ) returning *;"; - } - - private String recoveryPollingStatement( - final QueueConfiguration configuration, - final String deploymentId - ) { - return "update task_queue set state = 'PROCESSING', verticle_id = '" + deploymentId + "' where message_id in (" + - " select message_id from task_queue where " + - " state = 'RECOVERY' " + - " order by priority for update skip locked limit " + configuration.batchSize() + - " ) returning *;"; - } - - private Uni handleResults(List messages) { - return Uni.join().all(ack(messages), nack(messages)).andFailFast().replaceWithVoid(); - } - - private Uni nack(List messages) { - return Uni.join().all(requeueMessages(messages)).andFailFast().replaceWithVoid(); - } - - private Uni requeueMessages(List messages) { - final var messagesToRequeue = messages.stream() - .filter(entry -> entry.messageState() == MessageState.RETRY) - .toList(); - if (!messagesToRequeue.isEmpty()) { - LOGGER.info("re-queuing unhandled messages ->" + messagesToRequeue.stream().map(MessageRecord::id).toList()); - return messageQueue.updateByKeyBatch(messagesToRequeue); - } - return Uni.createFrom().voidItem(); - } - - private Uni ack(List messages) { - // todo move messages to dead-letter-queue - final var messagesToAckOrNack = messages.stream() - .filter(message -> message.messageState() == MessageState.PROCESSED || - message.messageState() == MessageState.FATAL_FAILURE || - message.messageState() == MessageState.RETRIES_EXHAUSTED || - message.messageState() == MessageState.EXPIRED - ) - .collect(groupingBy(q -> q.baseRecord().tenant())); - final var queries = messagesToAckOrNack.entrySet().stream() - .map(this::messageDropQuery) - .toList(); - final var deadLetters = messages.stream() - .filter(message -> - message.messageState() == MessageState.FATAL_FAILURE || - message.messageState() == MessageState.RETRIES_EXHAUSTED || message.messageState() == MessageState.EXPIRED - ) - .map(messageRecord -> new DeadLetterRecord( - messageRecord.id(), - messageRecord.scheduled(), - messageRecord.expiration(), - messageRecord.priority(), - messageRecord.retryCounter(), - messageRecord.messageState(), - messageRecord.payloadClass(), - messageRecord.payload(), - messageRecord.failedProcessors(), - messageRecord.verticleId(), - BaseRecord.newRecord(messageRecord.baseRecord().tenant()) - )) - .toList(); - if (!queries.isEmpty()) { - return Multi.createFrom().iterable(queries) - .onItem().transformToUniAndMerge(messageQueue::deleteQuery) - .collect().asList() - .flatMap(avoid -> deadLetters.isEmpty() ? Uni.createFrom().voidItem() : deadLetterQueue.insertBatch(deadLetters)) - .replaceWithVoid(); - } - return deadLetters.isEmpty() ? Uni.createFrom().voidItem() : deadLetterQueue.insertBatch(deadLetters); - } - - - private MessageRecordQuery messageDropQuery(Map.Entry> entry) { - return MessageRecordQueryBuilder.builder() - .ids(entry.getValue().stream().map(MessageRecord::id).toList()) - .options(QueryOptions.simple(entry.getKey())) - .build(); - } - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgRefresher.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgRefresher.java deleted file mode 100644 index 57429ba..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/PgRefresher.java +++ /dev/null @@ -1,110 +0,0 @@ -package io.es4j.infrastructure.messagebroker.postgres; - -import io.es4j.sql.RepositoryHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import io.es4j.infrastructure.messagebroker.models.QueueConfiguration; - -import java.util.concurrent.atomic.AtomicLong; - - -public class PgRefresher { - public final AtomicLong timer = new AtomicLong(); - private final RepositoryHandler repositoryHandler; - private final QueueConfiguration configuration; - private static final Logger logger = LoggerFactory.getLogger(PgRefresher.class); - - public PgRefresher( - final RepositoryHandler repositoryHandler, - final QueueConfiguration configuration - ) { - this.repositoryHandler = repositoryHandler; - this.configuration = configuration; - } - - public PgRefresher startRetryTimer(final long throttle) { - repositoryHandler.vertx().setTimer( - throttle, - delay -> { - logger.info("Running retry refresh timer"); - repositoryHandler.sqlClient().query(retryUpdates(configuration)).execute() - .subscribe() - .with( - avoid -> { - logger.info("Retry refresh will re-run in " + configuration.retryIntervalInSeconds() + " minutes"); - startRetryTimer(configuration.retryIntervalInSeconds() * 10000); - }, - throwable -> { - logger.info("Retry refresh will re-run in " + configuration.retryIntervalInSeconds() + " minutes"); - startRetryTimer(configuration.retryIntervalInSeconds() * 10000); - } - ); - } - ); - return this; - } - - public PgRefresher startRecoveryTimer(final long throttle) { - repositoryHandler.vertx().setTimer( - throttle, - delay -> { - logger.info("Running recovery refresh timer"); - repositoryHandler.sqlClient().query(recoveryUpdates(configuration)).execute() - .subscribe() - .with( - avoid -> { - logger.info("Recovery refresh will re-run in " + configuration.maxProcessingTimeInMinutes() + " minutes"); - startRecoveryTimer(configuration.retryIntervalInSeconds() * 60000); - }, - throwable -> { - logger.info("Recovery refresh will re-run in " + configuration.maxProcessingTimeInMinutes() + " minutes"); - startRecoveryTimer(configuration.retryIntervalInSeconds() * 60000); - } - ); - } - ); - return this; - } - - public PgRefresher startPurgeRefreshTimer(final long throttle) { - if (configuration.idempotentProcessors()) { - repositoryHandler.vertx().setTimer( - throttle, - delay -> { - logger.info("Running purge refresh timer"); - repositoryHandler.sqlClient().query(purgeIdempotency(configuration)).execute() - .subscribe() - .with( - avoid -> { - logger.info("Recovery refresh will re-run in " + configuration.maintenanceEvery() + " minutes"); - startPurgeRefreshTimer(configuration.retryIntervalInSeconds() * 1000); - }, - throwable -> { - logger.info("Recovery refresh will re-run in " + configuration.maintenanceEvery() + " minutes"); - startPurgeRefreshTimer(configuration.retryIntervalInSeconds() * 1000); - } - ); - } - ); - } - return this; - } - - - private static String purgeIdempotency(QueueConfiguration configuration) { - return "delete from task_queue_tx where inserted <= current_timestamp - interval '" + configuration.idempotencyNumberOfDays() + " days'"; - } - // todo instead of purging should move to another database. - - private static String recoveryUpdates(QueueConfiguration configuration) { - return "update task_queue set rec_version = version + 1, state = 'RECOVERY' where " + - " state = 'PROCESSING' and updated + interval '" + configuration.maxProcessingTimeInMinutes() + "minutes' <= current_timestamp;"; - } - - private static String retryUpdates(QueueConfiguration configuration) { - return "update task_queue set rec_version = rec_version + 1 where " + - " state = 'RETRY' and updated + interval '" + configuration.retryIntervalInSeconds() + "seconds' <= current_timestamp;"; - } - - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/DeadLetterMapper.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/DeadLetterMapper.java deleted file mode 100644 index b230f81..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/DeadLetterMapper.java +++ /dev/null @@ -1,140 +0,0 @@ -package io.es4j.infrastructure.messagebroker.postgres.mappers; - - -import io.es4j.sql.RecordMapper; -import io.es4j.sql.generator.filters.QueryBuilder; -import io.es4j.sql.models.QueryFilter; -import io.es4j.sql.models.QueryFilters; -import io.es4j.infrastructure.messagebroker.models.MessageState; -import io.es4j.infrastructure.messagebroker.postgres.models.DeadLetterKey; -import io.es4j.infrastructure.messagebroker.postgres.models.DeadLetterRecord; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordQuery; -import io.vertx.sqlclient.Row; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.Map; -import java.util.Set; - - -public class DeadLetterMapper implements RecordMapper { - private static final String MESSAGE_ID = "message_id"; - private static final String SCHEDULED = "scheduled"; - private static final String EXPIRATION = "expiration"; - private static final String PRIORITY = "priority"; - private static final String RETRY_COUNTER = "retry_counter"; - private static final String STATE = "state"; - private static final String PAYLOAD = "payload"; - private static final String FAILURES = "failure"; - private static final String VERTICLE_ID = "verticle_id"; - private static final String PAYLOAD_CLASS = "payload_class"; - public static final String JOB_QUEUE = "task_queue_dead_letter"; - - public static DeadLetterMapper INSTANCE = new DeadLetterMapper(); - private DeadLetterMapper(){} - - @Override - public String table() { - return JOB_QUEUE; - } - - @Override - public Set columns() { - return Set.of(MESSAGE_ID, SCHEDULED, EXPIRATION, PRIORITY, RETRY_COUNTER, STATE, PAYLOAD, FAILURES, VERTICLE_ID); - - } - - - @Override - public Set keyColumns() { - return Set.of(MESSAGE_ID); - } - - @Override - public DeadLetterRecord rowMapper(Row row) { - return new DeadLetterRecord( - row.getString(MESSAGE_ID), - row.getLocalDateTime(SCHEDULED) != null ? row.getLocalDateTime(SCHEDULED).toInstant(ZoneOffset.UTC) : null, - row.getLocalDateTime(EXPIRATION) != null ? row.getLocalDateTime(EXPIRATION).toInstant(ZoneOffset.UTC) : null, - row.getInteger(PRIORITY), - row.getInteger(RETRY_COUNTER), - MessageState.valueOf(row.getString(STATE)), - row.getString(PAYLOAD_CLASS), - row.getJsonObject(PAYLOAD), - row.getJsonObject(FAILURES), - row.getString(VERTICLE_ID), - baseRecord(row) - ); - } - - @Override - public void params(Map params, DeadLetterRecord actualRecord) { - params.put(MESSAGE_ID, actualRecord.id()); - if (actualRecord.scheduled() != null) { - params.put(SCHEDULED, LocalDateTime.ofInstant(actualRecord.scheduled(), ZoneOffset.UTC)); - } - if (actualRecord.expiration() != null) { - params.put(EXPIRATION, LocalDateTime.ofInstant(actualRecord.expiration(), ZoneOffset.UTC)); - } - params.put(PRIORITY, actualRecord.priority()); - params.put(RETRY_COUNTER, actualRecord.retryCounter()); - params.put(STATE, actualRecord.messageState().name()); - params.put(PAYLOAD, actualRecord.payload()); - params.put(PAYLOAD_CLASS, actualRecord.payloadClass()); - if (actualRecord.failedProcessors() != null && !actualRecord.failedProcessors().isEmpty()) { - params.put(FAILURES, actualRecord.failedProcessors()); - } - params.put(VERTICLE_ID, actualRecord.verticleId()); - } - - @Override - public void keyParams(Map params, DeadLetterKey key) { - params.put(MESSAGE_ID, key.messageID()); - } - - @Override - public void queryBuilder(MessageRecordQuery query, QueryBuilder builder) { - builder.iLike( - new QueryFilters<>(String.class) - .filterColumn(MESSAGE_ID) - .filterParams(query.ids()) - ) - .iLike( - new QueryFilters<>(String.class) - .filterColumn(STATE) - .filterParams(query.states().stream().map(Enum::name).toList()) - ) - .from( - new QueryFilter<>(Instant.class) - .filterColumn(SCHEDULED) - .filterParam(query.scheduledFrom()) - ) - .to( - new QueryFilter<>(Instant.class) - .filterColumn(SCHEDULED) - .filterParam(query.scheduledTo()) - ) - .from( - new QueryFilter<>(Integer.class) - .filterColumn(RETRY_COUNTER) - .filterParam(query.retryCounterFrom()) - ) - .to( - new QueryFilter<>(Integer.class) - .filterColumn(RETRY_COUNTER) - .filterParam(query.retryCounterTo()) - ) - .from( - new QueryFilter<>(Integer.class) - .filterColumn(PRIORITY) - .filterParam(query.priorityFrom()) - ) - .to( - new QueryFilter<>(Integer.class) - .filterColumn(PRIORITY) - .filterParam(query.priorityTo()) - ); - } - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageQueueMapper.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageQueueMapper.java deleted file mode 100644 index 8f8401c..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageQueueMapper.java +++ /dev/null @@ -1,139 +0,0 @@ -package io.es4j.infrastructure.messagebroker.postgres.mappers; - - -import io.es4j.sql.RecordMapper; -import io.es4j.sql.generator.filters.QueryBuilder; -import io.es4j.sql.models.QueryFilter; -import io.es4j.sql.models.QueryFilters; -import io.es4j.infrastructure.messagebroker.models.MessageState; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecord; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordID; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordQuery; -import io.vertx.sqlclient.Row; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.*; - - -public class MessageQueueMapper implements RecordMapper { - private static final String MESSAGE_ID = "message_id"; - private static final String SCHEDULED = "scheduled"; - private static final String EXPIRATION = "expiration"; - private static final String PRIORITY = "priority"; - private static final String RETRY_COUNTER = "retry_counter"; - private static final String STATE = "state"; - private static final String PAYLOAD = "payload"; - private static final String FAILURES = "failure"; - private static final String VERTICLE_ID = "verticle_id"; - private static final String PAYLOAD_CLASS = "payload_class"; - private static final String TASK_QUEUE = "task_queue"; - - public static MessageQueueMapper INSTANCE = new MessageQueueMapper(); - private MessageQueueMapper(){} - - @Override - public String table() { - return TASK_QUEUE; - } - - @Override - public Set columns() { - return Set.of(MESSAGE_ID,PAYLOAD_CLASS, SCHEDULED, EXPIRATION, PRIORITY, RETRY_COUNTER, STATE, PAYLOAD, FAILURES, VERTICLE_ID); - - } - - - @Override - public Set keyColumns() { - return Set.of(MESSAGE_ID); - } - - @Override - public MessageRecord rowMapper(Row row) { - return new MessageRecord( - row.getString(MESSAGE_ID), - row.getLocalDateTime(SCHEDULED) != null ? row.getLocalDateTime(SCHEDULED).toInstant(ZoneOffset.UTC) : null, - row.getLocalDateTime(EXPIRATION) != null ? row.getLocalDateTime(EXPIRATION).toInstant(ZoneOffset.UTC) : null, - row.getInteger(PRIORITY), - row.getInteger(RETRY_COUNTER), - MessageState.valueOf(row.getString(STATE)), - row.getString(PAYLOAD_CLASS), - row.getJsonObject(PAYLOAD), - row.getJsonObject(FAILURES), - row.getString(VERTICLE_ID), - baseRecord(row) - ); - } - - @Override - public void params(Map params, MessageRecord actualRecord) { - params.put(MESSAGE_ID, actualRecord.id()); - if (actualRecord.scheduled() != null) { - params.put(SCHEDULED, LocalDateTime.ofInstant(actualRecord.scheduled(), ZoneOffset.UTC)); - } - if (actualRecord.expiration() != null) { - params.put(EXPIRATION, LocalDateTime.ofInstant(actualRecord.expiration(), ZoneOffset.UTC)); - } - params.put(PRIORITY, actualRecord.priority()); - params.put(RETRY_COUNTER, actualRecord.retryCounter()); - params.put(STATE, actualRecord.messageState().name()); - params.put(PAYLOAD, actualRecord.payload()); - params.put(PAYLOAD_CLASS, actualRecord.payloadClass()); - if (actualRecord.failedProcessors() != null && !actualRecord.failedProcessors().isEmpty()) { - params.put(FAILURES, actualRecord.failedProcessors()); - } - params.put(VERTICLE_ID, actualRecord.verticleId()); - } - - @Override - public void keyParams(Map params, MessageRecordID key) { - params.put(MESSAGE_ID, key.id()); - } - - @Override - public void queryBuilder(MessageRecordQuery query, QueryBuilder builder) { - builder.iLike( - new QueryFilters<>(String.class) - .filterColumn(MESSAGE_ID) - .filterParams(query.ids()) - ) - .iLike( - new QueryFilters<>(MessageState.class) - .filterColumn(STATE) - .filterParams(query.states()) - ) - .from( - new QueryFilter<>(Instant.class) - .filterColumn(SCHEDULED) - .filterParam(query.scheduledFrom()) - ) - .to( - new QueryFilter<>(Instant.class) - .filterColumn(SCHEDULED) - .filterParam(query.scheduledTo()) - ) - .from( - new QueryFilter<>(Integer.class) - .filterColumn(RETRY_COUNTER) - .filterParam(query.retryCounterFrom()) - ) - .to( - new QueryFilter<>(Integer.class) - .filterColumn(RETRY_COUNTER) - .filterParam(query.retryCounterTo()) - ) - .from( - new QueryFilter<>(Integer.class) - .filterColumn(PRIORITY) - .filterParam(query.priorityFrom()) - ) - .to( - new QueryFilter<>(Integer.class) - .filterColumn(PRIORITY) - .filterParam(query.priorityTo()) - ); - } - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/PgQueueLiquibase.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/PgQueueLiquibase.java deleted file mode 100644 index 8414bd1..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/PgQueueLiquibase.java +++ /dev/null @@ -1,66 +0,0 @@ -package io.es4j.infrastructure.messagebroker.postgres.mappers; - -import io.es4j.sql.LiquibaseHandler; -import io.es4j.sql.RepositoryHandler; -import io.es4j.sql.misc.Constants; -import io.es4j.sql.misc.EnvVars; -import io.es4j.infrastructure.messagebroker.models.QueueConfiguration; -import io.es4j.infrastructure.messagebroker.postgres.PgRefresher; -import io.smallrye.mutiny.Uni; -import io.es4j.infrastructure.messagebroker.postgres.PgQueueSubscriber; - -import java.util.Map; - -public class PgQueueLiquibase { - - private PgQueueLiquibase(){} - - - public static Uni bootstrapQueue(final RepositoryHandler repositoryHandler, QueueConfiguration configuration) { - if (PgQueueSubscriber.LIQUIBASE_DEPLOYED.compareAndSet(false, true)) { - return liquibase(repositoryHandler) - .invoke(avoid -> bootstrapQueueRefresher(repositoryHandler, configuration)); - } - return Uni.createFrom().voidItem(); - } - - private static Uni liquibase(RepositoryHandler repositoryHandler) { - return LiquibaseHandler.liquibaseString( - repositoryHandler, - "message-broker.xml", - Map.of( - "schema", repositoryHandler.configuration().getString(Constants.SCHEMA, EnvVars.SCHEMA) - ) - ); - } - - private static PgRefresher bootstrapQueueRefresher(RepositoryHandler repositoryHandler, QueueConfiguration queueConfiguration) { - return new PgRefresher( - repositoryHandler, - queueConfiguration - ) - .startRetryTimer(1L) - .startRecoveryTimer(1L) - .startPurgeRefreshTimer(1L); - } - - public static String camelToSnake(String str) { - // Regular Expression - String regex = "([a-z])([A-Z]+)"; - - // Replacement string - String replacement = "$1_$2"; - - // Replace the given regex - // with replacement string - // and convert it to lower case. - str = str - .replaceAll( - regex, replacement) - .toLowerCase(); - - // return string - return str; - } - -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterRecord.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterRecord.java deleted file mode 100644 index f3c8f1f..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterRecord.java +++ /dev/null @@ -1,96 +0,0 @@ -package io.es4j.infrastructure.messagebroker.postgres.models; - -import io.es4j.infrastructure.messagebroker.models.MessageState; -import io.es4j.sql.models.BaseRecord; -import io.es4j.sql.models.RepositoryRecord; -import io.soabase.recordbuilder.core.RecordBuilder; -import io.vertx.core.json.JsonObject; - - -import java.time.Instant; -import java.util.Map; - -@RecordBuilder -public record DeadLetterRecord( - String id, - Instant scheduled, - Instant expiration, - Integer priority, - Integer retryCounter, - MessageState messageState, - String payloadClass, - JsonObject payload, - JsonObject failedProcessors, - String verticleId, - BaseRecord baseRecord -) implements RepositoryRecord { - - - public static DeadLetterRecord simpleTask(String id, String tenant, Object payload) { - return new DeadLetterRecord( - id, - null, - null, - 0, - 0, - MessageState.CREATED, - null, - JsonObject.mapFrom(payload), - null, - null, - BaseRecord.newRecord(tenant) - ); - } - - public static DeadLetterRecord task(String id, String tenant, Object payload, Instant scheduled, Instant expiration, Integer priority) { - return new DeadLetterRecord( - id, - scheduled, - expiration, - priority, - 0, - MessageState.CREATED, - null, - JsonObject.mapFrom(payload), - null, - null, - BaseRecord.newRecord(tenant) - ); - } - - public static DeadLetterRecord priorityTask(String id, String tenant, Object payload, Integer priority) { - return new DeadLetterRecord( - id, - null, - null, - priority, - 0, - MessageState.CREATED, - null, - JsonObject.mapFrom(payload), - null, - null, - BaseRecord.newRecord(tenant) - ); - } - - public DeadLetterRecord withState(final MessageState retry) { - return new DeadLetterRecord(id, scheduled, expiration, priority, 0, retry, payloadClass, payload, failedProcessors, verticleId, baseRecord); - } - - public DeadLetterRecord increaseCounter() { - return new DeadLetterRecord(id, scheduled, expiration, priority, retryCounter + 1, MessageState.RETRY, payloadClass, payload, failedProcessors, verticleId, baseRecord); - } - - public DeadLetterRecord withFailures(Map failures) { - if (!failures.isEmpty()) { - return new DeadLetterRecord(id, scheduled, expiration, priority, retryCounter + 1, MessageState.RETRY, payloadClass, payload, JsonObject.mapFrom(failures), verticleId, baseRecord); - } - return new DeadLetterRecord(id, scheduled, expiration, priority, retryCounter, MessageState.PROCESSED, payloadClass, payload, this.failedProcessors, verticleId, baseRecord); - } - - @Override - public DeadLetterRecord with(BaseRecord baseRecord) { - return new DeadLetterRecord(id, scheduled, expiration, priority, retryCounter, messageState, payloadClass, payload, failedProcessors, verticleId, baseRecord); - } -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransaction.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransaction.java deleted file mode 100644 index d7ef9cd..0000000 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransaction.java +++ /dev/null @@ -1,18 +0,0 @@ -package io.es4j.infrastructure.messagebroker.postgres.models; - -import io.es4j.sql.models.BaseRecord; -import io.es4j.sql.models.RepositoryRecord; -import io.soabase.recordbuilder.core.RecordBuilder; - -@RecordBuilder -public record MessageTransaction( - String id, - String processorClass, - String messageClass, - BaseRecord baseRecord -) implements RepositoryRecord { - @Override - public MessageTransaction with(BaseRecord persistedRecord) { - return new MessageTransaction(id, processorClass, messageClass,persistedRecord); - } -} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/ConsumerTransactionProvider.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/ConsumerTransactionProvider.java new file mode 100644 index 0000000..0d68105 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/ConsumerTransactionProvider.java @@ -0,0 +1,18 @@ +package io.es4j.infrastructure.pgbroker; + + +import io.es4j.infrastructure.pgbroker.models.Message; +import io.es4j.infrastructure.pgbroker.models.ConsumerTransaction; +import io.es4j.sql.RepositoryHandler; +import io.smallrye.mutiny.Uni; +import io.vertx.core.json.JsonObject; +import io.vertx.mutiny.core.Vertx; + +import java.util.function.BiFunction; + +public interface ConsumerTransactionProvider { + + void start(RepositoryHandler repositoryHandler); + + Uni transaction(String processorClass, Message message, BiFunction, ConsumerTransaction, Uni> function); +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/PgBroker.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/PgBroker.java new file mode 100644 index 0000000..43c41df --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/PgBroker.java @@ -0,0 +1,95 @@ +package io.es4j.infrastructure.pgbroker; + + +import io.es4j.infrastructure.pgbroker.models.PgBrokerConfiguration; +import io.es4j.sql.LiquibaseHandler; +import io.es4j.sql.RepositoryHandler; +import io.es4j.sql.misc.EnvVars; +import io.smallrye.mutiny.Multi; +import io.smallrye.mutiny.Uni; +import io.vertx.core.DeploymentOptions; +import io.vertx.core.Verticle; +import io.vertx.core.impl.cpu.CpuCoreSensor; +import io.vertx.core.json.JsonObject; +import io.vertx.mutiny.core.Vertx; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.function.Supplier; + +public class PgBroker { + private static final Logger LOGGER = LoggerFactory.getLogger(PgBroker.class); + private static final List deployments = new ArrayList<>(); + + private PgBroker() { + } + + public static Uni deploy(JsonObject configuration, Vertx vertx, Integer instances) { + try { + final var brokerConfiguration = PgBrokerConfiguration.defaultConfiguration(); + LOGGER.info("starting pg broker {}", brokerConfiguration); + LOGGER.info("broker infra configuration {}", configuration.encodePrettily()); + final var repositoryHandler = RepositoryHandler.leasePool(configuration, vertx); + return LiquibaseHandler.liquibaseString(repositoryHandler, "queue.xml", Map.of("schema", repositoryHandler.configuration().getString("schema", EnvVars.SCHEMA))) + .flatMap(avoid -> { + Supplier supplier = () -> new PgBrokerVerticle(brokerConfiguration); + return repositoryHandler.vertx().deployVerticle(supplier, new DeploymentOptions().setInstances(instances).setConfig(configuration)) + .map(deployments::add) + .replaceWithVoid(); + } + ); + } catch (Exception e) { + return Uni.createFrom().failure(e); + } + } + public static Uni deploy(JsonObject configuration, Vertx vertx) { + try { + final var brokerConfiguration = PgBrokerConfiguration.defaultConfiguration(); + LOGGER.info("starting pg broker {}", brokerConfiguration); + LOGGER.info("broker infra configuration {}", configuration.encodePrettily()); + final var repositoryHandler = RepositoryHandler.leasePool(configuration, vertx); + return LiquibaseHandler.liquibaseString(repositoryHandler, "queue.xml", Map.of("schema", repositoryHandler.configuration().getString("schema", EnvVars.SCHEMA))) + .flatMap(avoid -> { + Supplier supplier = () -> new PgBrokerVerticle(brokerConfiguration); + return repositoryHandler.vertx().deployVerticle(supplier, new DeploymentOptions().setInstances(1).setConfig(configuration)) + .map(deployments::add) + .replaceWithVoid(); + } + ); + } catch (Exception e) { + return Uni.createFrom().failure(e); + } + } + + public static Uni deploy(JsonObject configuration, Vertx vertx, PgBrokerConfiguration brokerConfiguration) { + try { + LOGGER.info("starting pg broker {}", brokerConfiguration); + LOGGER.info("broker infra configuration {}", configuration.encodePrettily()); + final var repositoryHandler = RepositoryHandler.leasePool(configuration, vertx); + return LiquibaseHandler.liquibaseString(repositoryHandler, "queue.xml", Map.of("schema", repositoryHandler.configuration().getString("schema", EnvVars.SCHEMA))) + .flatMap(avoid -> { + Supplier supplier = () -> new PgBrokerVerticle(brokerConfiguration); + return repositoryHandler.vertx().deployVerticle(supplier, new DeploymentOptions().setInstances(CpuCoreSensor.availableProcessors()).setConfig(configuration)) + .map(deployments::add) + .replaceWithVoid(); + } + ); + } catch (Exception e) { + return Uni.createFrom().failure(e); + } + } + + public static Uni undeploy(Vertx vertx) { + if (!deployments.isEmpty()) { + return Multi.createFrom().iterable(deployments) + .onItem().transformToUniAndMerge(vertx::undeploy) + .collect().asList() + .replaceWithVoid(); + } + return Uni.createFrom().voidItem(); + } + + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/PgBrokerVerticle.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/PgBrokerVerticle.java new file mode 100644 index 0000000..929756b --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/PgBrokerVerticle.java @@ -0,0 +1,139 @@ +package io.es4j.infrastructure.pgbroker; + + +import io.es4j.infrastructure.misc.Es4jServiceLoader; +import io.es4j.infrastructure.pgbroker.mappers.DeadLetterMapper; +import io.es4j.infrastructure.pgbroker.mappers.MessageQueueMapper; +import io.es4j.infrastructure.pgbroker.mappers.QueuePartitionMapper; +import io.es4j.infrastructure.pgbroker.messagebroker.PgChannel; +import io.es4j.infrastructure.pgbroker.models.ConsumerManager; +import io.es4j.infrastructure.pgbroker.models.ConsumerWrap; +import io.es4j.infrastructure.pgbroker.models.PgBrokerConfiguration; +import io.es4j.sql.Repository; +import io.es4j.sql.RepositoryHandler; +import io.smallrye.mutiny.Multi; +import io.smallrye.mutiny.Uni; +import io.smallrye.mutiny.vertx.core.AbstractVerticle; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.groupingBy; + +public class PgBrokerVerticle extends AbstractVerticle { + + private final PgBrokerConfiguration configuration; + private PgChannel pgChannel; + private RepositoryHandler repositoryHandler; + private static final Logger LOGGER = LoggerFactory.getLogger(PgBrokerVerticle.class); + + public PgBrokerVerticle(PgBrokerConfiguration configuration) { + this.configuration = configuration; + } + + @Override + public Uni asyncStart() { + this.repositoryHandler = RepositoryHandler.leasePool(config(), vertx); + this.pgChannel = createChannel(); + LOGGER.info("starting pg broker {}", deploymentID()); + return deploy(); + } + + @Override + public Uni asyncStop() { + return pgChannel.stop() + .flatMap(avoid -> repositoryHandler.close()) + .replaceWithVoid(); + } + + private final String deploymentID = UUID.randomUUID().toString(); + + @Override + public String deploymentID() { + return deploymentID; + } + + public Uni deploy() { + try { + final var consumers = ServiceLoader.load(QueueConsumer.class).stream().map(ServiceLoader.Provider::get).toList(); + final var consumerTransactionProvider = configuration.consumerTransactionProvider(); + consumerTransactionProvider.start(repositoryHandler); + return Multi.createFrom().iterable(consumers) + .onItem().transformToUniAndMerge(m -> m.start(vertx, config())) + .collect().asList() + .replaceWithVoid() + .flatMap(avoid -> pgChannel.start( + new ConsumerManager( + configuration, + wrap(deploymentID, consumers), + consumerTransactionProvider, + repositoryHandler.vertx() + ))); + + } catch (Exception e) { + return Uni.createFrom().failure(e); + } + } + + private PgChannel createChannel() { + final var infraPgSubscriber = io.vertx.mutiny.pgclient.pubsub.PgSubscriber.subscriber( + repositoryHandler.vertx(), + RepositoryHandler.connectionOptions(repositoryHandler.configuration()) + ); + infraPgSubscriber.reconnectPolicy(integer -> 0L); + return new PgChannel( + new Repository<>(MessageQueueMapper.INSTANCE, repositoryHandler), + new Repository<>(DeadLetterMapper.INSTANCE, repositoryHandler), + new Repository<>(QueuePartitionMapper.INSTANCE, repositoryHandler), + infraPgSubscriber, + Objects.requireNonNullElse(repositoryHandler.vertx().getOrCreateContext().deploymentID(), UUID.randomUUID().toString()) + ); + } + + public List wrap(String deploymentId, List queueConsumers) { + final var queueMap = queueConsumers.stream() + .map(impl -> { + Class firstGenericType = Es4jServiceLoader.getFirstGenericType(impl); + return ImmutablePair.of(firstGenericType, impl); + }) + .collect(Collectors.groupingBy(ImmutablePair::getLeft, Collectors.mapping(ImmutablePair::getRight, Collectors.toList()))); + return queueMap.entrySet().stream() + .map(entry -> { + final var tClass = entry.getKey(); + validateProcessors(entry.getValue(), tClass); + final var defaultProcessor = entry.getValue().stream().filter(p -> p.tenants() == null).findFirst().orElseThrow(); + final var customProcessors = entry.getValue().stream() + .filter(p -> p.tenants() != null) + .collect(groupingBy(QueueConsumer::tenants)); + final var processorWrapper = new ConsumerWrap( + deploymentId, + defaultProcessor, + customProcessors, + tClass + ); + return processorWrapper; + } + ) + .toList(); + } + + private static void validateProcessors(List queues, Class tClass) { + if (queues.stream().filter( + p -> p.tenants() == null + ).toList().size() > 1) { + throw new IllegalStateException("More than one default implementation for -> " + tClass); + } + queues.stream() + .filter(p -> p.tenants() != null) + .collect(groupingBy(QueueConsumer::tenants)) + .forEach((key, value) -> { + if (value.size() > 1) { + throw new IllegalStateException("More than one custom implementation for tenant " + key + " queue -> " + tClass); + } + } + ); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProcessor.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/QueueConsumer.java similarity index 54% rename from es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProcessor.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/QueueConsumer.java index 23e6111..573970c 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/MessageProcessor.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/QueueConsumer.java @@ -1,7 +1,9 @@ -package io.es4j.infrastructure.messagebroker; +package io.es4j.infrastructure.pgbroker; -import io.es4j.infrastructure.messagebroker.models.QueueTransaction; +import io.es4j.infrastructure.pgbroker.models.ConsumerTransaction; import io.smallrye.mutiny.Uni; +import io.vertx.core.json.JsonObject; +import io.vertx.mutiny.core.Vertx; import java.util.List; @@ -13,8 +15,13 @@ * * @param The payload, queue entry type */ -public interface MessageProcessor { - Uni process(T payload, QueueTransaction queueTransaction); +public interface QueueConsumer { + + default Uni start(Vertx vertx, JsonObject config) { + return Uni.createFrom().voidItem(); + } + + Uni process(T payload, ConsumerTransaction consumerTransaction); default List> fatalExceptions() { return List.of(); @@ -28,4 +35,8 @@ default Boolean blockingProcessor() { return Boolean.FALSE; } + default T parse(JsonObject jsonObject, Class payloadClass) { + return jsonObject.mapTo(payloadClass); + } + } diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/TopicConsumer.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/TopicConsumer.java new file mode 100644 index 0000000..ba372b0 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/TopicConsumer.java @@ -0,0 +1,42 @@ +package io.es4j.infrastructure.pgbroker; + +import io.es4j.infrastructure.pgbroker.models.ConsumerTransaction; +import io.smallrye.mutiny.Uni; +import io.vertx.core.Vertx; +import io.vertx.core.json.JsonObject; + +import java.util.List; + + +/** + * When using mono consumer the queue entry will be processed only ONCE by either the default implementation + * which is the one that returns tenant null or the tenant specific implementation which is the implementation that + * returns a matching tenant in the tenants() method + * + * @param The payload, queue entry type + */ +public interface TopicConsumer { + + Uni setup(Vertx vertx, JsonObject componentConfiguration); + + Uni start(Vertx vertx); + + Uni process(T payload, ConsumerTransaction consumerTransaction); + + default List> fatalExceptions() { + return List.of(); + } + + default List tenants() { + return null; + } + + default Boolean blockingProcessor() { + return Boolean.FALSE; + } + + default T parse(JsonObject jsonObject, Class payloadClass) { + return jsonObject.mapTo(payloadClass); + } + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/ConsumerExeception.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/ConsumerExeception.java new file mode 100644 index 0000000..199ca52 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/ConsumerExeception.java @@ -0,0 +1,15 @@ +package io.es4j.infrastructure.pgbroker.exceptions; + + +public class ConsumerExeception extends RuntimeException { + + public ConsumerExeception(String error) { + super(error); + } + + public ConsumerExeception(Throwable throwable) { + super(throwable); + } + + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/DuplicateMessage.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/DuplicateMessage.java new file mode 100644 index 0000000..b1dafca --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/DuplicateMessage.java @@ -0,0 +1,10 @@ +package io.es4j.infrastructure.pgbroker.exceptions; + +public class DuplicateMessage extends RuntimeException { + public DuplicateMessage(Throwable e) { + super(e); + } + public DuplicateMessage(String e) { + super(e); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/InterruptMessageStream.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/InterruptMessageStream.java new file mode 100644 index 0000000..a02feba --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/InterruptMessageStream.java @@ -0,0 +1,6 @@ +package io.es4j.infrastructure.pgbroker.exceptions; + + +public class InterruptMessageStream extends RuntimeException { + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/InvalidProcessorException.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/InvalidProcessorException.java new file mode 100644 index 0000000..e4a7f66 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/InvalidProcessorException.java @@ -0,0 +1,4 @@ +package io.es4j.infrastructure.pgbroker.exceptions; + +public class InvalidProcessorException extends RuntimeException{ +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/MessageParsingException.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/MessageParsingException.java new file mode 100644 index 0000000..26e15b8 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/MessageParsingException.java @@ -0,0 +1,4 @@ +package io.es4j.infrastructure.pgbroker.exceptions; + +public class MessageParsingException extends RuntimeException{ +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/PartitionNotFound.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/PartitionNotFound.java new file mode 100644 index 0000000..379796c --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/PartitionNotFound.java @@ -0,0 +1,4 @@ +package io.es4j.infrastructure.pgbroker.exceptions; + +public class PartitionNotFound extends RuntimeException{ +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/PartitionTakenException.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/PartitionTakenException.java new file mode 100644 index 0000000..4c5ee67 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/PartitionTakenException.java @@ -0,0 +1,4 @@ +package io.es4j.infrastructure.pgbroker.exceptions; + +public class PartitionTakenException extends RuntimeException{ +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/ProducerExeception.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/ProducerExeception.java new file mode 100644 index 0000000..deac27d --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/ProducerExeception.java @@ -0,0 +1,15 @@ +package io.es4j.infrastructure.pgbroker.exceptions; + + + +public class ProducerExeception extends RuntimeException { + + public ProducerExeception(Throwable throwable) { + super(throwable); + } + + public ProducerExeception(String throwable) { + super(throwable); + } + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/QueueEmptyException.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/QueueEmptyException.java new file mode 100644 index 0000000..2261d70 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/exceptions/QueueEmptyException.java @@ -0,0 +1,4 @@ +package io.es4j.infrastructure.pgbroker.exceptions; + +public class QueueEmptyException extends RuntimeException{ +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/DeadLetterMapper.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/DeadLetterMapper.java new file mode 100644 index 0000000..98b2092 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/DeadLetterMapper.java @@ -0,0 +1,164 @@ +package io.es4j.infrastructure.pgbroker.mappers; + + +import io.es4j.infrastructure.pgbroker.models.DeadLetterKey; +import io.es4j.infrastructure.pgbroker.models.DeadLetterRecord; +import io.es4j.infrastructure.pgbroker.models.MessageRecordQuery; +import io.es4j.infrastructure.pgbroker.models.MessageState; +import io.es4j.sql.RecordMapper; +import io.es4j.sql.generator.filters.QueryBuilder; +import io.es4j.sql.models.QueryFilter; +import io.es4j.sql.models.QueryFilters; +import io.vertx.sqlclient.Row; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Map; +import java.util.Set; + + +public class DeadLetterMapper implements RecordMapper { + private static final String MESSAGE_ID = "message_id"; + private static final String SCHEDULED = "scheduled"; + private static final String EXPIRATION = "expiration"; + private static final String PRIORITY = "priority"; + private static final String RETRY_COUNTER = "retry_counter"; + private static final String STATE = "state"; + private static final String PAYLOAD = "payload"; + private static final String FAILURES = "failure"; + private static final String VERTICLE_ID = "verticle_id"; + private static final String QUEUE = "queue"; + public static final String JOB_QUEUE = "queue_dead_letter"; + private static final String PARTITION_ID = "partition_id"; + + private static final String PARTITION_KEY = "partition_key"; + + public static DeadLetterMapper INSTANCE = new DeadLetterMapper(); + + private DeadLetterMapper() { + } + + @Override + public String table() { + return JOB_QUEUE; + } + + @Override + public Set columns() { + return Set.of(MESSAGE_ID, SCHEDULED, EXPIRATION, PRIORITY, RETRY_COUNTER, STATE, PAYLOAD, QUEUE, FAILURES, VERTICLE_ID, PARTITION_ID, PARTITION_KEY); + + } + + + @Override + public Set keyColumns() { + return Set.of(MESSAGE_ID); + } + + @Override + public DeadLetterRecord rowMapper(Row row) { + return new DeadLetterRecord( + row.getString(MESSAGE_ID), + row.getLocalDateTime(SCHEDULED) != null ? row.getLocalDateTime(SCHEDULED).toInstant(ZoneOffset.UTC) : null, + row.getLocalDateTime(EXPIRATION) != null ? row.getLocalDateTime(EXPIRATION).toInstant(ZoneOffset.UTC) : null, + row.getInteger(PRIORITY), + row.getInteger(RETRY_COUNTER), + MessageState.valueOf(row.getString(STATE)), + row.getString(QUEUE), + row.getJsonObject(PAYLOAD), + row.getJsonObject(FAILURES), + row.getString(VERTICLE_ID), + row.getString(PARTITION_ID), + row.getString(PARTITION_KEY), + baseRecord(row) + ); + } + + @Override + public void params(Map params, DeadLetterRecord actualRecord) { + params.put(MESSAGE_ID, actualRecord.id()); + if (actualRecord.scheduled() != null) { + params.put(SCHEDULED, LocalDateTime.ofInstant(actualRecord.scheduled(), ZoneOffset.UTC)); + } + if (actualRecord.expiration() != null) { + params.put(EXPIRATION, LocalDateTime.ofInstant(actualRecord.expiration(), ZoneOffset.UTC)); + } + params.put(PRIORITY, actualRecord.priority()); + params.put(RETRY_COUNTER, actualRecord.retryCounter()); + params.put(STATE, actualRecord.messageState().name()); + params.put(PAYLOAD, actualRecord.payload()); + params.put(QUEUE, actualRecord.payloadClass()); + if (actualRecord.failedProcessors() != null && !actualRecord.failedProcessors().isEmpty()) { + params.put(FAILURES, actualRecord.failedProcessors()); + } + params.put(VERTICLE_ID, actualRecord.verticleId()); + params.put(PARTITION_KEY, actualRecord.partitionKey()); + params.put(PARTITION_ID, actualRecord.partitionId()); + } + + @Override + public void keyParams(Map params, DeadLetterKey key) { + params.put(MESSAGE_ID, key.messageID()); + } + + @Override + public void queryBuilder(MessageRecordQuery query, QueryBuilder builder) { + builder.iLike( + new QueryFilters<>(String.class) + .filterColumn(MESSAGE_ID) + .filterParams(query.ids()) + ) + .eq( + new QueryFilters<>(String.class) + .filterColumn(QUEUE) + .filterParam(query.payloadClass()) + ) + .iLike( + new QueryFilters<>(String.class) + .filterColumn(PARTITION_KEY) + .filterParam(query.partitionKey()) + ) + .iLike( + new QueryFilters<>(String.class) + .filterColumn(PARTITION_ID) + .filterParam(query.partition()) + ) + .iLike( + new QueryFilters<>(MessageState.class) + .filterColumn(STATE) + .filterParams(query.states()) + ) + .from( + new QueryFilter<>(Instant.class) + .filterColumn(SCHEDULED) + .filterParam(query.scheduledFrom()) + ) + .to( + new QueryFilter<>(Instant.class) + .filterColumn(SCHEDULED) + .filterParam(query.scheduledTo()) + ) + .from( + new QueryFilter<>(Integer.class) + .filterColumn(RETRY_COUNTER) + .filterParam(query.retryCounterFrom()) + ) + .to( + new QueryFilter<>(Integer.class) + .filterColumn(RETRY_COUNTER) + .filterParam(query.retryCounterTo()) + ) + .from( + new QueryFilter<>(Integer.class) + .filterColumn(PRIORITY) + .filterParam(query.priorityFrom()) + ) + .to( + new QueryFilter<>(Integer.class) + .filterColumn(PRIORITY) + .filterParam(query.priorityTo()) + ); + } + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/MessageQueueMapper.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/MessageQueueMapper.java new file mode 100644 index 0000000..7fa4755 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/MessageQueueMapper.java @@ -0,0 +1,155 @@ +package io.es4j.infrastructure.pgbroker.mappers; + + + +import io.es4j.infrastructure.pgbroker.models.MessageRecord; +import io.es4j.infrastructure.pgbroker.models.MessageRecordID; +import io.es4j.infrastructure.pgbroker.models.MessageRecordQuery; +import io.es4j.infrastructure.pgbroker.models.MessageState; +import io.es4j.sql.RecordMapper; +import io.es4j.sql.generator.filters.QueryBuilder; +import io.es4j.sql.models.QueryFilter; +import io.es4j.sql.models.QueryFilters; +import io.vertx.sqlclient.Row; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Map; +import java.util.Set; + + +public class MessageQueueMapper implements RecordMapper { + private static final String MESSAGE_ID = "message_id"; + private static final String SCHEDULED = "scheduled"; + private static final String EXPIRATION = "expiration"; + private static final String PRIORITY = "priority"; + private static final String RETRY_COUNTER = "retry_counter"; + private static final String STATE = "state"; + private static final String PAYLOAD = "payload"; + private static final String FAILURES = "failure"; + private static final String PARTITION_KEY = "partition_key"; + private static final String VERTICLE_ID = "verticle_id"; + private static final String PAYLOAD_CLASS = "queue"; + private static final String MESSAGE_SEQUENCE = "message_sequence"; + private static final String PARTITION_ID = "partition_id"; + private static final String TASK_QUEUE = "queue"; + + public static MessageQueueMapper INSTANCE = new MessageQueueMapper(); + + private MessageQueueMapper() { + } + + @Override + public String table() { + return TASK_QUEUE; + } + + @Override + public Set columns() { + return Set.of(MESSAGE_ID, PAYLOAD_CLASS, SCHEDULED, EXPIRATION, PRIORITY, RETRY_COUNTER, STATE, PAYLOAD, FAILURES, VERTICLE_ID, PARTITION_ID, PARTITION_KEY); + } + + @Override + public Set updatableColumns() { + return Set.of(RETRY_COUNTER, STATE, FAILURES, VERTICLE_ID); + } + + @Override + public Set keyColumns() { + return Set.of(MESSAGE_ID); + } + + @Override + public MessageRecord rowMapper(Row row) { + return new MessageRecord( + row.getString(MESSAGE_ID), + row.getLocalDateTime(SCHEDULED) != null ? row.getLocalDateTime(SCHEDULED).toInstant(ZoneOffset.UTC) : null, + row.getLocalDateTime(EXPIRATION) != null ? row.getLocalDateTime(EXPIRATION).toInstant(ZoneOffset.UTC) : null, + row.getInteger(PRIORITY), + row.getInteger(RETRY_COUNTER), + MessageState.valueOf(row.getString(STATE)), + row.getString(PAYLOAD_CLASS), + row.getJsonObject(PAYLOAD), + row.getJsonObject(FAILURES), + row.getString(VERTICLE_ID), + row.getLong(MESSAGE_SEQUENCE), + row.getString(PARTITION_ID), + row.getString(PARTITION_KEY), + baseRecord(row) + ); + } + + @Override + public void params(Map params, MessageRecord actualRecord) { + params.put(MESSAGE_ID, actualRecord.id()); + if (actualRecord.scheduled() != null) { + params.put(SCHEDULED, LocalDateTime.ofInstant(actualRecord.scheduled(), ZoneOffset.UTC)); + } + if (actualRecord.expiration() != null) { + params.put(EXPIRATION, LocalDateTime.ofInstant(actualRecord.expiration(), ZoneOffset.UTC)); + } + params.put(PRIORITY, actualRecord.priority()); + params.put(RETRY_COUNTER, actualRecord.retryCounter()); + params.put(STATE, actualRecord.messageState().name()); + params.put(PAYLOAD, actualRecord.payload()); + params.put(PAYLOAD_CLASS, actualRecord.payloadClass()); + if (actualRecord.failedProcessors() != null && !actualRecord.failedProcessors().isEmpty()) { + params.put(FAILURES, actualRecord.failedProcessors()); + } + params.put(VERTICLE_ID, actualRecord.verticleId()); + params.put(MESSAGE_SEQUENCE, actualRecord.messageSequence()); + params.put(PARTITION_KEY, actualRecord.partitionKey()); + params.put(PARTITION_ID, actualRecord.partitionId()); + } + + @Override + public void keyParams(Map params, MessageRecordID key) { + params.put(MESSAGE_ID, key.id()); + } + + @Override + public void queryBuilder(MessageRecordQuery query, QueryBuilder builder) { + builder.iLike( + new QueryFilters<>(String.class) + .filterColumn(MESSAGE_ID) + .filterParams(query.ids()) + ) + .iLike( + new QueryFilters<>(MessageState.class) + .filterColumn(STATE) + .filterParams(query.states()) + ) + .from( + new QueryFilter<>(Instant.class) + .filterColumn(SCHEDULED) + .filterParam(query.scheduledFrom()) + ) + .to( + new QueryFilter<>(Instant.class) + .filterColumn(SCHEDULED) + .filterParam(query.scheduledTo()) + ) + .from( + new QueryFilter<>(Integer.class) + .filterColumn(RETRY_COUNTER) + .filterParam(query.retryCounterFrom()) + ) + .to( + new QueryFilter<>(Integer.class) + .filterColumn(RETRY_COUNTER) + .filterParam(query.retryCounterTo()) + ) + .from( + new QueryFilter<>(Integer.class) + .filterColumn(PRIORITY) + .filterParam(query.priorityFrom()) + ) + .to( + new QueryFilter<>(Integer.class) + .filterColumn(PRIORITY) + .filterParam(query.priorityTo()) + ); + } + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageTransactionMapper.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/MessageTransactionMapper.java similarity index 83% rename from es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageTransactionMapper.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/MessageTransactionMapper.java index 9b8d3ed..bf6ccf7 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/mappers/MessageTransactionMapper.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/MessageTransactionMapper.java @@ -1,23 +1,24 @@ -package io.es4j.infrastructure.messagebroker.postgres.mappers; +package io.es4j.infrastructure.pgbroker.mappers; +import io.es4j.infrastructure.pgbroker.models.MessageTransaction; +import io.es4j.infrastructure.pgbroker.models.MessageTransactionID; +import io.es4j.infrastructure.pgbroker.models.MessageTransactionQuery; import io.es4j.sql.RecordMapper; import io.es4j.sql.generator.filters.QueryBuilder; import io.es4j.sql.models.QueryFilters; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransaction; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransactionID; -import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransactionQuery; import io.vertx.sqlclient.Row; import java.util.Map; import java.util.Set; + public class MessageTransactionMapper implements RecordMapper { public static final MessageTransactionMapper INSTANCE = new MessageTransactionMapper(); private static final String PROCESSOR = "processor"; private static final String MESSAGE_CLASS = "message_class"; private static final String MESSAGE_ID = "message_id"; - public static final String TASK_QUEUE_TX = "task_queue_tx"; + public static final String TASK_QUEUE_TX = "queue_tx"; private MessageTransactionMapper() { } diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/QueuePartitionMapper.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/QueuePartitionMapper.java new file mode 100644 index 0000000..c6da096 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/mappers/QueuePartitionMapper.java @@ -0,0 +1,73 @@ +package io.es4j.infrastructure.pgbroker.mappers; + + +import io.es4j.infrastructure.pgbroker.models.MessagePartition; +import io.es4j.infrastructure.pgbroker.models.PartitionKey; +import io.es4j.infrastructure.pgbroker.models.PartitionQuery; +import io.es4j.sql.RecordMapper; +import io.es4j.sql.generator.filters.QueryBuilder; +import io.es4j.sql.models.QueryFilters; +import io.vertx.sqlclient.Row; + +import java.util.Map; +import java.util.Set; + + +public class QueuePartitionMapper implements RecordMapper { + + private static final String PARTITION_ID = "partition_id"; + private static final String LOCKED = "locked"; + private static final String TABLE_NAME = "queue_partition"; + + public static final QueuePartitionMapper INSTANCE = new QueuePartitionMapper(); + public static final String DEPLOYMENT_ID = "verticle_id"; + + private QueuePartitionMapper() { + } + + @Override + public String table() { + return TABLE_NAME; + } + + @Override + public Set columns() { + return Set.of(PARTITION_ID, LOCKED, DEPLOYMENT_ID); + + } + + @Override + public Set keyColumns() { + return Set.of(PARTITION_ID); + } + + @Override + public MessagePartition rowMapper(Row row) { + return new MessagePartition( + row.getString(PARTITION_ID), + row.getString(DEPLOYMENT_ID), + row.getBoolean(LOCKED), + baseRecord(row) + ); + } + @Override + public void params(Map params, MessagePartition actualRecord) { + params.put(PARTITION_ID, actualRecord.partitionId()); + params.put(LOCKED, actualRecord.locked()); + } + + @Override + public void keyParams(Map params, PartitionKey key) { + params.put(PARTITION_ID, key.partitionId()); + } + + @Override + public void queryBuilder(PartitionQuery query, QueryBuilder builder) { + builder.eq( + new QueryFilters<>(Long.class) + .filterColumn(PARTITION_ID) + .filterParams(query.lockIds()) + ); + } + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/ConcurrentPollingSession.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/ConcurrentPollingSession.java new file mode 100644 index 0000000..e9eadf5 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/ConcurrentPollingSession.java @@ -0,0 +1,126 @@ +package io.es4j.infrastructure.pgbroker.messagebroker; + + +import io.es4j.infrastructure.pgbroker.exceptions.QueueEmptyException; +import io.es4j.infrastructure.pgbroker.models.*; +import io.es4j.sql.Repository; +import io.es4j.sql.exceptions.NotFound; +import io.smallrye.mutiny.Uni; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.time.Instant; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; + + +public class ConcurrentPollingSession { + private final ConcurrentLinkedQueue messages = new ConcurrentLinkedQueue<>(); + private static final Logger LOGGER = LoggerFactory.getLogger(ConcurrentPollingSession.class); + private final Repository messageQueue; + private final PgBrokerConfiguration configuration; + private final String deploymentId; + private final AtomicBoolean claimMessages = new AtomicBoolean(false); + + public ConcurrentPollingSession( + Repository messageQueue, + PgBrokerConfiguration configuration, + String deploymentId + ) { + this.messageQueue = messageQueue; + this.configuration = configuration; + this.deploymentId = deploymentId; + } + + public List pollMessages() { + final List polledMessages = new ArrayList<>(); + MessageRecord message; + while ((message = messages.poll()) != null) { + polledMessages.add(message); + } + return polledMessages; + } + + + public io.es4j.task.TimerTask provideTask() { + return () -> { + if (claimMessages.get()) { + claimMessages.set(false); + LOGGER.debug("Checking queue"); + final var params = Map.of( + "deploymentId", Objects.requireNonNullElse(deploymentId, UUID.randomUUID().toString()), + "batchSize", Objects.requireNonNullElse(configuration.batchSize(), 10), + "retryInterval", Objects.requireNonNullElse(configuration.retryBackOffInterval(), Duration.ofMinutes(5)).getSeconds() + ); + final var messagesClaimed = claimMessages(params); + final var messagesRecovered = recoverStuckMessages(params); + return Uni.join().all(messagesClaimed, messagesRecovered).andCollectFailures() + .onFailure().recoverWithNull().replaceWithVoid() + .eventually(() -> { + if (messages.isEmpty()) { + LOGGER.debug("Messages not found"); + } else { + LOGGER.info("Found {} messages", messages.size()); + } + }); + } else { + LOGGER.debug("Nothing to check"); + return Uni.createFrom().voidItem(); + } + }; + } + + private Uni claimMessages(Map params) { + return messageQueue.query(pollingStatement(), params) + .map(messageRecords -> { + LOGGER.debug("Claimed {} messages", messageRecords.size()); + messages.addAll(messageRecords); + return messageRecords; + }) + .onFailure(NotFound.class) + .recoverWithNull() + .replaceWithVoid(); + } + + private Uni recoverStuckMessages(Map params) { + // this is a hack to recover messages that are stuck in the PROCESSING state... + // this can happen if the verticle crashes before the message is committed + return messageQueue.query(recoveryStatement(), params) + .map(messageRecords -> { + messages.addAll(messageRecords.stream().map(m -> m.withState(MessageState.RECOVERY)).toList()); + LOGGER.debug("Recovered {} messages", messageRecords.size()); + return messageRecords; + }) + .onFailure(NotFound.class).recoverWithNull() + .replaceWithVoid(); + } + + protected String pollingStatement() { +// updated + interval '" + configuration.retryBackOffInterval().getSeconds() + " seconds' + return "update queue set state = 'PROCESSING', verticle_id = #{deploymentId} where message_id in (" + + " select message_id from queue where " + + " state in ('CREATED','SCHEDULED','RETRY')" + + " and partition_id = 'none' " + + " and (scheduled is null or scheduled <= current_timestamp)" + + " and (expiration is null or expiration >= current_timestamp)" + + " and (retry_counter = 0 or updated + interval '#{retryInterval} seconds' <= now() ) " + + " order by priority for update skip locked limit #{batchSize} " + + " ) returning *;"; + } + + protected String recoveryStatement() { + return "update queue set state = 'PROCESSING', verticle_id = #{deploymentId} where message_id in (" + + " select message_id from queue where " + + " state = 'RECOVERY' " + + " and partition_id = 'none' " + + " order by priority for update skip locked limit #{batchSize}" + + " ) returning *;"; + } + + + public void signalMessage() { + claimMessages.set(true); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/MessageProcessor.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/MessageProcessor.java new file mode 100644 index 0000000..190a4fe --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/MessageProcessor.java @@ -0,0 +1,232 @@ +package io.es4j.infrastructure.pgbroker.messagebroker; + +import io.es4j.infrastructure.pgbroker.exceptions.InterruptMessageStream; +import io.es4j.infrastructure.pgbroker.models.*; +import io.es4j.sql.Repository; +import io.es4j.sql.exceptions.NotFound; +import io.es4j.sql.models.BaseRecord; +import io.es4j.sql.models.QueryOptions; +import io.smallrye.mutiny.Multi; +import io.smallrye.mutiny.Uni; +import io.smallrye.mutiny.subscription.FixedDemandPacer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static java.util.stream.Collectors.groupingBy; + +public class MessageProcessor { + private static final EnumSet DEAD_LETTER_QUEUE_STATES = EnumSet.of(MessageState.PARKED, MessageState.FATAL_FAILURE, MessageState.RETRIES_EXHAUSTED, MessageState.EXPIRED); + private static final Logger LOGGER = LoggerFactory.getLogger(MessageProcessor.class); + private final Repository messageQueue; + private final Repository deadLetterQueue; + private final ConsumerManager consumerManager; + + public MessageProcessor( + final ConsumerManager consumerManager, + final Repository messageQueue, + final Repository deadLetterQueue + ) { + this.consumerManager = consumerManager; + this.messageQueue = messageQueue; + this.deadLetterQueue = deadLetterQueue; + } + + + public Uni processPartitions(List messageRecords) { + if (messageRecords.stream().anyMatch(m -> Objects.isNull(m.partitionId()) || Objects.equals(m.partitionId(), "none"))) { + throw new IllegalArgumentException("Message missing partitionId"); + } + if (messageRecords.stream().anyMatch(m -> Objects.isNull(m.partitionKey()))) { + throw new IllegalArgumentException("Message missing partitionKey"); + } + return startPacedStream(splitOnPartitionKey(messageRecords).entrySet()) + .onItem().transformToUniAndMerge( + partitionStream -> searchPartitionKeyInDeadletterQueue(partitionStream.getKey()) + .flatMap(deadLetterRecords -> { + if (deadLetterRecords.isEmpty()) { + LOGGER.debug("Processing messages {}", partitionStream.getValue()); + return processPartitionStream(partitionStream.getValue()); + } + LOGGER.debug("Parking messages {}", partitionStream.getValue()); + return Uni.createFrom().item(partitionStream.getValue().stream().map(RawMessage::park).toList()); + } + ) + ) + .collect().asList() + .map(lists -> lists.stream().flatMap(List::stream).toList()) + .flatMap(rawMessages -> persistResults(rawMessages.stream().map(MessageRecord::from).toList())); + } + + + public Uni processConcurrent(List concurrentMessages) { + if (concurrentMessages.stream().anyMatch(m -> !Objects.equals(m.partitionId(), "none"))) { + throw new IllegalArgumentException("Message with partition"); + } + return startPacedStream(concurrentMessages).onItem() + .transformToUniAndMerge(messageRecord -> consumerManager.consumeMessage(parseRecord(messageRecord))) + .collect().asList() + .flatMap(rawMessages -> persistResults(rawMessages.stream().map(MessageRecord::from).toList())); + } + + + private Uni> searchPartitionKeyInDeadletterQueue(String partitionKey) { + return deadLetterQueue.query(deadLetterQuery(partitionKey)) + .onFailure(NotFound.class) + .recoverWithItem(Collections.emptyList()); + } + + private Multi startPacedStream(Iterable messageRecords) { + if (consumerManager.pgBrokerConfiguration().concurrency() != null) { + final var pacer = new FixedDemandPacer(consumerManager.pgBrokerConfiguration().concurrency(), consumerManager.pgBrokerConfiguration().throttle()); + return Multi.createFrom().iterable(messageRecords).paceDemand().using(pacer); + } + return Multi.createFrom().iterable(messageRecords); + } + + private Multi>> startPacedStream(Set>> partitions) { + if (consumerManager.pgBrokerConfiguration().concurrency() != null) { + final var pacer = new FixedDemandPacer(consumerManager.pgBrokerConfiguration().concurrency(), consumerManager.pgBrokerConfiguration().throttle()); + return Multi.createFrom().iterable(partitions).paceDemand().using(pacer); + } + return Multi.createFrom().iterable(partitions); + } + + private Uni> processPartitionStream(List partitionedMessages) { + final var messagesToProcess = fillStack(partitionedMessages); + LOGGER.debug("Processing partition messages {}", messagesToProcess); + final var processedMessages = new ArrayList(partitionedMessages.size()); + return Multi.createBy().repeating().supplier(messagesToProcess::pop) + .whilst((avoid) -> !messagesToProcess.isEmpty()) + .onItem().transformToUniAndConcatenate( + rawMessage -> consumerManager.consumeMessage(rawMessage) + .map(resultingMessage -> handleProcessResult(messagesToProcess, processedMessages, resultingMessage)) + ) + .collect().asList() + .onItemOrFailure() + .transform((i, f) -> processedMessages); + } + + private static Class handleProcessResult(Stack messagesToProcess, ArrayList processedMessages, RawMessage resultingMessage) { + processedMessages.add(resultingMessage); + if (resultingMessage.messageState() != MessageState.PROCESSED) { + LOGGER.debug("Message failed parking the rest of the stream {}", messagesToProcess); + messagesToProcess.forEach(unprocessedMessage -> processedMessages.add(unprocessedMessage.park())); + throw new InterruptMessageStream(); + } + return Void.TYPE; + } + + private static Stack fillStack(List partitionedMessages) { + final var messagesToProcess = new Stack(); + partitionedMessages.stream().sorted(Comparator.comparing(RawMessage::messageSequence)) + .forEach(messagesToProcess::push); + return messagesToProcess; + } + + private static MessageRecordQuery deadLetterQuery(String partitionKey) { + return MessageRecordQueryBuilder.builder() + .partitionKey(partitionKey) + .options(QueryOptions.simple()) + .build(); + } + + + private Map> splitOnPartitionKey(List messageRecords) { + return messageRecords.stream() + .map(this::parseRecord) + .collect(groupingBy(RawMessage::partitionKey)); + } + + private RawMessage parseRecord(MessageRecord messageRecord) { + return new RawMessage(messageRecord.id(), + messageRecord.scheduled(), + messageRecord.expiration(), + messageRecord.priority(), + messageRecord.retryCounter(), + messageRecord.messageState(), + messageRecord.payloadClass(), + messageRecord.payload(), + messageRecord.failedProcessors(), + messageRecord.baseRecord().tenant(), + messageRecord.messageSequence(), + messageRecord.partitionId(), + messageRecord.partitionKey()); + } + + private Uni persistResults(List messages) { + LOGGER.debug("Persisting results for {}", messages); + return Uni.join().all(ack(messages), nack(messages)).andFailFast().replaceWithVoid(); + } + + private Uni nack(List messages) { + final var messagesToRequeue = messages.stream() + .filter(entry -> entry.messageState() == MessageState.RETRY) + .toList(); + if (!messagesToRequeue.isEmpty()) { + LOGGER.debug("re-queuing unhandled messages {}", messagesToRequeue.stream().map(MessageRecord::id).toList()); + return messageQueue.updateByKeyBatch(messagesToRequeue); + } + return Uni.createFrom().voidItem(); + } + + private Uni ack(List messages) { + final var unis = new ArrayList>(); + final var messagesToDrop = messages.stream() + .filter(message -> message.messageState() == MessageState.PROCESSED || DEAD_LETTER_QUEUE_STATES.contains(message.messageState())) + .collect(groupingBy(msg -> msg.baseRecord().tenant())); + final var queries = messagesToDrop.entrySet().stream() + .map(this::messageDropQuery) + .toList(); + final var messagesToMoveToDeadLetter = messages.stream() + .filter(message -> DEAD_LETTER_QUEUE_STATES.contains(message.messageState())) + .map(MessageProcessor::parseDeadLetter) + .toList(); + if (!queries.isEmpty()) { + LOGGER.debug("Dropping messages {}", messagesToDrop); + unis.add( + Multi.createFrom().iterable(queries) + .onItem().transformToUniAndMerge(messageQueue::deleteQuery) + .collect().asList().replaceWithVoid() + ); + } + if (!messagesToMoveToDeadLetter.isEmpty()) { + LOGGER.debug("Moving messages to dead-letter {}", messagesToMoveToDeadLetter); + unis.add(deadLetterQueue.insertBatch(messagesToMoveToDeadLetter)); + } + if (!unis.isEmpty()) { + return Uni.join().all(unis).andCollectFailures().replaceWithVoid(); + } + return Uni.createFrom().voidItem(); + } + + private static DeadLetterRecord parseDeadLetter(MessageRecord messageRecord) { + return new DeadLetterRecord( + messageRecord.id(), + messageRecord.scheduled(), + messageRecord.expiration(), + messageRecord.priority(), + messageRecord.retryCounter(), + messageRecord.messageState(), + messageRecord.payloadClass(), + messageRecord.payload(), + messageRecord.failedProcessors(), + messageRecord.verticleId(), + messageRecord.partitionId(), + messageRecord.partitionKey(), + BaseRecord.newRecord(messageRecord.baseRecord().tenant()) + ); + } + + private MessageRecordQuery messageDropQuery(Map.Entry> entry) { + return MessageRecordQueryBuilder.builder() + .ids( + entry.getValue().stream() + .map(MessageRecord::id) + .toList() + ) + .options(QueryOptions.simple(entry.getKey())) + .build(); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PartitionHashRing.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PartitionHashRing.java new file mode 100644 index 0000000..656f62a --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PartitionHashRing.java @@ -0,0 +1,56 @@ +package io.es4j.infrastructure.pgbroker.messagebroker; + +import io.es4j.infrastructure.pgbroker.exceptions.PartitionNotFound; +import io.es4j.infrastructure.pgbroker.models.MessagePartition; +import io.es4j.infrastructure.pgbroker.models.PartitionKey; +import io.es4j.infrastructure.pgbroker.models.PartitionQuery; +import io.es4j.sql.Repository; +import io.es4j.sql.exceptions.NotFound; +import io.smallrye.mutiny.Uni; +import org.ishugaliy.allgood.consistent.hash.HashRing; +import org.ishugaliy.allgood.consistent.hash.hasher.DefaultHasher; +import org.ishugaliy.allgood.consistent.hash.node.SimpleNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Objects; +import java.util.concurrent.atomic.AtomicBoolean; + + +public class PartitionHashRing { + + private static final AtomicBoolean INITIALIZED = new AtomicBoolean(false); + private static final Logger LOGGER = LoggerFactory.getLogger(PartitionHashRing.class); + private static final HashRing PARTITION_HASH_RING = HashRing.newBuilder() + .name("queue-partitions-hash-ring") // set hash ring fileName + .hasher(DefaultHasher.MURMUR_3) // hash function to distribute partitions + .partitionRate(100000) // number of partitions per node + .build(); + + // todo put a subscription on the partitionId store so that we can update the hash ring dynamically when partitions are added or removed + public static Uni populateHashRing(Repository partitionRepository) { + if (INITIALIZED.compareAndSet(false, true)) { + return partitionRepository.query("select * from queue_partition") + .flatMap(partitions -> partitionRepository.repositoryHandler().vertx().executeBlocking( + Uni.createFrom().item( + () -> { + partitions.forEach(partition -> PARTITION_HASH_RING.add(SimpleNode.of(partition.partitionId()))); + return Void.TYPE; + } + ) + )) + .onFailure(NotFound.class).recoverWithNull().replaceWithVoid(); + } + return Uni.createFrom().voidItem(); + } + + public static String resolve(String partitionKey) { + LOGGER.debug("Locating partition for key {}", partitionKey); + if (Objects.nonNull(partitionKey) && PARTITION_HASH_RING.size() > 0) { + return PARTITION_HASH_RING.locate(partitionKey).orElseThrow(PartitionNotFound::new).getKey(); + } + return "none"; + } + + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PartitionPollingSession.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PartitionPollingSession.java new file mode 100644 index 0000000..ab0561d --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PartitionPollingSession.java @@ -0,0 +1,247 @@ +package io.es4j.infrastructure.pgbroker.messagebroker; + + +import io.es4j.infrastructure.pgbroker.exceptions.PartitionTakenException; +import io.es4j.infrastructure.pgbroker.models.*; +import io.es4j.sql.Repository; +import io.es4j.sql.exceptions.NotFound; +import io.es4j.task.LockLevel; +import io.es4j.task.TimerTaskConfiguration; +import io.es4j.task.TimerTaskDeployer; +import io.smallrye.mutiny.Uni; +import io.smallrye.mutiny.unchecked.Unchecked; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.*; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.atomic.AtomicBoolean; + +public class PartitionPollingSession { + private static final Logger LOGGER = LoggerFactory.getLogger(PartitionPollingSession.class); + private final ConcurrentLinkedQueue messages = new ConcurrentLinkedQueue<>(); + private final Repository partitionRepository; + private final Repository messageQueue; + private final AtomicBoolean claimMessages = new AtomicBoolean(false); + private final AtomicBoolean partitionActive = new AtomicBoolean(true); + private final PgBrokerConfiguration configuration; + private final String verticleId; + private final String partitionId; + public MessagePartition messagePartition; + + + public PartitionPollingSession( + Repository partitionRepository, + Repository messageQueue, + PgBrokerConfiguration configuration, + String verticleId, + String partitionId + ) { + this.partitionRepository = partitionRepository; + this.messageQueue = messageQueue; + this.configuration = configuration; + this.verticleId = verticleId; + this.partitionId = partitionId; + } + + public Uni start(TimerTaskDeployer timerTaskDeployer) { + return takePartition(partitionId, verticleId) + .onFailure(PartitionTakenException.class).retry().withBackOff(Duration.ofSeconds(30)).atMost(1) + .flatMap( + partition -> { + messagePartition = partition; + timerTaskDeployer.deploy(partitionHeartBeat()); + timerTaskDeployer.deploy(partitionPolling()); + claimMessages.set(true); + return Uni.createFrom().voidItem(); + } + ); + } + + public io.es4j.task.TimerTask partitionPolling() { + return new io.es4j.task.TimerTask() { + @Override + public Uni performTask() { + if (claimMessages.get()) { + claimMessages.set(false); + LOGGER.info("Something to claim on partition {}", messagePartition); + if (partitionActive.get()) { + return pollPartition(partitionId, verticleId, configuration) + .onFailure().recoverWithNull().replaceWithVoid(); + } + throw new PartitionTakenException(); + } else { + LOGGER.debug("Nothing to claim on partition {}", messagePartition); + } + return Uni.createFrom().voidItem(); + } + + @Override + public TimerTaskConfiguration configuration() { + return new TimerTaskConfiguration( + LockLevel.NONE, + Duration.ofMillis(25), + Duration.ofMillis(25), + Duration.ofMillis(25), + Optional.of(PartitionTakenException.class) + ); + } + + }; + } + + public io.es4j.task.TimerTask partitionHeartBeat() { + return new io.es4j.task.TimerTask() { + @Override + public Uni performTask() { + return partitionRepository.query( + partitionHeartBeatStatement(), + Map.of( + "partitionId", partitionId, + "verticleId", verticleId + ) + ) + .onFailure().transform( + Unchecked.function((throwable) -> { + LOGGER.error("Lost partition, heartbeat interrupted"); + partitionActive.set(false); + throw new PartitionTakenException(); + } + ) + ).replaceWithVoid(); + } + + @Override + public TimerTaskConfiguration configuration() { + return new TimerTaskConfiguration( + LockLevel.NONE, + Duration.ofSeconds(30), + Duration.ofSeconds(30), + Duration.ofSeconds(30), + Optional.of(PartitionTakenException.class) + ); + } + }; + } + + public Uni close() { + LOGGER.info("Closing polling session for {}", messagePartition); + partitionActive.set(false); + return partitionRepository.updateByKey(messagePartition.release()).replaceWithVoid(); + } + + public List pollMessages() { + final List polledMessages = new ArrayList<>(); + MessageRecord message; + while ((message = messages.poll()) != null) { + polledMessages.add(message); + } + return polledMessages; + } + + private String pollingStatement() { + return "update queue set state = 'PROCESSING', verticle_id = #{deploymentId} where message_id in (" + + " select message_id from queue where " + + " state in ('CREATED','SCHEDULED','RETRY')" + + " and partition_id = #{partitionId} " + + " and (scheduled is null or scheduled <= current_timestamp )" + + " and (expiration is null or expiration >= current_timestamp )" + + " and (retry_counter = 0 or updated + interval '#{retryInterval} seconds' <= now() ) " + + " order by message_sequence for update skip locked limit #{batchSize}" + + " ) returning *;"; + } + + private String recoveryStatement() { + return "update queue set state = 'PROCESSING', verticle_id = #{deploymentId} where message_id in (" + + " select message_id from queue where " + + " state = 'RECOVERY' and partition_id = #{partitionId} " + + " order by message_sequence for update skip locked limit #{batchSize}" + + " ) returning *;"; + } + + private Uni pollPartition(String partitionId, String deploymentId, PgBrokerConfiguration configuration) { + final var recoveredMessages = recoverMessages(partitionId, deploymentId, configuration); + final var claimedMessages = claimMessages(partitionId, deploymentId, configuration); + return Uni.join().all(recoveredMessages, claimedMessages).andCollectFailures() + .onFailure().recoverWithNull().replaceWithVoid(); + } + + private Uni recoverMessages(String partitionId, String deploymentId, PgBrokerConfiguration configuration) { + // this is a hack to recover messages that are stuck in the PROCESSING state... + // this can happen if the verticle crashes before the message is committed + return recoverPartitionMessages(partitionId, deploymentId, configuration) + .onFailure(NotFound.class).recoverWithItem(Collections::emptyList) + .map(msg -> { + LOGGER.info("Recovered {} messages from partitionId {}", msg.size(), partitionId); + msg.stream().map(m -> m.withState(MessageState.RECOVERY)).forEach(messages::add); + return msg; + } + ) + .replaceWithVoid(); + } + + private Uni claimMessages(String partitionId, String deploymentId, PgBrokerConfiguration configuration) { + return messageQueue.query(pollingStatement(), Map.of( + "partitionId", partitionId, + "deploymentId", deploymentId, + "batchSize", configuration.batchSize(), + "retryInterval", Objects.requireNonNullElse(configuration.retryBackOffInterval(), Duration.ofMinutes(5)).getSeconds() + ) + ) + .onFailure(NotFound.class).recoverWithItem(Collections::emptyList) + .map(msg -> { + LOGGER.info("Claimed {} messages from partitionId {}", msg.size(), partitionId); + messages.addAll(msg); + return msg; + } + ) + .replaceWithVoid(); + } + + private Uni> recoverPartitionMessages(String partitionId, String deploymentId, PgBrokerConfiguration configuration) { + return messageQueue.query(recoveryStatement(), Map.of( + "partitionId", partitionId, + "deploymentId", deploymentId, + "batchSize", configuration.batchSize() + ) + ); + } + + + private String claimPartitionStatement() { + return ( + " update queue_partition set updated = now(), locked = true, verticle_id = #{verticleId} where partition_id = #{partitionId} and (locked = false or updated + interval '1 minute' <= now() ) returning *" + ); + } + + private String partitionHeartBeatStatement() { + return ( + "update queue_partition set updated = now() where partition_id = #{partitionId} and verticle_id = #{verticleId} and locked = true returning *" + ); + } + + public Uni takePartition(String partitionId, String deploymentId) { + return partitionRepository.query(claimPartitionStatement(), + Map.of( + "verticleId", deploymentId, + "partitionId", partitionId + ) + ) + .onFailure().transform(Unchecked.function(throwable -> { + if (throwable instanceof NotFound) { + LOGGER.info("Partition not available {}", partitionId); + } else { + LOGGER.info("Partition claiming dropped {} ", partitionId, throwable); + } + throw new PartitionTakenException(); + })) + .map(partition -> partition.stream().findFirst().orElseThrow()); + } + + public void signalMessage() { + claimMessages.set(true); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PgChannel.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PgChannel.java new file mode 100644 index 0000000..eabba7a --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/PgChannel.java @@ -0,0 +1,82 @@ +package io.es4j.infrastructure.pgbroker.messagebroker; + + +import io.es4j.infrastructure.pgbroker.models.*; +import io.es4j.sql.Repository; +import io.es4j.sql.misc.Constants; +import io.es4j.sql.misc.EnvVars; +import io.es4j.task.TimerTaskDeployer; +import io.smallrye.mutiny.Uni; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class PgChannel { + private static final Logger LOGGER = LoggerFactory.getLogger(PgChannel.class); + public static final AtomicBoolean REFRESHER_DEPLOYED = new AtomicBoolean(false); + private final Repository messageQueue; + private final Repository deadLetterQueue; + private final io.vertx.mutiny.pgclient.pubsub.PgSubscriber pgSubscriber; + private final Repository partitionRepository; + private final String verticleId; + private TimerTaskDeployer timerTasks; + private SessionManager sessionManager; + + + public PgChannel( + Repository messageQueue, + Repository deadLetterQueue, + Repository partitionRepository, + io.vertx.mutiny.pgclient.pubsub.PgSubscriber pgSubscriber, + String verticleId + ) { + this.messageQueue = messageQueue; + this.deadLetterQueue = deadLetterQueue; + this.pgSubscriber = pgSubscriber; + this.partitionRepository = partitionRepository; + this.verticleId = verticleId; + } + + public Uni stop() { + return sessionManager.close().flatMap(avoid -> pgSubscriber.close()); + } + + public Uni start(ConsumerManager consumerManager) { + this.timerTasks = new TimerTaskDeployer(messageQueue.repositoryHandler().vertx()); + if (!REFRESHER_DEPLOYED.get()) { + SessionRefresher.refreshTimers(consumerManager, timerTasks, messageQueue); + } + return PartitionHashRing.populateHashRing(partitionRepository) + .flatMap(avoid -> { + this.sessionManager = new SessionManager( + verticleId, + consumerManager, + messageQueue, + deadLetterQueue, + partitionRepository, + timerTasks + ); + sessionManager.start(); + final var pgChannel = pgSubscriber.channel(parseChannel()); + pgChannel.handler( + partitionId -> { + LOGGER.info("Incoming message for partition {}", partitionId); + if (partitionId.isEmpty()) { + throw new IllegalArgumentException("partition not present in channel message"); + } + sessionManager.signal(partitionId); + }) + .endHandler(() -> LOGGER.info("channel stopped")) + .subscribeHandler(() -> LOGGER.info("channel started")) + .exceptionHandler(throwable -> LOGGER.error("channel error", throwable)); + return pgSubscriber.connect(); + }); + } + + private String parseChannel() { + return messageQueue.repositoryHandler().configuration().getString(Constants.SCHEMA, EnvVars.SCHEMA) + "-queue-channel"; + } + + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/SessionManager.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/SessionManager.java new file mode 100644 index 0000000..9bb96a9 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/SessionManager.java @@ -0,0 +1,164 @@ +package io.es4j.infrastructure.pgbroker.messagebroker; + + +import io.es4j.infrastructure.pgbroker.exceptions.PartitionTakenException; +import io.es4j.infrastructure.pgbroker.models.*; +import io.es4j.sql.Repository; +import io.es4j.task.LockLevel; +import io.es4j.task.TimerTaskConfiguration; +import io.es4j.task.TimerTaskDeployer; +import io.smallrye.mutiny.Multi; +import io.smallrye.mutiny.Uni; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; + +public class SessionManager { + private final static Map partitionedSessions = new HashMap<>(); + private final ConcurrentPollingSession concurrentSession; + private final Repository messageQueue; + private final Repository partitionRepository; + private final String verticleId; + private final Map lastAttempt = new HashMap<>(); + private final ConsumerManager consumerManager; + private final MessageProcessor messageProcessor; + private TimerTaskDeployer timerTasks; + + private static final Logger LOGGER = LoggerFactory.getLogger(SessionManager.class); + + public Uni close() { + timerTasks.close(); + if (!partitionedSessions.isEmpty()) { + return Multi.createFrom().iterable(partitionedSessions.values()) + .onItem().transformToUniAndMerge( + PartitionPollingSession::close + ).collect().asList() + .replaceWithVoid(); + } + return Uni.createFrom().voidItem(); + } + + public SessionManager( + final String verticleId, + final ConsumerManager consumerManager, + final Repository messageQueue, + final Repository deadLetterQueue, + final Repository partitionRepository, + final TimerTaskDeployer timerTasks + ) { + this.consumerManager = consumerManager; + this.concurrentSession = new ConcurrentPollingSession(messageQueue, consumerManager.pgBrokerConfiguration(), verticleId); + this.messageQueue = messageQueue; + this.timerTasks = timerTasks; + this.partitionRepository = partitionRepository; + this.verticleId = verticleId; + this.messageProcessor = new MessageProcessor(consumerManager, messageQueue, deadLetterQueue); + } + + + public void start() { + timerTasks.deploy(concurrentSession.provideTask()); + timerTasks.deploy(processorTask()); + } + + + private io.es4j.task.TimerTask processorTask() { + return new io.es4j.task.TimerTask() { + @Override + public Uni performTask() { + final var unis = new ArrayList>(); + if (!partitionedSessions.isEmpty()) { + unis.add(Multi.createFrom().iterable(partitionedSessions.entrySet()) + .onItem().transformToUniAndMerge(partitionPollingSession -> { + final var messages = partitionPollingSession.getValue().pollMessages(); + if (!messages.isEmpty()) { + LOGGER.info("Processing {} messages in partition {}", messages.size(), partitionPollingSession.getKey()); + return messageProcessor.processPartitions(messages); + } + return Uni.createFrom().voidItem(); + } + ).collect().asList() + .replaceWithVoid() + ); + } + final var messages = concurrentSession.pollMessages(); + if (!messages.isEmpty()) { + LOGGER.info("Processing {} messages", messages.size()); + unis.add(messageProcessor.processConcurrent(messages)); + } + if (!unis.isEmpty()) { + return Uni.join().all(unis).andCollectFailures().onFailure() + .invoke(throwable -> LOGGER.warn("Consumer dropped exception", throwable)) + .onFailure().recoverWithNull().replaceWithVoid(); + } + return Uni.createFrom().voidItem(); + + } + + @Override + public TimerTaskConfiguration configuration() { + return new TimerTaskConfiguration( + LockLevel.NONE, + Duration.ofMillis(10), + Duration.ofMillis(10), + Duration.ofMillis(10), + Optional.empty() + ); + } + }; + } + + + public boolean contains(String partitionId) { + return partitionedSessions.containsKey(partitionId); + } + + public void signal(String partitionId) { + if (partitionId.equals("none")) { + concurrentSession.signalMessage(); + } else if (this.contains(partitionId)) { + lastAttempt.remove(partitionId); + Objects.requireNonNull(partitionedSessions.get(partitionId)).signalMessage(); + } else { + startPartitionSession(partitionId); + } + } + + private void startPartitionSession(String partitionId) { + if (lastAttempt.containsKey(partitionId)) { + final var lastRetry = lastAttempt.get(partitionId); + LOGGER.debug("Partition was previously taken by another verticle, last claim attempt {}", lastRetry); + final var has5MinutesPassedSinceLasAttempt = Instant.now().isAfter(lastRetry.plus(5, ChronoUnit.MINUTES)); + if (has5MinutesPassedSinceLasAttempt) { + lastAttempt.put(partitionId, Instant.now()); + startSession(partitionId); + } + } else { + lastAttempt.put(partitionId, Instant.now()); + startSession(partitionId); + } + } + + private void startSession(String partitionId) { + LOGGER.debug("Trying to claim partition {}", partitionId); + final var partitionPollingSession = new PartitionPollingSession(partitionRepository, messageQueue, consumerManager.pgBrokerConfiguration(), verticleId, partitionId); + partitionPollingSession.start(timerTasks) + .subscribe() + .with(item -> { + LOGGER.error("subscribed to partition {}", partitionId); + partitionedSessions.put(partitionId, partitionPollingSession); + }, + throwable -> { + if (throwable instanceof PartitionTakenException) { + LOGGER.info("partition already taken {}", partitionId); + } else { + LOGGER.error("partition claiming dropped exception", throwable); + } + } + ); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/SessionRefresher.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/SessionRefresher.java new file mode 100644 index 0000000..a2ebf8f --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/messagebroker/SessionRefresher.java @@ -0,0 +1,102 @@ +package io.es4j.infrastructure.pgbroker.messagebroker; + +import io.es4j.infrastructure.pgbroker.models.*; +import io.es4j.sql.Repository; +import io.es4j.task.LockLevel; +import io.es4j.task.TimerTask; +import io.es4j.task.TimerTaskConfiguration; +import io.es4j.task.TimerTaskDeployer; +import io.smallrye.mutiny.Uni; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.Optional; + +public class SessionRefresher { + private static final Logger LOGGER = LoggerFactory.getLogger(SessionRefresher.class); + + public static void refreshTimers(ConsumerManager consumerManager, TimerTaskDeployer timerTasks, Repository messageQueue) { + timerTasks.deploy( + new TimerTask() { + @Override + public Uni performTask() { + LOGGER.info("Retry interval refresher"); + return messageQueue.repositoryHandler().sqlClient().query(initiateRetry(consumerManager.pgBrokerConfiguration())).execute() + .replaceWithVoid(); + } + + @Override + public TimerTaskConfiguration configuration() { + return new TimerTaskConfiguration( + LockLevel.CLUSTER_WIDE, + Duration.ofMinutes(15), + Duration.ofMinutes(15), + Duration.ofMinutes(15), + Optional.empty() + ); + } + } + + ); + timerTasks.deploy( + new TimerTask() { + @Override + public Uni performTask() { + LOGGER.info("Trying to recover lost messages"); + return messageQueue.repositoryHandler().sqlClient().query(recoverLostMessages(consumerManager.pgBrokerConfiguration())).execute() + .replaceWithVoid(); + } + + @Override + public TimerTaskConfiguration configuration() { + return new TimerTaskConfiguration( + LockLevel.CLUSTER_WIDE, + Duration.ofMinutes(15), + Duration.ofMinutes(15), + Duration.ofMinutes(15), + Optional.empty() + ); + } + } + ); + if (consumerManager.pgBrokerConfiguration().idempotency()) { + timerTasks.deploy( + new TimerTask() { + @Override + public Uni performTask() { + LOGGER.info("Purging tx table"); + return messageQueue.repositoryHandler().sqlClient().query(purgeTransactions(consumerManager.pgBrokerConfiguration())).execute() + .replaceWithVoid(); + } + + @Override + public TimerTaskConfiguration configuration() { + return new TimerTaskConfiguration( + LockLevel.CLUSTER_WIDE, + Duration.ofHours(4), + Duration.ofHours(4), + Duration.ofHours(4), + Optional.empty() + ); + } + } + ); + } + + } + + public static String purgeTransactions(PgBrokerConfiguration configuration) { + return "delete from queue_tx where inserted <= current_timestamp - interval '" + configuration.purgeTxAfter().toDays() + " days'"; + } + + public static String recoverLostMessages(PgBrokerConfiguration configuration) { + return "update queue set rec_version = rec_version + 1, state = 'RECOVERY' where " + + " state = 'PROCESSING' and updated + interval '" + configuration.maxProcessingTime().getSeconds() + " seconds' <= current_timestamp;"; + } + + public static String initiateRetry(PgBrokerConfiguration configuration) { + return "update queue set rec_version = rec_version + 1 where " + + " state = 'RETRY' and updated + interval '" + configuration.retryBackOffInterval().getSeconds() + " seconds' <= current_timestamp;"; + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/ConsumerManager.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/ConsumerManager.java new file mode 100644 index 0000000..0700283 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/ConsumerManager.java @@ -0,0 +1,116 @@ +package io.es4j.infrastructure.pgbroker.models; + + +import io.es4j.infrastructure.pgbroker.ConsumerTransactionProvider; +import io.es4j.infrastructure.pgbroker.QueueConsumer; +import io.es4j.infrastructure.pgbroker.exceptions.DuplicateMessage; +import io.es4j.infrastructure.pgbroker.exceptions.ConsumerExeception; +import io.es4j.infrastructure.pgbroker.exceptions.InvalidProcessorException; +import io.es4j.infrastructure.pgbroker.exceptions.MessageParsingException; +import io.smallrye.mutiny.Uni; +import io.vertx.mutiny.core.Vertx; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + + +public record ConsumerManager( + PgBrokerConfiguration pgBrokerConfiguration, + List consumerWraps, + ConsumerTransactionProvider consumerTransactionProvider, + Vertx vertx +) { + private static final Logger LOGGER = LoggerFactory.getLogger(ConsumerManager.class); + + public Uni consumeMessage(RawMessage rawMessage) { + QueueConsumer processor; + Message parsedMessage; + try { + processor = resolveProcessor(rawMessage); + parsedMessage = parseMessage(processor, rawMessage); + } catch (Exception exception) { + return Uni.createFrom().item(rawMessage.withState(MessageState.FATAL_FAILURE)); + } + return consumerTransactionProvider.transaction( + processor.getClass().getName(), parsedMessage, (msg, taskTransaction) -> { + try { + if (processor.blockingProcessor()) { + return vertx.executeBlocking(process(parsedMessage, processor, taskTransaction) + .onFailure().transform(ConsumerExeception::new) + ); + } + return process(parsedMessage, processor, taskTransaction) + .onFailure().transform(ConsumerExeception::new); + } catch (Exception exception) { + throw new ConsumerExeception(exception); + } + } + ) + .onItemOrFailure().transform( + (avoid, throwable) -> { + if (throwable != null) { + if (throwable instanceof DuplicateMessage duplicateMessage) { + LOGGER.debug("Duplicated message will be ignored {} ", rawMessage); + return rawMessage.withState(MessageState.PROCESSED); + } else { + return retryableFailure(pgBrokerConfiguration, rawMessage, throwable, processor); + } + } else { + LOGGER.debug("Consumer {} has correctly processed the message {} ", processor.getClass().getName(), rawMessage); + return rawMessage.withState(MessageState.PROCESSED); + } + } + ); + } + + private Uni process(Message message, QueueConsumer processor, ConsumerTransaction consumerTransaction) { + return processor.process(message.payload(), consumerTransaction); + } + + private QueueConsumer resolveProcessor(RawMessage messageRecord) { + return consumerWraps.stream() + .filter(processor -> processor.doesMessageMatch(messageRecord)) + .findFirst() + .map(processor -> processor.resolveProcessor(messageRecord.tenant())) + .orElseThrow(() -> { + LOGGER.error("Unable to find processor for message -> " + messageRecord.id()); + return new InvalidProcessorException(); + }); + } + + private Message parseMessage(QueueConsumer processor, RawMessage rawMessage) { + final Class tClass; + try { + tClass = Class.forName(rawMessage.payloadClass()); + return new Message<>( + rawMessage.id(), + rawMessage.tenant(), + String.valueOf(rawMessage.partitionId()), + rawMessage.scheduled(), + rawMessage.expiration(), + rawMessage.priority(), + processor.parse(rawMessage.payload(), tClass) + ); + } catch (ClassNotFoundException e) { + LOGGER.error("Could not find class for message -> " + rawMessage.id()); + throw new MessageParsingException(); + } + } + + private RawMessage retryableFailure(PgBrokerConfiguration configuration, RawMessage rawMessage, Throwable throwable, QueueConsumer processor) { + MessageState failureState; + if (processor.fatalExceptions().stream().anyMatch(f -> f.isAssignableFrom(throwable.getCause().getClass()))) { + LOGGER.error("Fatal failure {}, message will be sent to dead letter queue {} ", processor.getClass().getName(), rawMessage, throwable.getCause()); + failureState = MessageState.FATAL_FAILURE; + } else if (rawMessage.retryCounter() + 1 > configuration.maxRetry()) { + LOGGER.error("Retries exhausted for {} ", rawMessage, throwable.getCause()); + failureState = MessageState.RETRIES_EXHAUSTED; + } else { + LOGGER.error(String.format("Failure in task processor %s || Message %s will be queued for retry -> ", processor.getClass().getName(), rawMessage.id()), throwable.getCause()); + failureState = MessageState.RETRY; + } + return rawMessage.withFailure(failureState, throwable); + } + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/ConsumerTransaction.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/ConsumerTransaction.java new file mode 100644 index 0000000..64eb746 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/ConsumerTransaction.java @@ -0,0 +1,10 @@ +package io.es4j.infrastructure.pgbroker.models; + +public record ConsumerTransaction( + Object connection +) { + + public T getDelegate(Class connectionClass) { + return connectionClass.cast(connection); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorWrapper.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/ConsumerWrap.java similarity index 58% rename from es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorWrapper.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/ConsumerWrap.java index e6e43b8..fe849d7 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageProcessorWrapper.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/ConsumerWrap.java @@ -1,19 +1,21 @@ -package io.es4j.infrastructure.messagebroker.models; +package io.es4j.infrastructure.pgbroker.models; -import io.es4j.infrastructure.messagebroker.MessageProcessor; + + +import io.es4j.infrastructure.pgbroker.QueueConsumer; import java.util.List; import java.util.Map; -public record MessageProcessorWrapper ( +public record ConsumerWrap ( String deploymentId, - MessageProcessor defaultProcessor, - Map, MessageProcessor> customProcessors, + QueueConsumer defaultProcessor, + Map, QueueConsumer> customProcessors, Class payloadClass ) { - public MessageProcessor resolveProcessor(String tenant) { + public QueueConsumer resolveProcessor(String tenant) { return customProcessors.entrySet().stream() .filter(wrapper -> wrapper.getKey().stream().anyMatch(tenant::equals)) .map(Map.Entry::getValue) diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterKey.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/DeadLetterKey.java similarity index 77% rename from es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterKey.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/DeadLetterKey.java index c8ccb7c..52eccfc 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/DeadLetterKey.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/DeadLetterKey.java @@ -1,9 +1,11 @@ -package io.es4j.infrastructure.messagebroker.postgres.models; +package io.es4j.infrastructure.pgbroker.models; + import io.es4j.sql.models.RepositoryRecordKey; import io.soabase.recordbuilder.core.RecordBuilder; @RecordBuilder + public record DeadLetterKey( String messageID, String tenant diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/DeadLetterRecord.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/DeadLetterRecord.java new file mode 100644 index 0000000..7271c24 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/DeadLetterRecord.java @@ -0,0 +1,34 @@ +package io.es4j.infrastructure.pgbroker.models; + + +import io.es4j.sql.models.BaseRecord; +import io.es4j.sql.models.RepositoryRecord; +import io.soabase.recordbuilder.core.RecordBuilder; +import io.vertx.core.json.JsonObject; + +import java.time.Instant; + +@RecordBuilder +public record DeadLetterRecord( + String id, + Instant scheduled, + Instant expiration, + Integer priority, + Integer retryCounter, + MessageState messageState, + String payloadClass, + JsonObject payload, + JsonObject failedProcessors, + String verticleId, + String partitionId, + String partitionKey, + BaseRecord baseRecord +) implements RepositoryRecord { + + @Override + public DeadLetterRecord with(BaseRecord baseRecord) { + return DeadLetterRecordBuilder.builder(this) + .baseRecord(baseRecord) + .build(); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/Message.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/Message.java new file mode 100644 index 0000000..5b78c47 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/Message.java @@ -0,0 +1,38 @@ +package io.es4j.infrastructure.pgbroker.models; + + +import io.soabase.recordbuilder.core.RecordBuilder; + +import java.time.Instant; +@RecordBuilder +public record Message( + String messageId, + String tenant, + String partitionKey, + Instant scheduled, + Instant expiration, + Integer priority, + T payload +) { + + public Message { + if (priority != null && priority > 10) { + throw new IllegalArgumentException("Max priority is 10"); + } + if (messageId == null) { + throw new IllegalArgumentException("Id must not be null"); + } + } + + public static

Message

simple(String messageId, P payload) { + return new Message<>( + messageId, + "default", + null, + null, + null, + null, + payload + ); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageID.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageID.java similarity index 70% rename from es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageID.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageID.java index d00427c..a6a54e9 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/models/MessageID.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageID.java @@ -1,5 +1,4 @@ -package io.es4j.infrastructure.messagebroker.models; - +package io.es4j.infrastructure.pgbroker.models; import io.soabase.recordbuilder.core.RecordBuilder; diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessagePartition.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessagePartition.java new file mode 100644 index 0000000..782f7cc --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessagePartition.java @@ -0,0 +1,24 @@ +package io.es4j.infrastructure.pgbroker.models; + + +import io.es4j.sql.models.BaseRecord; +import io.es4j.sql.models.RepositoryRecord; +import io.soabase.recordbuilder.core.RecordBuilder; + +@RecordBuilder +public record MessagePartition( + String partitionId, + String deploymentId, + Boolean locked, + BaseRecord baseRecord +) implements RepositoryRecord { + + @Override + public MessagePartition with(BaseRecord baseRecord) { + return new MessagePartition(partitionId, deploymentId, locked, baseRecord); + } + + public MessagePartition release() { + return new MessagePartition(partitionId, deploymentId, false, baseRecord); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecord.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageRecord.java similarity index 54% rename from es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecord.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageRecord.java index 311c8ee..b868dd8 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecord.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageRecord.java @@ -1,15 +1,16 @@ -package io.es4j.infrastructure.messagebroker.postgres.models; +package io.es4j.infrastructure.pgbroker.models; + import io.es4j.sql.models.BaseRecord; import io.es4j.sql.models.RepositoryRecord; import io.soabase.recordbuilder.core.RecordBuilder; -import io.es4j.infrastructure.messagebroker.models.MessageState; -import io.es4j.infrastructure.messagebroker.models.RawMessage; import io.vertx.core.json.JsonObject; import java.time.Instant; import java.util.Map; +import static java.util.Objects.requireNonNullElse; + @RecordBuilder public record MessageRecord( String id, @@ -22,80 +23,37 @@ public record MessageRecord( JsonObject payload, JsonObject failedProcessors, String verticleId, + Long messageSequence, + String partitionId, + String partitionKey, BaseRecord baseRecord ) implements RepositoryRecord { - - public static MessageRecord simpleTask(String id, String tenant, Object payload) { - return new MessageRecord( - id, - null, - null, - 0, - 0, - MessageState.CREATED, - null, - JsonObject.mapFrom(payload), - null, - null, - BaseRecord.newRecord(tenant) - ); - } - - public static MessageRecord task(String id, String tenant, Object payload, Instant scheduled, Instant expiration, Integer priority) { - return new MessageRecord( - id, - scheduled, - expiration, - priority, - 0, - MessageState.CREATED, - null, - JsonObject.mapFrom(payload), - null, - null, - BaseRecord.newRecord(tenant) - ); - } - - public static MessageRecord priorityTask(String id, String tenant, Object payload, Integer priority) { - return new MessageRecord( - id, - null, - null, - priority, - 0, - MessageState.CREATED, - null, - JsonObject.mapFrom(payload), - null, - null, - BaseRecord.newRecord(tenant) - ); - } - public MessageRecord withState(final MessageState retry) { - return new MessageRecord(id, scheduled, expiration, priority, 0, retry, payloadClass, payload, failedProcessors, verticleId, baseRecord); + return new MessageRecord(id, scheduled, expiration, priority, 0, retry, payloadClass, payload, failedProcessors, verticleId, messageSequence, partitionId, partitionKey, baseRecord); } public MessageRecord increaseCounter() { - return new MessageRecord(id, scheduled, expiration, priority, retryCounter + 1, MessageState.RETRY, payloadClass, payload, failedProcessors, verticleId, baseRecord); + return new MessageRecord(id, scheduled, expiration, priority, retryCounter + 1, MessageState.RETRY, payloadClass, payload, failedProcessors, verticleId, messageSequence, partitionId, partitionKey, baseRecord); } public MessageRecord withFailures(Map failures) { if (!failures.isEmpty()) { - return new MessageRecord(id, scheduled, expiration, priority, retryCounter + 1, MessageState.RETRY, payloadClass, payload, JsonObject.mapFrom(failures), verticleId, baseRecord); + return new MessageRecord(id, scheduled, expiration, priority, retryCounter + 1, MessageState.RETRY, payloadClass, payload, JsonObject.mapFrom(failures), verticleId, messageSequence, partitionId, partitionKey, baseRecord); } - return new MessageRecord(id, scheduled, expiration, priority, retryCounter, MessageState.PROCESSED, payloadClass, payload, this.failedProcessors, verticleId, baseRecord); + return new MessageRecord(id, scheduled, expiration, priority, retryCounter, MessageState.PROCESSED, payloadClass, payload, this.failedProcessors, verticleId, messageSequence, partitionId, partitionKey, baseRecord); } @Override public MessageRecord with(BaseRecord baseRecord) { - return new MessageRecord(id, scheduled, expiration, priority, retryCounter, messageState, payloadClass, payload, failedProcessors, verticleId, baseRecord); + return new MessageRecord( + id, + scheduled, + expiration, priority, retryCounter, messageState, payloadClass, payload, failedProcessors, verticleId, messageSequence, partitionId, partitionKey, baseRecord); } public static MessageRecord from(RawMessage rawMessage) { - return new MessageRecord ( + return new MessageRecord( rawMessage.id(), rawMessage.scheduled(), rawMessage.expiration(), @@ -106,7 +64,10 @@ public static MessageRecord from(RawMessage rawMessage) { rawMessage.payload(), rawMessage.failures(), null, - BaseRecord.newRecord(rawMessage.tenant()) + rawMessage.messageSequence(), + rawMessage.partitionId(), + rawMessage.partitionKey(), + BaseRecord.newRecord(requireNonNullElse(rawMessage.tenant(), "default")) ); } } diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordID.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageRecordID.java similarity index 77% rename from es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordID.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageRecordID.java index 9ef6ca2..5161627 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordID.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageRecordID.java @@ -1,4 +1,5 @@ -package io.es4j.infrastructure.messagebroker.postgres.models; +package io.es4j.infrastructure.pgbroker.models; + import io.es4j.sql.models.RepositoryRecordKey; import io.soabase.recordbuilder.core.RecordBuilder; diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordQuery.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageRecordQuery.java similarity index 80% rename from es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordQuery.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageRecordQuery.java index 04235e0..a9448e5 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageRecordQuery.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageRecordQuery.java @@ -1,13 +1,12 @@ -package io.es4j.infrastructure.messagebroker.postgres.models; +package io.es4j.infrastructure.pgbroker.models; + import io.es4j.sql.models.Query; import io.es4j.sql.models.QueryOptions; import io.soabase.recordbuilder.core.RecordBuilder; -import io.es4j.infrastructure.messagebroker.models.MessageState; import java.time.Instant; import java.util.List; - @RecordBuilder public record MessageRecordQuery( List ids, @@ -20,6 +19,9 @@ public record MessageRecordQuery( Integer priorityTo, Integer retryCounterFrom, Integer retryCounterTo, + String payloadClass, + String partition, + String partitionKey, QueryOptions options ) implements Query { } diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageState.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageState.java new file mode 100644 index 0000000..c9031b9 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageState.java @@ -0,0 +1,5 @@ +package io.es4j.infrastructure.pgbroker.models; + +public enum MessageState { + CREATED, PROCESSING, SCHEDULED, EXPIRED, RETRY, RETRIES_EXHAUSTED, RECOVERY, PROCESSED, FATAL_FAILURE, PARKED +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageTransaction.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageTransaction.java new file mode 100644 index 0000000..a513dda --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageTransaction.java @@ -0,0 +1,27 @@ +package io.es4j.infrastructure.pgbroker.models; + + +import io.es4j.sql.models.BaseRecord; +import io.es4j.sql.models.RepositoryRecord; +import io.soabase.recordbuilder.core.RecordBuilder; + +@RecordBuilder +public record MessageTransaction( + String id, + String processorClass, + String messageClass, + BaseRecord baseRecord +) implements RepositoryRecord { + + public MessageTransaction(String id, String processorClass, String messageClass, BaseRecord baseRecord) { + this.id = id; + this.processorClass = processorClass; + this.messageClass = messageClass; + this.baseRecord = baseRecord; + } + + @Override + public MessageTransaction with(BaseRecord baseRecord) { + return new MessageTransaction(id, processorClass, messageClass, baseRecord); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionID.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageTransactionID.java similarity index 78% rename from es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionID.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageTransactionID.java index f6b8619..c2346a7 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionID.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageTransactionID.java @@ -1,4 +1,5 @@ -package io.es4j.infrastructure.messagebroker.postgres.models; +package io.es4j.infrastructure.pgbroker.models; + import io.es4j.sql.models.RepositoryRecordKey; import io.soabase.recordbuilder.core.RecordBuilder; diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionQuery.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageTransactionQuery.java similarity index 82% rename from es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionQuery.java rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageTransactionQuery.java index 283911f..7c93105 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/messagebroker/postgres/models/MessageTransactionQuery.java +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/MessageTransactionQuery.java @@ -1,4 +1,7 @@ -package io.es4j.infrastructure.messagebroker.postgres.models; +package io.es4j.infrastructure.pgbroker.models; + + + import io.es4j.sql.models.Query; @@ -6,7 +9,6 @@ import io.soabase.recordbuilder.core.RecordBuilder; import java.util.List; - @RecordBuilder public record MessageTransactionQuery( List ids, diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PartitionKey.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PartitionKey.java new file mode 100644 index 0000000..fffc04c --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PartitionKey.java @@ -0,0 +1,11 @@ +package io.es4j.infrastructure.pgbroker.models; + + +import io.es4j.sql.models.RepositoryRecordKey; +import io.soabase.recordbuilder.core.RecordBuilder; + +@RecordBuilder +public record PartitionKey( + String partitionId +) implements RepositoryRecordKey { +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PartitionQuery.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PartitionQuery.java new file mode 100644 index 0000000..33ef4ed --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PartitionQuery.java @@ -0,0 +1,15 @@ +package io.es4j.infrastructure.pgbroker.models; + + + +import io.es4j.sql.models.Query; +import io.es4j.sql.models.QueryOptions; +import io.soabase.recordbuilder.core.RecordBuilder; + +import java.util.List; +@RecordBuilder +public record PartitionQuery( + List lockIds, + QueryOptions options +) implements Query { +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PgBrokerConfiguration.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PgBrokerConfiguration.java new file mode 100644 index 0000000..0460e8b --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/PgBrokerConfiguration.java @@ -0,0 +1,45 @@ +package io.es4j.infrastructure.pgbroker.models; + + +import io.es4j.infrastructure.pgbroker.ConsumerTransactionProvider; +import io.es4j.infrastructure.pgbroker.vertx.VertxConsumerTransaction; +import io.soabase.recordbuilder.core.RecordBuilder; + +import java.time.Duration; + +@RecordBuilder +public record PgBrokerConfiguration( + Boolean bootstrapQueue, + Boolean idempotency, + Duration purgeTxAfter, + ConsumerTransactionProvider consumerTransactionProvider, + Duration emptyBackOff, + Duration throttle, + Integer concurrency, + Long errorBackOffInMinutes, + Duration retryBackOffInterval, + Duration maxProcessingTime, + Long batchSize, + Integer maxRetry, + Duration maintenanceEvery +) { + + + public static PgBrokerConfiguration defaultConfiguration() { + return PgBrokerConfigurationBuilder.builder() + .bootstrapQueue(false) + .idempotency(true) + .concurrency(100) + .purgeTxAfter(Duration.ofDays(5)) + .retryBackOffInterval(Duration.ofMillis(10)) + .maxProcessingTime(Duration.ofMinutes(30)) + .consumerTransactionProvider(new VertxConsumerTransaction()) + .batchSize(10L) + .maxRetry(5) + .throttle(Duration.ofMillis(10)) + .maintenanceEvery(Duration.ofMinutes(30)) + .build(); + } + + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/RawMessage.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/RawMessage.java new file mode 100644 index 0000000..7375653 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/models/RawMessage.java @@ -0,0 +1,54 @@ +package io.es4j.infrastructure.pgbroker.models; + +import io.soabase.recordbuilder.core.RecordBuilder; +import io.vertx.core.json.JsonObject; + +import java.time.Instant; + +@RecordBuilder +public record RawMessage( + String id, + Instant scheduled, + Instant expiration, + Integer priority, + Integer retryCounter, + MessageState messageState, + String payloadClass, + JsonObject payload, + JsonObject failures, + String tenant, + Long messageSequence, + String partitionId, + String partitionKey +) { + + public RawMessage withState(MessageState messageState) { + return RawMessageBuilder.builder(this) + .messageState(messageState) + .build(); + } + + public RawMessage withFailure(MessageState messageState, Throwable throwable) { + return RawMessageBuilder.builder(this) + .retryCounter(retryCounter + 1) + .messageState(messageState) + .failures( + new JsonObject() + .put("throwable", throwable.toString()) + .put("cause", throwable.getCause() != null ? throwable.getCause().toString() : null) + ) + .build(); + } + + public RawMessage copyStateAndFailures(RawMessage source) { + return RawMessageBuilder.builder(this) + .retryCounter(retryCounter + 1) + .messageState(source.messageState) + .failures(source.failures) + .build(); + } + + public RawMessage park() { + return withState(MessageState.PARKED); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/vertx/VertxConsumerTransaction.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/vertx/VertxConsumerTransaction.java new file mode 100644 index 0000000..5dfc53c --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/vertx/VertxConsumerTransaction.java @@ -0,0 +1,37 @@ +package io.es4j.infrastructure.pgbroker.vertx; + +import io.es4j.infrastructure.pgbroker.ConsumerTransactionProvider; +import io.es4j.infrastructure.pgbroker.exceptions.DuplicateMessage; +import io.es4j.infrastructure.pgbroker.mappers.MessageTransactionMapper; +import io.es4j.infrastructure.pgbroker.models.*; +import io.es4j.sql.Repository; +import io.es4j.sql.RepositoryHandler; +import io.es4j.sql.exceptions.Conflict; +import io.es4j.sql.models.BaseRecord; +import io.smallrye.mutiny.Uni; +import java.util.function.BiFunction; + +public class VertxConsumerTransaction implements ConsumerTransactionProvider { + private Repository transactionStore; + + @Override + public void start(RepositoryHandler repositoryHandler) { + this.transactionStore = new Repository<>(MessageTransactionMapper.INSTANCE, repositoryHandler); + } + + @Override + public Uni transaction(String processorClass, Message message, BiFunction, ConsumerTransaction, Uni> function) { + return transactionStore.transaction(sqlConnection -> transactionStore.insert( + new MessageTransaction( + message.messageId(), + processorClass, + message.payload().getClass().getName(), + BaseRecord.newRecord(message.tenant()) + ), + sqlConnection + ) + .onFailure(Conflict.class).transform(DuplicateMessage::new) + .flatMap(avoid -> function.apply(message, new ConsumerTransaction(sqlConnection))) + ); + } +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/vertx/VertxMessageProducer.java b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/vertx/VertxMessageProducer.java new file mode 100644 index 0000000..3de910e --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/java/io/es4j/infrastructure/pgbroker/vertx/VertxMessageProducer.java @@ -0,0 +1,169 @@ +package io.es4j.infrastructure.pgbroker.vertx; + + +import io.es4j.infrastructure.pgbroker.messagebroker.PartitionHashRing; +import io.es4j.infrastructure.pgbroker.exceptions.ProducerExeception; +import io.es4j.infrastructure.pgbroker.mappers.MessageQueueMapper; +import io.es4j.infrastructure.pgbroker.models.*; +import io.es4j.sql.Repository; +import io.es4j.sql.RepositoryHandler; +import io.es4j.sql.models.BaseRecord; +import io.smallrye.mutiny.Uni; +import io.vertx.core.json.JsonObject; +import io.vertx.mutiny.sqlclient.SqlConnection; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + + +public class VertxMessageProducer { + private static final Logger LOGGER = LoggerFactory.getLogger(VertxMessageProducer.class); + + private final Repository queue; + + public VertxMessageProducer(RepositoryHandler repositoryHandler) { + this.queue = new Repository<>(MessageQueueMapper.INSTANCE, repositoryHandler); + } + + public Uni enqueue(Message message, ConsumerTransaction consumerTransaction) { + log(message); + final var queueEntry = new MessageRecord( + message.messageId(), + message.scheduled(), + message.expiration(), + message.priority(), + 0, + MessageState.CREATED, + message.payload().getClass().getName(), + JsonObject.mapFrom(message.payload()), + null, + null, + null, + PartitionHashRing.resolve(message.partitionKey()), + message.partitionKey(), + BaseRecord.newRecord(message.tenant()) + ); + return queue.insert(queueEntry, (SqlConnection) consumerTransaction.connection()) + .replaceWithVoid() + .onFailure().transform(ProducerExeception::new); + } + + private static void log(Message message) { + LOGGER.debug("Enqueuing {}", message); + } + + + public Uni enqueue(Message message) { + final var queueEntry = new MessageRecord( + message.messageId(), + message.scheduled(), + message.expiration(), + message.priority(), + 0, + MessageState.CREATED, + message.payload().getClass().getName(), + JsonObject.mapFrom(message.payload()), + null, + null, + null, + PartitionHashRing.resolve(message.partitionKey()), + message.partitionKey(), + BaseRecord.newRecord(message.tenant()) + ); + log(message); + return queue.insert(queueEntry) + .replaceWithVoid() + .onFailure().transform(ProducerExeception::new); + } + + public Uni enqueue(List> entries, ConsumerTransaction consumerTransaction) { + final var messageRecords = entries.stream().map( + message -> { + log(message); + return new MessageRecord( + message.messageId(), + message.scheduled(), + message.expiration(), + message.priority(), + 0, + MessageState.CREATED, + message.payload().getClass().getName(), + JsonObject.mapFrom(message.payload()), + null, + null, + null, + PartitionHashRing.resolve(message.partitionKey()), + message.partitionKey(), + BaseRecord.newRecord(message.tenant()) + ); + } + ).toList(); + return queue.insertBatch(messageRecords, consumerTransaction.getDelegate(SqlConnection.class)) + .replaceWithVoid() + .onFailure().transform(ProducerExeception::new); + } + + + public Uni enqueue(List> entries) { + final var messageRecords = entries.stream().map( + message -> { + log(message); + return new MessageRecord( + message.messageId(), + message.scheduled(), + message.expiration(), + message.priority(), + 0, + MessageState.CREATED, + message.payload().getClass().getName(), + JsonObject.mapFrom(message.payload()), + null, + null, + null, + PartitionHashRing.resolve(message.partitionKey()), + message.partitionKey(), + BaseRecord.newRecord(message.tenant()) + ); + } + ).toList(); + return queue.insertBatch(messageRecords) + .replaceWithVoid() + .onFailure().transform(ProducerExeception::new); + } + + public Uni cancel(MessageID messageID) { + LOGGER.info("Cancelling message {}", messageID.id()); + return queue.deleteByKey(new MessageRecordID(messageID.id(), messageID.tenant())); + } + + public Uni> get(MessageID messageID, Class tClass) { + return queue.selectByKey(new MessageRecordID(messageID.id(), messageID.tenant())) + .map(entry -> new Message<>( + entry.id(), + entry.baseRecord().tenant(), + entry.partitionId(), + entry.scheduled(), + entry.expiration(), + entry.priority(), + entry.payload().mapTo(tClass) + ) + ); + } + + public Uni>> query(MessageRecordQuery query, Class tClass) { + return queue.query(query) + .map(entries -> entries.stream() + .map(entry -> new Message<>( + entry.id(), + entry.baseRecord().tenant(), + entry.partitionId(), + entry.scheduled(), + entry.expiration(), + entry.priority(), + entry.payload().mapTo(tClass) + )).toList() + ); + } + +} diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/queue.xml b/es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/queue.xml new file mode 100644 index 0000000..54f5726 --- /dev/null +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/queue.xml @@ -0,0 +1,173 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + CREATE + OR REPLACE FUNCTION ${schema}.queue_channel_publisher() + RETURNS + trigger + AS + $$ + BEGIN + PERFORM pg_notify('${schema}-queue-channel', NEW.partition_id::text); + RETURN NEW; + END; + $$ + LANGUAGE plpgsql; + + + + + + + CREATE TRIGGER queue_trigger + AFTER INSERT OR + UPDATE OF rec_version + ON ${schema}.queue + FOR EACH ROW EXECUTE PROCEDURE queue_channel_publisher(); + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-1'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-2'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-3'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-4'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-5'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-6'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-7'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-8'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-9'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-10'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-11'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-12'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-13'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-14'); + INSERT INTO ${schema}.queue_partition (partition_id) VALUES ('partition-15'); + + + diff --git a/es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/message-broker.xml b/es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/topic.xml similarity index 54% rename from es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/message-broker.xml rename to es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/topic.xml index 4dd92de..9e64afe 100644 --- a/es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/message-broker.xml +++ b/es4j-infrastructure/es4j-postgres-message-broker/src/main/resources/topic.xml @@ -6,11 +6,11 @@ > - - - + + + - + @@ -19,16 +19,18 @@ - + + + - - - + + + @@ -37,57 +39,65 @@ - - + + CREATE - OR REPLACE FUNCTION ${schema}.task_queue_pub() RETURNS + OR REPLACE FUNCTION + ${schema} + . + topic_channel_publisher + ( + ) + RETURNS trigger AS $$ BEGIN - PERFORM pg_notify('task_queue_ch', NEW.message_id::text); - RETURN NEW; + PERFORM + pg_notify('${schema}-topic-channel', NEW.partition_id::text); + RETURN NEW; END; - $$ LANGUAGE plpgsql; + $$ + LANGUAGE plpgsql; - - + + - CREATE TRIGGER task_queue_pub_trg + CREATE TRIGGER topic_trigger AFTER INSERT OR UPDATE OF rec_version - ON ${schema}.task_queue - EXECUTE PROCEDURE task_queue_pub(); + ON ${schema}.topic + FOR EACH ROW EXECUTE PROCEDURE topic_channel_publisher(); - - + + - + - - - + + + - + @@ -95,10 +105,9 @@ - - - - + + + @@ -109,7 +118,9 @@ - + + + @@ -117,5 +128,20 @@ - + + + + + + + + + + + + + + + + diff --git a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java index e352e41..6628d4a 100644 --- a/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java +++ b/es4j-infrastructure/es4j-postgres-storage/src/main/java/io/es4j/infra/pg/PgEventStore.java @@ -8,6 +8,7 @@ import io.es4j.infrastructure.models.*; import io.es4j.sql.LiquibaseHandler; import io.es4j.sql.RepositoryHandler; +import io.es4j.sql.exceptions.Conflict; import io.es4j.sql.exceptions.NotFound; import io.es4j.sql.models.QueryOptions; import io.smallrye.mutiny.Uni; @@ -121,7 +122,14 @@ public Uni> fetch(EventStream eventStream) { @Override public Uni append(AppendInstruction appendInstruction) { - return eventJournal.insertBatch(parseInstruction(appendInstruction)); + return eventJournal.insertBatch(parseInstruction(appendInstruction)) + .onFailure().transform(throwable -> { + if (throwable instanceof Conflict conflict) { + throw new ConcurrentAppend(conflict); + } else { + throw new EventStoreExeception(throwable); + } + }); } @Override diff --git a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java index 3913be5..8dffb1b 100644 --- a/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java +++ b/es4j-infrastructure/es4j-redis-storage/src/main/java/io/es4j/infra/redis/RedisEventStore.java @@ -7,7 +7,6 @@ import io.es4j.core.objects.Es4jErrorBuilder; import io.es4j.infrastructure.EventStore; import io.es4j.infrastructure.models.*; -import io.es4j.sql.LiquibaseHandler; import io.smallrye.mutiny.Multi; import io.smallrye.mutiny.Uni; import io.vertx.core.impl.cpu.CpuCoreSensor; @@ -36,7 +35,7 @@ public class RedisEventStore implements EventStore { public static final String EVENT = "event"; public static final String TAGS = "tags"; public static final String SCHEMA_VERSION = "schema-version"; - private static final Logger LOGGER = LoggerFactory.getLogger(LiquibaseHandler.class); + private static final Logger LOGGER = LoggerFactory.getLogger(RedisEventStore.class); private Redis redisClient; private RedisAPI redisApi; private Class aggregateClass; diff --git a/es4j-infrastructure/es4j-sql/pom.xml b/es4j-infrastructure/es4j-sql/pom.xml index 5d8b749..2b59c1c 100644 --- a/es4j-infrastructure/es4j-sql/pom.xml +++ b/es4j-infrastructure/es4j-sql/pom.xml @@ -30,12 +30,6 @@ - - io.vertx - vertx-sql-client - 4.4.3 - runtime - io.smallrye.reactive smallrye-mutiny-vertx-pg-client diff --git a/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/RecordMapper.java b/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/RecordMapper.java index 605d5c2..b7b6a25 100644 --- a/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/RecordMapper.java +++ b/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/RecordMapper.java @@ -12,79 +12,90 @@ import java.util.*; import java.util.stream.Collectors; +import static io.es4j.sql.misc.Constants.VERSION; + public interface RecordMapper, Q extends Query> { - /** - * @return table name - */ - String table(); - - /** - * Should return a set that contains all the columns that make up the table - * * base record columns can be skipped - * - * @return - */ - Set columns(); - - /** - * Should return a set that contains all the columns that can be updated during update queries. used for updateByKey(), updateById() - * - * @return - */ - default Set updatableColumns() { - return columns().stream() - .filter(c -> keyColumns().stream().noneMatch(k -> k.equalsIgnoreCase(c))) - .collect(Collectors.toSet()); + /** + * @return table name + */ + String table(); + + /** + * Should return a set that contains all the columns that make up the table + * * base record columns can be skipped + * + * @return + */ + Set columns(); + + /** + * Should return a set that contains all the columns that can be updated during update queries. used for updateByKey(), updateById() + * + * @return + */ + default Set updatableColumns() { + return columns().stream() + .filter(c -> keyColumns().stream().noneMatch(k -> k.equalsIgnoreCase(c))) + .collect(Collectors.toSet()); + } + + /** + * Should return all the columns that make up the key of a record + * + * @return + */ + Set keyColumns(); + + /** + * Should map a vert.x Row to the domain object V + * + * @param row the vertx row that represents the row's from the database + * @return + */ + V rowMapper(Row row); + + + /** + * The map should be filled with the tuple(column,valueParam) of a record that will be stored in the database, used for inserts only + * + * @param params + */ + void params(Map params, V actualRecord); + + /** + * The map should be filled with tuple(column,valueParam) that compose the key of the record, used for selectByKey() + * + * @param params + */ + void keyParams(Map params, K key); + + /** + * Fill up the builder with entries Column,Value this mapping will be subjected to run time validation thus the values will always be checked before being added to the final query + * + * @param query the object that represents the queryable fields in the record + * @param builder where queries for that object can be built + */ + void queryBuilder(Q query, QueryBuilder builder); + + default BaseRecord baseRecord(Row row) { + return new BaseRecord( + row.getString(Constants.TENANT), + getVersion(row), + row.getLocalDateTime(Constants.CREATION_DATE).toInstant(ZoneOffset.UTC), + row.getLocalDateTime(Constants.LAST_UPDATE).toInstant(ZoneOffset.UTC) + ); + } + + private static Integer getVersion(Row row) { + try { + return row.getInteger(VERSION); + } catch (Exception e) { + return null; } - /** - * Should return all the columns that make up the key of a record - * - * @return - */ - Set keyColumns(); - - /** - * Should map a vert.x Row to the domain object V - * - * @param row the vertx row that represents the row's from the database - * @return - */ - V rowMapper(Row row); - - - /** - * The map should be filled with the tuple(column,valueParam) of a record that will be stored in the database, used for inserts only - * - * @param params - */ - void params(Map params, V actualRecord); - - /** - * The map should be filled with tuple(column,valueParam) that compose the key of the record, used for selectByKey() - * - * @param params - */ - void keyParams(Map params, K key); - - /** - * Fill up the builder with entries Column,Value this mapping will be subjected to run time validation thus the values will always be checked before being added to the final query - * - * @param query the object that represents the queryable fields in the record - * @param builder where queries for that object can be built - */ - void queryBuilder(Q query, QueryBuilder builder); - - default BaseRecord baseRecord(Row row) { - return new BaseRecord( - row.getString(Constants.TENANT), - null, // todo fix by implementing a way to have versionless records - row.getLocalDateTime(Constants.CREATION_DATE).toInstant(ZoneOffset.UTC), - row.getLocalDateTime(Constants.LAST_UPDATE).toInstant(ZoneOffset.UTC) - ); - } + } } diff --git a/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/models/QueryFilters.java b/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/models/QueryFilters.java index f6c344a..fde0e69 100644 --- a/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/models/QueryFilters.java +++ b/es4j-infrastructure/es4j-sql/src/main/java/io/es4j/sql/models/QueryFilters.java @@ -28,11 +28,18 @@ public QueryFilters filterParams(List params) { this.params = params; return this; } + public QueryFilters filterParams(T... params) { this.params = unpackValues(params); return this; } + public QueryFilters filterParam(T param) { + if (param != null) { + this.params = List.of(param); + } + return this; + } public QueryFilters(Class paramType) { @@ -52,7 +59,7 @@ private static List unpackValues(T[] values) { } public QueryFilters validate() { - Objects.requireNonNull(column,"Column shouldn't be null !"); + Objects.requireNonNull(column, "Column shouldn't be null !"); return this; } diff --git a/es4j-infrastructure/es4j-task/pom.xml b/es4j-infrastructure/es4j-task/pom.xml index b995dfd..a853f17 100644 --- a/es4j-infrastructure/es4j-task/pom.xml +++ b/es4j-infrastructure/es4j-task/pom.xml @@ -32,20 +32,14 @@ - io.es4j - es4j-sql - ${project.version} + io.smallrye.reactive + smallrye-mutiny-vertx-core com.cronutils cron-utils 9.2.0 - - io.es4j - es4j-postgres-message-broker - ${project.version} - io.soabase.record-builder record-builder-processor diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTask.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTask.java index 859e16b..a6c248c 100644 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTask.java +++ b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTask.java @@ -3,11 +3,8 @@ import com.cronutils.model.CronType; import com.cronutils.model.definition.CronDefinitionBuilder; import com.cronutils.parser.CronParser; -import io.es4j.sql.exceptions.NotFound; import io.smallrye.mutiny.Uni; -import java.util.List; - public interface CronTask { Uni performTask(); @@ -15,8 +12,7 @@ public interface CronTask { default CronTaskConfiguration configuration() { return new CronTaskConfiguration( new CronParser(CronDefinitionBuilder.instanceDefinitionFor(CronType.UNIX)).parse("0 0 * * *"), - LockLevel.LOCAL, - List.of(NotFound.class) + LockLevel.LOCAL ); } diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskConfiguration.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskConfiguration.java index 234d7a5..915eae3 100644 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskConfiguration.java +++ b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskConfiguration.java @@ -9,8 +9,7 @@ @RecordBuilder public record CronTaskConfiguration( Cron cron, - LockLevel lockLevel, - List> knownInterruptions + LockLevel lockLevel ) { } diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskDeployer.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskDeployer.java index 8499e13..e38ea42 100644 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskDeployer.java +++ b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskDeployer.java @@ -57,9 +57,7 @@ public void triggerTask(TaskWrapper taskWrapper, Vertx vertx, Duration throttle) }, throwable -> { taskWrapper.logger().info("cron-task ran in {}", Duration.between(start, Instant.now()).toMillis()); - if (taskWrapper.task.configuration().knownInterruptions().stream().anyMatch(t -> t.isAssignableFrom(throwable.getClass()))) { - taskWrapper.logger().debug("Interrupted by {} ", throwable.getClass().getSimpleName()); - } else if (throwable instanceof NoStackTraceThrowable noStackTraceThrowable && noStackTraceThrowable.getMessage().contains("Timed out waiting to get lock")) { + if (throwable instanceof NoStackTraceThrowable noStackTraceThrowable && noStackTraceThrowable.getMessage().contains("Timed out waiting to get lock")) { taskWrapper.logger().debug("Unable to acquire lock"); } else { taskWrapper.logger().error("Error handling cron-task", throwable); diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskKey.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskKey.java deleted file mode 100644 index c438503..0000000 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskKey.java +++ /dev/null @@ -1,8 +0,0 @@ -package io.es4j.task; - -import io.es4j.sql.models.RepositoryRecordKey; - -public record CronTaskKey( - String taskClass -)implements RepositoryRecordKey { -} diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskMapper.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskMapper.java deleted file mode 100644 index 7d78933..0000000 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskMapper.java +++ /dev/null @@ -1,92 +0,0 @@ -package io.es4j.task; - -import io.es4j.sql.generator.filters.QueryBuilder; -import io.es4j.sql.models.QueryFilter; -import io.es4j.sql.models.QueryFilters; -import io.es4j.sql.RecordMapper; -import io.vertx.sqlclient.Row; - -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.Map; -import java.util.Set; - -public class CronTaskMapper implements RecordMapper { - - public static final String CRON_TASKS = "cron_tasks"; - public static final String LAST_EXECUTION = "last_execution"; - public static final String NEXT_EXECUTION = "next_execution"; - public static final String TASK_CLASS = "task_class"; - - private CronTaskMapper() {} - - public static final CronTaskMapper INSTANCE = new CronTaskMapper(); - - @Override - public String table() { - return CRON_TASKS; - } - - @Override - public Set columns() { - return Set.of(TASK_CLASS, NEXT_EXECUTION, LAST_EXECUTION); - } - - @Override - public Set keyColumns() { - return Set.of(TASK_CLASS); - } - - @Override - public CronTaskRecord rowMapper(Row row) { - return new CronTaskRecord( - row.getString(TASK_CLASS), - row.getLocalDateTime(LAST_EXECUTION).toInstant(ZoneOffset.UTC), - row.getLocalDateTime(NEXT_EXECUTION).toInstant(ZoneOffset.UTC), - baseRecord(row) - ); - } - - @Override - public void params(Map params, CronTaskRecord actualRecord) { - params.put(TASK_CLASS, actualRecord.taskClass()); - params.put(LAST_EXECUTION, LocalDateTime.ofInstant(actualRecord.lastExecutionTime(),ZoneOffset.UTC)); - params.put(NEXT_EXECUTION, LocalDateTime.ofInstant(actualRecord.nextExecutionTime(), ZoneOffset.UTC)); - } - - @Override - public void keyParams(Map params, CronTaskKey key) { - params.put(TASK_CLASS, key.taskClass()); - } - - @Override - public void queryBuilder(CronTaskQuery query, QueryBuilder builder) { - builder - .iLike( - new QueryFilters<>(String.class) - .filterColumn(TASK_CLASS) - .filterParams(query.taskClasses()) - ) - .from( - new QueryFilter<>(Instant.class) - .filterColumn(NEXT_EXECUTION) - .filterParam(query.nextExecutionFrom()) - ) - .to( - new QueryFilter<>(Instant.class) - .filterColumn(NEXT_EXECUTION) - .filterParam(query.nextExecutionTo()) - ) - .from( - new QueryFilter<>(Instant.class) - .filterColumn(LAST_EXECUTION) - .filterParam(query.lastExecutionFrom()) - ) - .to( - new QueryFilter<>(Instant.class) - .filterColumn(LAST_EXECUTION) - .filterParam(query.lastExecutionTo()) - ); - } -} diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskQuery.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskQuery.java deleted file mode 100644 index 4b45f95..0000000 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskQuery.java +++ /dev/null @@ -1,20 +0,0 @@ -package io.es4j.task; - -import io.es4j.sql.models.Query; -import io.es4j.sql.models.QueryOptions; -import io.soabase.recordbuilder.core.RecordBuilder; - -import java.time.Instant; -import java.util.List; - -@RecordBuilder -public record CronTaskQuery( - List taskClasses, - Instant nextExecutionFrom, - Instant nextExecutionTo, - - Instant lastExecutionFrom, - Instant lastExecutionTo, - QueryOptions options -) implements Query { -} diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskRecord.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskRecord.java deleted file mode 100644 index e745d62..0000000 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/CronTaskRecord.java +++ /dev/null @@ -1,25 +0,0 @@ -package io.es4j.task; - -import io.es4j.sql.models.RepositoryRecord; -import io.soabase.recordbuilder.core.RecordBuilder; -import io.es4j.sql.models.BaseRecord; - -import java.time.Instant; - -@RecordBuilder -public record CronTaskRecord( - String taskClass, - Instant lastExecutionTime, - Instant nextExecutionTime, - BaseRecord baseRecord -) implements RepositoryRecord { - @Override - public CronTaskRecord with(BaseRecord baseRecord) { - return new CronTaskRecord(taskClass, lastExecutionTime, nextExecutionTime, baseRecord); - } - - public CronTaskRecord newExecutionTime(Instant executionTime) { - return new CronTaskRecord(taskClass, nextExecutionTime, executionTime, baseRecord); - - } -} diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTask.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTask.java index 249fbd7..24a230a 100644 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTask.java +++ b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTask.java @@ -1,10 +1,9 @@ package io.es4j.task; -import io.es4j.sql.exceptions.NotFound; import io.smallrye.mutiny.Uni; import java.time.Duration; -import java.util.List; +import java.util.Optional; public interface TimerTask { @@ -16,8 +15,7 @@ default TimerTaskConfiguration configuration() { Duration.ofSeconds(5), Duration.ofSeconds(5), Duration.ofMinutes(1), - Duration.ofMinutes(1), - List.of(NotFound.class) + Optional.empty() ); } diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskConfiguration.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskConfiguration.java index b8c1afb..4099dd3 100644 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskConfiguration.java +++ b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskConfiguration.java @@ -6,13 +6,13 @@ import java.time.Duration; import java.util.List; +import java.util.Optional; @RecordBuilder public record TimerTaskConfiguration( LockLevel lockLevel, Duration throttle, - Duration interruptionBackOff, Duration lockBackOff, Duration errorBackOff, - List> knownInterruptions + Optional> finalInterruption ) {} diff --git a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskDeployer.java b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskDeployer.java index 5cb53ff..c714641 100644 --- a/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskDeployer.java +++ b/es4j-infrastructure/es4j-task/src/main/java/io/es4j/task/TimerTaskDeployer.java @@ -38,7 +38,7 @@ public void deploy(TimerTask timerTask) { public static void triggerTask(TaskWrapper taskWrapper, Vertx vertx, Duration throttle) { timers.remove(taskWrapper.task().getClass()); - taskWrapper.logger().info("Starting task"); + taskWrapper.logger().debug("Starting task"); final var timerId = vertx.setTimer( throttle.toMillis(), delay -> { @@ -54,23 +54,20 @@ public static void triggerTask(TaskWrapper taskWrapper, Vertx vertx, Duration th .with(avoid -> { final var end = Instant.now(); final var emptyTaskBackOff = taskWrapper.task().configuration().throttle(); - taskWrapper.logger().info("Task ran in " + Duration.between(start, end).toMillis() + "ms. Next execution in " + emptyTaskBackOff.getSeconds() + "s"); + taskWrapper.logger().debug("Task ran in " + Duration.between(start, end).toMillis() + "ms. Next execution in " + emptyTaskBackOff.getSeconds() + "s"); triggerTask(taskWrapper, vertx, emptyTaskBackOff); }, throwable -> { final var end = Instant.now(); - if (taskWrapper.task.configuration().knownInterruptions().stream().anyMatch(t -> t.isAssignableFrom(throwable.getClass()))) { - taskWrapper.logger().debug("Task interrupted by" + throwable.getClass().getSimpleName() + " after " + Duration.between(start, end).toMillis() + "ms"); - taskWrapper.logger().info("Interrupted, backing off for {}", taskWrapper.task().configuration().interruptionBackOff()); - triggerTask(taskWrapper, vertx, taskWrapper.task().configuration().interruptionBackOff()); + if (taskWrapper.task.configuration().finalInterruption().isPresent() && taskWrapper.task.configuration().finalInterruption().get().isAssignableFrom(throwable.getClass())) { + taskWrapper.logger().info("Final interruption {} reached, task wont be rescheduled", throwable.getClass().getSimpleName()); } else if (throwable instanceof NoStackTraceThrowable noStackTraceThrowable && noStackTraceThrowable.getMessage().contains("Timed out waiting to get lock")) { - taskWrapper.logger().info("Unable to acquire lock, will back off for {}", taskWrapper.task().configuration().lockBackOff()); + taskWrapper.logger().debug("Unable to acquire lock after {}ms, will back off for {}", Duration.between(start, end).toMillis(), taskWrapper.task().configuration().lockBackOff()); triggerTask(taskWrapper, vertx, taskWrapper.task().configuration().lockBackOff()); } else { - taskWrapper.logger().info("Error handling task, will back off for {}", taskWrapper.task().configuration().errorBackOff(), throwable); + taskWrapper.logger().debug("Error handling task after {}ms, will back off for {}", Duration.between(start, end).toMillis(), taskWrapper.task().configuration().errorBackOff(), throwable); triggerTask(taskWrapper, vertx, taskWrapper.task().configuration().errorBackOff()); } - } ); } diff --git a/es4j-infrastructure/es4j-task/src/main/resources/task.xml b/es4j-infrastructure/es4j-task/src/main/resources/task.xml deleted file mode 100644 index 2e0a833..0000000 --- a/es4j-infrastructure/es4j-task/src/main/resources/task.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/es4j-infrastructure/pom.xml b/es4j-infrastructure/pom.xml index 53afe33..deda927 100644 --- a/es4j-infrastructure/pom.xml +++ b/es4j-infrastructure/pom.xml @@ -18,7 +18,6 @@ es4j-task es4j-postgres-message-broker es4j-http-bridge - es4j-config-storage es4j-postgres-storage es4j-redis-storage es4j-http-client diff --git a/es4j-test/pom.xml b/es4j-test/pom.xml index 4aee833..d5826af 100644 --- a/es4j-test/pom.xml +++ b/es4j-test/pom.xml @@ -73,6 +73,11 @@ es4j-config-storage ${project.version} + + io.es4j + es4j-postgres-message-broker + ${project.version} + com.kjetland mbknor-jackson-jsonschema_2.13 diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/DeadPayloadConsumer.java b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/DeadPayloadConsumer.java new file mode 100644 index 0000000..7431ba8 --- /dev/null +++ b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/DeadPayloadConsumer.java @@ -0,0 +1,26 @@ +package io.es4j.infrastructure.messagebroker; + + +import com.google.auto.service.AutoService; +import io.es4j.infrastructure.pgbroker.QueueConsumer; +import io.es4j.infrastructure.pgbroker.models.ConsumerTransaction; +import io.smallrye.mutiny.Uni; + +import java.util.List; + + +@AutoService(QueueConsumer.class) +public class DeadPayloadConsumer implements QueueConsumer { + @Override + public Uni process(MockDeadPayload payload, ConsumerTransaction queueTransaction) { + if (payload.fatal()) { + return Uni.createFrom().failure(new FatalException("Fatal failure !")); + } + return Uni.createFrom().failure(new RuntimeException("Mocking failure !")); + } + + @Override + public List> fatalExceptions() { + return List.of(FatalException.class); + } +} diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/FatalException.java b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/FatalException.java similarity index 73% rename from es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/FatalException.java rename to es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/FatalException.java index d6a33db..8725301 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/FatalException.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/FatalException.java @@ -1,4 +1,4 @@ -package io.es4j.infrastructure.taskqueue; +package io.es4j.infrastructure.messagebroker; public class FatalException extends RuntimeException { diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockDeadPayload.java b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/MockDeadPayload.java similarity index 60% rename from es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockDeadPayload.java rename to es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/MockDeadPayload.java index d4842ad..2633340 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockDeadPayload.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/MockDeadPayload.java @@ -1,4 +1,4 @@ -package io.es4j.infrastructure.taskqueue; +package io.es4j.infrastructure.messagebroker; public record MockDeadPayload( String data, diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockPayload.java b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/MockPayload.java similarity index 51% rename from es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockPayload.java rename to es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/MockPayload.java index 634a6e6..bd0ce6c 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockPayload.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/MockPayload.java @@ -1,4 +1,4 @@ -package io.es4j.infrastructure.taskqueue; +package io.es4j.infrastructure.messagebroker; public record MockPayload( String data diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/MockPayloadConsumer.java b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/MockPayloadConsumer.java new file mode 100644 index 0000000..4eb9617 --- /dev/null +++ b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/MockPayloadConsumer.java @@ -0,0 +1,16 @@ +package io.es4j.infrastructure.messagebroker; + +import com.google.auto.service.AutoService; +import io.es4j.infrastructure.pgbroker.QueueConsumer; +import io.es4j.infrastructure.pgbroker.models.ConsumerTransaction; +import io.smallrye.mutiny.Uni; + +@AutoService(QueueConsumer.class) +public class MockPayloadConsumer implements QueueConsumer { + + @Override + public Uni process(MockPayload payload, ConsumerTransaction queueTransaction) { + return Uni.createFrom().voidItem(); + } + +} diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/PgTaskQueueTest.java b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/PgTaskQueueTest.java new file mode 100644 index 0000000..3f846f0 --- /dev/null +++ b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/PgTaskQueueTest.java @@ -0,0 +1,218 @@ +package io.es4j.infrastructure.messagebroker; + +import io.es4j.infrastructure.pgbroker.PgBroker; +import io.es4j.infrastructure.pgbroker.mappers.DeadLetterMapper; +import io.es4j.infrastructure.pgbroker.mappers.MessageQueueMapper; +import io.es4j.infrastructure.pgbroker.mappers.MessageTransactionMapper; +import io.es4j.infrastructure.pgbroker.models.*; +import io.es4j.infrastructure.pgbroker.vertx.VertxMessageProducer; +import io.es4j.infrastructure.sql.SqlBootstrap; +import io.es4j.sql.Repository; +import io.es4j.sql.exceptions.NotFound; +import io.es4j.sql.models.QueryOptions; +import io.smallrye.mutiny.Uni; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.UUID; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.*; + +public class PgTaskQueueTest { + + public static final SqlBootstrap BOOTSTRAP = new SqlBootstrap().setPostgres(false); + final VertxMessageProducer producer = new VertxMessageProducer(SqlBootstrap.REPOSITORY_HANDLER); + final Repository queueTx = new Repository<>(MessageTransactionMapper.INSTANCE, SqlBootstrap.REPOSITORY_HANDLER); + final Repository deadLetters = new Repository<>(DeadLetterMapper.INSTANCE, SqlBootstrap.REPOSITORY_HANDLER); + final Repository messageQueue = new Repository<>(MessageQueueMapper.INSTANCE, SqlBootstrap.REPOSITORY_HANDLER); + + @AfterAll + static void destroy() throws Exception { + BOOTSTRAP.destroy(); + } + + @BeforeAll + static void start() { + BOOTSTRAP.start(); + PgBroker.deploy(BOOTSTRAP.CONFIGURATION, SqlBootstrap.vertx).await().indefinitely(); + } + + + @Test + void test_pg_producer() { + final var fakeMessage = fakeMessage(); + sendMessage(fakeMessage); + messageQueue.selectByKey(new MessageRecordID(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely(); + } + + @Test + void test_unpartitioned_message() throws InterruptedException { + final var fakeMessage = fakeMessage(); + sendMessage(fakeMessage); + Thread.sleep(5000); + queueTx.selectByKey(new MessageTransactionID(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely(); + } + + @Test + void test_partitioned_message() throws InterruptedException { + final var fakeMessage = fakePartitonedMessage(); + sendMessage(fakeMessage); + Thread.sleep(1000); + queueTx.selectByKey(new MessageTransactionID(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely(); + } + + @Test + void test_partition_distribution() throws InterruptedException { + final var messages = IntStream.range(0, 10).mapToObj(i -> fakePartitonedMessage()).toList(); + sendMessages(messages); + Thread.sleep(1000); + final var txs = queueTx.query( + new MessageTransactionQuery( + messages.stream().map(Message::messageId).toList(), + null, + QueryOptions.simple() + ) + ).await().indefinitely(); + Assertions.assertEquals(messages.size(), txs.size()); + } + + private void sendMessages(List> messages) { + producer.enqueue(messages).await().indefinitely(); + } + + @Test + void test_dead_letter_fatal_message() throws InterruptedException { + final var fakeMessage = fatalMessage(); + sendMessage(fakeMessage); + Thread.sleep(2000); + deadLetters.selectByKey(new DeadLetterKey(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely(); + } + + @Test + void test_dead_letter_fatal_message_partition() throws InterruptedException { + final var partionKey = "dummy-key"; + final var fakeMessage = fatalMessage(partionKey); + sendMessage(fakeMessage); + Thread.sleep(2000); + deadLetters.selectByKey(new DeadLetterKey(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely(); + } + + @Test + void test_parking_after_dead_letter_partition() throws InterruptedException { + final var partionKey = "dummy-key"; + final var fakeMessage = fatalMessage(partionKey); + sendMessage(fakeMessage); + Thread.sleep(2000); + deadLetters.selectByKey(new DeadLetterKey(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely(); + final var toBeParked = IntStream.range(0, 10).mapToObj(i -> fatalMessage(partionKey)).toList(); + sendMessages(toBeParked); + Thread.sleep(2000); + final var parkedMessages = deadLetters.query( + MessageRecordQueryBuilder.builder() + .ids(toBeParked.stream().map(Message::messageId).toList()) + .options(QueryOptions.simple()) + .build() + ).await().indefinitely(); + Assertions.assertEquals(toBeParked.size(), parkedMessages.size()); + Assertions.assertTrue(parkedMessages.stream().allMatch(m -> m.messageState() == MessageState.PARKED)); + } + + @Test + void test_nack() throws InterruptedException { + final var fakeMessage = failForRetryMessage(); + sendMessage(fakeMessage); + Thread.sleep(1000); + final var actualQueue = new Repository<>(MessageQueueMapper.INSTANCE, SqlBootstrap.REPOSITORY_HANDLER); + final var nackedMessage = actualQueue.selectByKey(new MessageRecordID(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely(); + assertNotEquals(0, nackedMessage.retryCounter()); + assertThrows(NotFound.class, () -> deadLetters.selectByKey(new DeadLetterKey(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely()); + } + + @Test + void test_recovery_mechanism() { + + + } + + private void sendMessage(Message fakeMessage) { + SqlBootstrap.REPOSITORY_HANDLER.pgPool().withTransaction( + sqlConnection -> producer.enqueue(fakeMessage, new ConsumerTransaction(sqlConnection)) + ).await().indefinitely(); + } + + private Uni sendMessageUni(Message fakeMessage) { + return SqlBootstrap.REPOSITORY_HANDLER.pgPool().withTransaction( + sqlConnection -> producer.enqueue(fakeMessage, new ConsumerTransaction(sqlConnection)) + ); + } + + @NotNull + private static Message fakeMessage() { + return new Message<>( + UUID.randomUUID().toString(), + "default", + null, + null, + null, + 0, + new MockPayload("data") + ); + } + + @NotNull + private static Message fakePartitonedMessage() { + final var id = UUID.randomUUID().toString(); + return new Message<>( + id, + "default", + id, + null, + null, + 0, + new MockPayload("data") + ); + } + + @NotNull + private static Message fatalMessage() { + return new Message<>( + UUID.randomUUID().toString(), + "default", + null, + null, + null, + 0, + new MockDeadPayload("data", true) + ); + } + @NotNull + private static Message fatalMessage(String partitionKey) { + return new Message<>( + UUID.randomUUID().toString(), + "default", + partitionKey, + null, + null, + 0, + new MockDeadPayload("data", true) + ); + } + + @NotNull + private static Message failForRetryMessage() { + return new Message<>( + UUID.randomUUID().toString(), + "default", + null, + null, + null, + 0, + new MockDeadPayload("data", false) + ); + } +} diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/RecoveryTestPayload.java b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/RecoveryTestPayload.java similarity index 53% rename from es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/RecoveryTestPayload.java rename to es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/RecoveryTestPayload.java index 738cf78..539cec7 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/RecoveryTestPayload.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/messagebroker/RecoveryTestPayload.java @@ -1,4 +1,4 @@ -package io.es4j.infrastructure.taskqueue; +package io.es4j.infrastructure.messagebroker; public record RecoveryTestPayload(String data) { } diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/sql/SqlBootstrap.java b/es4j-test/src/test/java/io/es4j/infrastructure/sql/SqlBootstrap.java index 26df33a..36e1373 100644 --- a/es4j-test/src/test/java/io/es4j/infrastructure/sql/SqlBootstrap.java +++ b/es4j-test/src/test/java/io/es4j/infrastructure/sql/SqlBootstrap.java @@ -45,7 +45,9 @@ public class SqlBootstrap { public void start() { vertx = Vertx.vertx(); CONFIGURATION = configuration().put("schema", schema); - deployPgContainer(); + if (postgres) { + deployPgContainer(); + } REPOSITORY_HANDLER = RepositoryHandler.leasePool(CONFIGURATION, vertx); if (!liquibase.isEmpty()) { Multi.createFrom().iterable(liquibase.entrySet()) diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockDeadPayloadProcessor.java b/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockDeadPayloadProcessor.java deleted file mode 100644 index 1f367af..0000000 --- a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockDeadPayloadProcessor.java +++ /dev/null @@ -1,24 +0,0 @@ -//package io.es4j.infrastructure.taskqueue; -// -//import io.smallrye.mutiny.Uni; -//import io.es4j.queue.MessageProcessor; -//import io.es4j.queue.models.QueueTransaction; -// -//import java.util.List; -// -//public class MockDeadPayloadProcessor implements MessageProcessor { -// @Override -// public Uni process(MockDeadPayload payload, QueueTransaction queueTransaction) { -// if (payload.fatal()) { -// return Uni.createFrom().failure(new FatalException("Fatal failure !")); -// } -// return Uni.createFrom().failure(new RuntimeException("Mocking failure !")); -// } -// -// @Override -// public List> fatalExceptions() { -// return List.of(FatalException.class); -// } -// -// -//} diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java b/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java deleted file mode 100644 index 4b9b5a4..0000000 --- a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/MockProcessor.java +++ /dev/null @@ -1,13 +0,0 @@ -//package io.es4j.infrastructure.taskqueue; -// -//import io.smallrye.mutiny.Uni; -// -// -// -//public class MockProcessor implements MessageProcessor { -// -// @Override -// public Uni process(MockPayload payload, QueueTransaction queueTransaction) { -// return Uni.createFrom().voidItem(); -// } -//} diff --git a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/PgTaskQueueTest.java b/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/PgTaskQueueTest.java deleted file mode 100644 index aa9389b..0000000 --- a/es4j-test/src/test/java/io/es4j/infrastructure/taskqueue/PgTaskQueueTest.java +++ /dev/null @@ -1,136 +0,0 @@ -//package io.es4j.infrastructure.taskqueue; -// -// -//import io.es4j.core.verticles.TaskProcessorVerticle; -//import io.es4j.sql.Repository; -//import io.es4j.sql.exceptions.NotFound; -//import io.es4j.InfrastructureBootstrap; -//import io.es4j.infrastructure.messagebroker.models.QueueTransaction; -//import io.es4j.infrastructure.messagebroker.postgres.PgMessageProducer; -//import io.es4j.infrastructure.messagebroker.models.Message; -//import io.es4j.infrastructure.messagebroker.postgres.mappers.DeadLetterMapper; -//import io.es4j.infrastructure.messagebroker.postgres.mappers.MessageQueueMapper; -//import io.es4j.infrastructure.messagebroker.postgres.mappers.MessageTransactionMapper; -//import io.es4j.infrastructure.messagebroker.postgres.models.DeadLetterKey; -//import io.es4j.infrastructure.messagebroker.postgres.models.MessageRecordID; -//import io.es4j.infrastructure.messagebroker.postgres.models.MessageTransactionID; -//import org.jetbrains.annotations.NotNull; -//import org.junit.jupiter.api.AfterAll; -//import org.junit.jupiter.api.BeforeAll; -//import org.junit.jupiter.api.Test; -//import java.util.Map; -//import java.util.UUID; -// -//import static org.junit.jupiter.api.Assertions.*; -// -//public class PgTaskQueueTest { -// -// public static final InfrastructureBootstrap BOOTSTRAP = new InfrastructureBootstrap() -// .setPostgres(true) -// .addLiquibaseRun("task-queue.xml", Map.of("schema", "es4j")); -// -// @AfterAll -// static void destroy() throws Exception { -// BOOTSTRAP.destroy(); -// } -// -// @BeforeAll -// static void start() { -// BOOTSTRAP.start(); -// TaskProcessorVerticle.deploy(InfrastructureBootstrap.vertx, BOOTSTRAP.CONFIGURATION).await().indefinitely(); -// } -// -// -// @Test -// void test_pg_producer() { -// // todo produce to the db queue and assert record has been written -// final var producer = new PgMessageProducer(InfrastructureBootstrap.REPOSITORY_HANDLER); -// final var queue = new Repository<>(MessageQueueMapper.INSTANCE, InfrastructureBootstrap.REPOSITORY_HANDLER); -// final var fakeMessage = fakeMessage(); -// InfrastructureBootstrap.REPOSITORY_HANDLER.pgPool().withTransaction( -// sqlConnection -> producer.enqueue(fakeMessage, new QueueTransaction(sqlConnection)) -// ).await().indefinitely(); -// queue.selectByKey(new MessageRecordID(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely(); -// } -// -// @Test -// void test_pg_subscriber() throws InterruptedException { -// final var producer = new PgMessageProducer(InfrastructureBootstrap.REPOSITORY_HANDLER); -// final var queueTx = new Repository<>(MessageTransactionMapper.INSTANCE, InfrastructureBootstrap.REPOSITORY_HANDLER); -// final var fakeMessage = fakeMessage(); -// InfrastructureBootstrap.REPOSITORY_HANDLER.pgPool().withTransaction( -// sqlConnection -> producer.enqueue(fakeMessage, new QueueTransaction(sqlConnection)) -// ).await().indefinitely(); -// Thread.sleep(1000); -// queueTx.selectByKey(new MessageTransactionID(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely(); -// } -// -// @Test -// void test_refresh_retry_and_dead_letter_queue() throws InterruptedException { -// final var producer = new PgMessageProducer(InfrastructureBootstrap.REPOSITORY_HANDLER); -// final var deadLetters = new Repository<>(DeadLetterMapper.INSTANCE, InfrastructureBootstrap.REPOSITORY_HANDLER); -// final var fakeMessage = fakeFatal(); -// InfrastructureBootstrap.REPOSITORY_HANDLER.pgPool().withTransaction( -// sqlConnection -> producer.enqueue(fakeMessage, new QueueTransaction(sqlConnection)) -// ).await().indefinitely(); -// Thread.sleep(1000); -// deadLetters.selectByKey(new DeadLetterKey(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely(); -// } -// -// @Test -// void test_nack() throws InterruptedException { -// final var producer = new PgMessageProducer(InfrastructureBootstrap.REPOSITORY_HANDLER); -// final var deadLetters = new Repository<>(DeadLetterMapper.INSTANCE, InfrastructureBootstrap.REPOSITORY_HANDLER); -// final var fakeMessage = fakeDeadMessage(); -// InfrastructureBootstrap.REPOSITORY_HANDLER.pgPool().withTransaction( -// sqlConnection -> producer.enqueue(fakeMessage, new QueueTransaction(sqlConnection)) -// ).await().indefinitely(); -// Thread.sleep(1000); -// final var actualQueue = new Repository<>(MessageQueueMapper.INSTANCE, InfrastructureBootstrap.REPOSITORY_HANDLER); -// final var nackedMessage = actualQueue.selectByKey(new MessageRecordID(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely(); -// assertNotEquals(0, nackedMessage.retryCounter()); -// assertThrows(NotFound.class, () -> deadLetters.selectByKey(new DeadLetterKey(fakeMessage.messageId(), fakeMessage.tenant())).await().indefinitely()); -// } -// -// @Test -// void test_recovery_mechanism() { -// // todo should only be processed if transaction is not present in the tx table. -// // todo should not process if present in tx table. -// } -// -// @NotNull -// private static Message fakeMessage() { -// return new Message<>( -// UUID.randomUUID().toString(), -// "default", -// null, -// null, -// 0, -// new MockPayload("data") -// ); -// } -// -// @NotNull -// private static Message fakeFatal() { -// return new Message<>( -// UUID.randomUUID().toString(), -// "default", -// null, -// null, -// 0, -// new MockDeadPayload("data", true) -// ); -// } -// -// @NotNull -// private static Message fakeDeadMessage() { -// return new Message<>( -// UUID.randomUUID().toString(), -// "default", -// null, -// null, -// 0, -// new MockDeadPayload("data", false) -// ); -// } -//} diff --git a/es4j-test/src/test/resources/logback-test.xml b/es4j-test/src/test/resources/logback-test.xml index f926811..1c83cc6 100644 --- a/es4j-test/src/test/resources/logback-test.xml +++ b/es4j-test/src/test/resources/logback-test.xml @@ -6,7 +6,7 @@ %d{HH:mm:ss.SSS} [%thread] [%vcl{AGGREGATE}] %-5level %logger{0} - %msg%n - + diff --git a/pom.xml b/pom.xml index 0989894..e24f73b 100644 --- a/pom.xml +++ b/pom.xml @@ -15,13 +15,14 @@ es4j-dependencies es4j-infrastructure es4j-stack + es4j-extensions es4j - A framework based on the vert.x stack that aims to facilitate the implementation of event sourcing + A framework based on the vert.x stack that aims to facilitate the implementation of event sourcing + https://github.com/Reef3rm4n/es4j - 17 17 @@ -59,61 +60,61 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From e7281bf068f7bc1bbe4186e3274bcd93c71b59e5 Mon Sep 17 00:00:00 2001 From: reeferman Date: Sat, 22 Jul 2023 22:15:45 +0200 Subject: [PATCH 16/17] - postgres broker tests --- pom.xml | 110 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/pom.xml b/pom.xml index e24f73b..53a7bb1 100644 --- a/pom.xml +++ b/pom.xml @@ -60,61 +60,61 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + org.apache.maven.plugins + maven-gpg-plugin + ${maven-gpg-plugin.version} + + + sign-artifacts + verify + + sign + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.2.0 + + + attach-javadoc + + jar + + + + + io.es4j.core.objects + io.es4j.infrastructure.models + + + + org.apache.maven.plugins + maven-source-plugin + ${maven-source-plugin.version} + + + attach-source + + jar + + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + ${nexus-staging-maven-plugin.version} + true + + ossrh + https://s01.oss.sonatype.org/ + true + + From 43f6d8e4a939f9ac813a5f337c14d56a4fda9603 Mon Sep 17 00:00:00 2001 From: reeferman Date: Sat, 22 Jul 2023 22:21:59 +0200 Subject: [PATCH 17/17] - postgres broker tests --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 53a7bb1..3957a0f 100644 --- a/pom.xml +++ b/pom.xml @@ -87,8 +87,8 @@ - io.es4j.core.objects - io.es4j.infrastructure.models + io.es4j.* + io.es4j