Skip to content

Commit 105690e

Browse files
jeortizquanjeortizquan
committed
MET-5960 debias processing
1 parent 3d5ddc1 commit 105690e

15 files changed

+396
-24
lines changed

‎src/main/java/eu/europeana/metis/sandbox/common/Step.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ public enum Step {
1414
ENRICH("enrich", 8),
1515
MEDIA_PROCESS("process media", 9),
1616
PUBLISH("publish", 10),
17-
CLOSE("close", 11);
17+
CLOSE("close", 11),
18+
DEBIAS("debias", 12);
1819

1920
private final String value;
2021
private final int precedence;

‎src/main/java/eu/europeana/metis/sandbox/config/DeBiasConfig.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@
33

44
import eu.europeana.metis.debias.detect.client.DeBiasClient;
55
import eu.europeana.metis.sandbox.repository.DatasetRepository;
6+
import eu.europeana.metis.sandbox.repository.RecordLogRepository;
67
import eu.europeana.metis.sandbox.repository.debias.DetectRepository;
78
import eu.europeana.metis.sandbox.service.debias.DeBiasDetectService;
89
import eu.europeana.metis.sandbox.service.debias.DetectService;
10+
import eu.europeana.metis.sandbox.service.debias.RecordPublishable;
911
import org.springframework.beans.factory.annotation.Value;
1012
import org.springframework.context.annotation.Bean;
1113
import org.springframework.context.annotation.Configuration;
@@ -32,8 +34,11 @@ public class DeBiasConfig {
3234
* @return the detect service
3335
*/
3436
@Bean
35-
public DetectService debiasMachine(DetectRepository detectRepository, DatasetRepository datasetRepository) {
36-
return new DeBiasDetectService(detectRepository, datasetRepository);
37+
public DetectService debiasMachine(DetectRepository detectRepository,
38+
DatasetRepository datasetRepository,
39+
RecordLogRepository recordLogRepository,
40+
RecordPublishable recordPublishable) {
41+
return new DeBiasDetectService(detectRepository, datasetRepository, recordLogRepository, recordPublishable);
3742
}
3843

3944
@Bean

‎src/main/java/eu/europeana/metis/sandbox/config/SandboxConfig.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ class SandboxConfig {
6666
@Value("${sandbox.rabbitmq.queues.record.created.queue}")
6767
private String createdQueue;
6868

69+
@Value("${sandbox.rabbitmq.queues.record.debias.ready.queue}")
70+
private String deBiasReadyQueue;
71+
6972
@Value("${sandbox.rabbitmq.queues.record.transformation.edm.external.queue}")
7073
private String transformationToEdmExternalQueue;
7174

@@ -134,6 +137,11 @@ String createdQueue() {
134137
return createdQueue;
135138
}
136139

140+
@Bean(name = "deBiasReadyQueue")
141+
String deBiasReadyQueue() {
142+
return deBiasReadyQueue;
143+
}
144+
137145
@Bean(name = "transformationToEdmExternalQueue")
138146
String transformationToEdmExternalQueue() {
139147
return transformationToEdmExternalQueue;

‎src/main/java/eu/europeana/metis/sandbox/config/amqp/AmqpConfiguration.java

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,11 @@ public class AmqpConfiguration {
8585
@Value("${sandbox.rabbitmq.queues.record.published.dlq:#{null}}")
8686
private String publishedDlq;
8787

88+
@Value("${sandbox.rabbitmq.queues.record.debias.ready.queue:#{null}}")
89+
private String deBiasReadyQueue;
90+
91+
@Value("${sandbox.rabbitmq.queues.record.debias.ready.dlq:#{null}}")
92+
private String deBiasReadyDlq;
8893

8994
public AmqpConfiguration(MessageConverter messageConverter, AmqpAdmin amqpAdmin) {
9095
this.messageConverter = messageConverter;
@@ -120,22 +125,23 @@ Declarables deadQueues() {
120125
QueueBuilder.durable(internalValidatedDlq).build(),
121126
QueueBuilder.durable(enrichedDlq).build(),
122127
QueueBuilder.durable(mediaProcessedDlq).build(),
123-
QueueBuilder.durable(publishedDlq).build()
128+
QueueBuilder.durable(publishedDlq).build(),
129+
QueueBuilder.durable(deBiasReadyDlq).build()
124130
);
125131
}
126132

127133
@Bean
128134
Declarables bindings() {
129135
return getDeclarables(exchange, createdQueue, transformationToEdmExternalQueue,
130136
externalValidatedQueue, transformedQueue, normalizedQueue, internalValidatedQueue,
131-
enrichedQueue, mediaProcessedQueue, publishedQueue);
137+
enrichedQueue, mediaProcessedQueue, publishedQueue, deBiasReadyQueue);
132138
}
133139

134140
@Bean
135141
Declarables dlqBindings() {
136142
return getDeclarables(exchangeDlq, createdDlq, transformationToEdmExternalDlq,
137143
externalValidatedDlq, transformedDlq, normalizedDlq, internalValidatedDlq, enrichedDlq,
138-
mediaProcessedDlq, publishedDlq);
144+
mediaProcessedDlq, publishedDlq, deBiasReadyDlq);
139145
}
140146

141147
//Suppress: Methods should not have too many parameters warning
@@ -145,7 +151,7 @@ Declarables dlqBindings() {
145151
private Declarables getDeclarables(String exchange, String created,
146152
String transformationToEdmExternal, String externalValidated, String transformed,
147153
String normalized, String internalValidated, String enriched, String mediaProcessed,
148-
String published) {
154+
String published, String deBiasReady) {
149155
return new Declarables(
150156
new Binding(created, DestinationType.QUEUE, exchange, created, null),
151157
new Binding(transformationToEdmExternal, DestinationType.QUEUE, exchange,
@@ -156,7 +162,8 @@ private Declarables getDeclarables(String exchange, String created,
156162
new Binding(internalValidated, DestinationType.QUEUE, exchange, internalValidated, null),
157163
new Binding(enriched, DestinationType.QUEUE, exchange, enriched, null),
158164
new Binding(mediaProcessed, DestinationType.QUEUE, exchange, mediaProcessed, null),
159-
new Binding(published, DestinationType.QUEUE, exchange, published, null)
165+
new Binding(published, DestinationType.QUEUE, exchange, published, null),
166+
new Binding(deBiasReady, DestinationType.QUEUE, exchange, deBiasReady, null)
160167
);
161168
}
162169

@@ -181,7 +188,9 @@ Declarables queues() {
181188
QueueBuilder.durable(mediaProcessedQueue).deadLetterExchange(exchangeDlq)
182189
.deadLetterRoutingKey(mediaProcessedDlq).build(),
183190
QueueBuilder.durable(publishedQueue).deadLetterExchange(exchangeDlq)
184-
.deadLetterRoutingKey(publishedDlq).build()
191+
.deadLetterRoutingKey(publishedDlq).build(),
192+
QueueBuilder.durable(deBiasReadyQueue).deadLetterExchange(deBiasReadyDlq)
193+
.deadLetterRoutingKey(deBiasReadyDlq).build()
185194
);
186195
}
187196

@@ -271,6 +280,14 @@ public String getPublishedDlq() {
271280
return publishedDlq;
272281
}
273282

283+
public String getDeBiasReadyQueue() {
284+
return deBiasReadyQueue;
285+
}
286+
287+
public String getDeBiasReadyDlq() {
288+
return deBiasReadyDlq;
289+
}
290+
274291
public AmqpAdmin getAmqpAdmin(){
275292
return amqpAdmin;
276293
}
@@ -279,6 +296,6 @@ public List<String> getAllQueuesNames(){
279296
return List.of(createdQueue, createdDlq, transformationToEdmExternalQueue, transformationToEdmExternalDlq,
280297
externalValidatedQueue, externalValidatedDlq, transformedQueue, transformedDlq, internalValidatedQueue,
281298
internalValidatedDlq, normalizedQueue, normalizedDlq, enrichedQueue, enrichedDlq, mediaProcessedQueue,
282-
mediaProcessedDlq, publishedQueue, publishedDlq);
299+
mediaProcessedDlq, publishedQueue, publishedDlq, deBiasReadyQueue, deBiasReadyDlq);
283300
}
284301
}

‎src/main/java/eu/europeana/metis/sandbox/config/amqp/DeBiasReadyQueueConfig.java

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package eu.europeana.metis.sandbox.config.amqp;
2+
3+
import org.springframework.amqp.rabbit.config.SimpleRabbitListenerContainerFactory;
4+
import org.springframework.amqp.rabbit.connection.ConnectionFactory;
5+
import org.springframework.amqp.support.converter.MessageConverter;
6+
import org.springframework.beans.factory.annotation.Value;
7+
import org.springframework.boot.autoconfigure.amqp.SimpleRabbitListenerContainerFactoryConfigurer;
8+
import org.springframework.context.annotation.Bean;
9+
import org.springframework.context.annotation.Configuration;
10+
11+
/**
12+
* The type De bias ready queue config.
13+
*/
14+
@Configuration
15+
public class DeBiasReadyQueueConfig {
16+
17+
private final MessageConverter messageConverter;
18+
19+
@Value("${sandbox.rabbitmq.queues.record.debias.ready.concurrency}")
20+
private int concurrentConsumers;
21+
22+
@Value("${sandbox.rabbitmq.queues.record.debias.ready.max-concurrency}")
23+
private int maxConsumers;
24+
25+
@Value("${sandbox.rabbitmq.queues.record.debias.ready.prefetch}")
26+
private int messagePrefetchCount;
27+
28+
@Value("${sandbox.rabbitmq.queues.record.debias.ready.batch-size}")
29+
private int batchSize;
30+
31+
/**
32+
* Instantiates a new De bias ready queue config.
33+
*
34+
* @param messageConverter the message converter
35+
*/
36+
public DeBiasReadyQueueConfig(MessageConverter messageConverter) {
37+
this.messageConverter = messageConverter;
38+
}
39+
40+
/**
41+
* Closing factory simple rabbit listener container factory.
42+
*
43+
* @param configurer the configurer
44+
* @param connectionFactory the connection factory
45+
* @return the simple rabbit listener container factory
46+
*/
47+
@Bean
48+
SimpleRabbitListenerContainerFactory deBiasFactory(
49+
SimpleRabbitListenerContainerFactoryConfigurer configurer,
50+
ConnectionFactory connectionFactory) {
51+
var factory = new SimpleRabbitListenerContainerFactory();
52+
configurer.configure(factory, connectionFactory);
53+
factory.setConcurrentConsumers(concurrentConsumers);
54+
factory.setMaxConcurrentConsumers(maxConsumers);
55+
factory.setPrefetchCount(messagePrefetchCount);
56+
factory.setMessageConverter(messageConverter);
57+
factory.setConsumerBatchEnabled(true);
58+
factory.setBatchListener(true);
59+
factory.setBatchSize(batchSize);
60+
return factory;
61+
62+
}
63+
}

‎src/main/java/eu/europeana/metis/sandbox/executor/workflow/DeBiasExecutor.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package eu.europeana.metis.sandbox.executor.workflow;
2+
3+
import eu.europeana.metis.sandbox.common.Step;
4+
import eu.europeana.metis.sandbox.domain.RecordProcessEvent;
5+
import eu.europeana.metis.sandbox.service.workflow.DeBiasProcessService;
6+
import java.util.List;
7+
import org.slf4j.Logger;
8+
import org.slf4j.LoggerFactory;
9+
import org.springframework.amqp.core.AmqpTemplate;
10+
import org.springframework.amqp.rabbit.annotation.RabbitListener;
11+
import org.springframework.beans.factory.annotation.Value;
12+
import org.springframework.stereotype.Component;
13+
14+
/**
15+
* Consumes debias events and performs processing to the contained record <br/> the result will be stored on database
16+
*/
17+
@Component
18+
class DeBiasExecutor extends StepExecutor {
19+
20+
private static final Logger LOGGER = LoggerFactory.getLogger(DeBiasExecutor.class);
21+
private final DeBiasProcessService service;
22+
@Value("${sandbox.rabbitmq.queues.record.debias.ready.queue}")
23+
private String routingKey;
24+
25+
public DeBiasExecutor(AmqpTemplate amqpTemplate,
26+
DeBiasProcessService service) {
27+
super(amqpTemplate);
28+
this.service = service;
29+
}
30+
31+
@RabbitListener(queues = "${sandbox.rabbitmq.queues.record.debias.ready.queue}",
32+
containerFactory = "deBiasFactory",
33+
autoStartup = "${sandbox.rabbitmq.queues.record.transformed.auto-start:true}")
34+
public void debiasProcess(List<RecordProcessEvent> input) {
35+
input.forEach(r -> LOGGER.info("pulling record {} from queue", r.getRecord().getRecordId()));
36+
consumeBatch(routingKey, input, Step.DEBIAS,
37+
() -> service.process(input.stream().map(RecordProcessEvent::getRecord).toList()));
38+
}
39+
}

‎src/main/java/eu/europeana/metis/sandbox/executor/workflow/StepExecutor.java

Lines changed: 53 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import eu.europeana.metis.sandbox.domain.RecordError;
77
import eu.europeana.metis.sandbox.domain.RecordInfo;
88
import eu.europeana.metis.sandbox.domain.RecordProcessEvent;
9+
import java.util.ArrayList;
910
import java.util.List;
1011
import java.util.function.Supplier;
1112
import org.slf4j.Logger;
@@ -20,10 +21,23 @@ class StepExecutor {
2021
private static final Logger LOGGER = LoggerFactory.getLogger(StepExecutor.class);
2122
private final AmqpTemplate amqpTemplate;
2223

24+
/**
25+
* Instantiates a new Step executor.
26+
*
27+
* @param amqpTemplate the amqp template
28+
*/
2329
StepExecutor(AmqpTemplate amqpTemplate) {
2430
this.amqpTemplate = amqpTemplate;
2531
}
2632

33+
/**
34+
* Consume.
35+
*
36+
* @param routingKey the routing key
37+
* @param input the input
38+
* @param step the step
39+
* @param recordInfoSupplier the record info supplier
40+
*/
2741
public void consume(String routingKey, RecordProcessEvent input, Step step,
2842
Supplier<RecordInfo> recordInfoSupplier) {
2943
if (input.getStatus() == Status.FAIL) {
@@ -48,10 +62,48 @@ public void consume(String routingKey, RecordProcessEvent input, Step step,
4862
}
4963
}
5064

65+
/**
66+
* Consume batch.
67+
*
68+
* @param routingKey the routing key
69+
* @param input the input
70+
* @param step the step
71+
* @param recordInfoSupplier the record info supplier
72+
*/
73+
public void consumeBatch(String routingKey, List<RecordProcessEvent> input, Step step,
74+
Supplier<List<RecordInfo>> recordInfoSupplier) {
75+
List<RecordProcessEvent> outputRecordProcessEvents = new ArrayList<>();
76+
try {
77+
var listRecordInfo = recordInfoSupplier.get();
78+
for (RecordInfo recordInfo : listRecordInfo) {
79+
var status = recordInfo.getErrors().isEmpty() ? Status.SUCCESS : Status.WARN;
80+
outputRecordProcessEvents.add(new RecordProcessEvent(recordInfo, step, status));
81+
}
82+
} catch (RecordProcessingException ex) {
83+
for (RecordProcessEvent recordProcessEvent : input) {
84+
outputRecordProcessEvents.add(createFailEvent(recordProcessEvent, step, ex));
85+
}
86+
} catch (Exception ex) {
87+
for (RecordProcessEvent recordProcessEvent : input) {
88+
outputRecordProcessEvents.add(
89+
createFailEvent(recordProcessEvent, step,
90+
new RecordProcessingException(Long.toString(recordProcessEvent.getRecord().getRecordId()), ex)
91+
)
92+
);
93+
}
94+
}
95+
try {
96+
outputRecordProcessEvents.forEach(output -> amqpTemplate.convertAndSend(routingKey, output));
97+
} catch (RuntimeException rabbitException) {
98+
LOGGER.error("Queue step execution error", rabbitException);
99+
}
100+
}
101+
51102
private RecordProcessEvent createFailEvent(RecordProcessEvent input, Step step, RecordProcessingException ex) {
52103
final String stepName = step.value();
53104
final RecordError recordError = new RecordError(ex);
54-
final RecordProcessEvent output = new RecordProcessEvent(new RecordInfo(input.getRecord(), List.of(recordError)), step, Status.FAIL);
105+
final RecordProcessEvent output = new RecordProcessEvent(new RecordInfo(input.getRecord(), List.of(recordError)), step,
106+
Status.FAIL);
55107
LOGGER.error("Exception while performing step: [{}]. ", stepName, ex);
56108
return output;
57109
}

‎src/main/java/eu/europeana/metis/sandbox/repository/RecordLogRepository.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,16 @@ public interface RecordLogRepository extends JpaRepository<RecordLogEntity, Long
6868
"AND rle.recordId.datasetId = ?2 AND rle.step IN ?3")
6969
Set<RecordLogEntity> findRecordLogByRecordIdDatasetIdAndStepIn(String recordId, String datasetId, Set<Step> steps);
7070

71+
/**
72+
* Find record log by dataset id and step set.
73+
*
74+
* @param datasetId the dataset id
75+
* @param step the step
76+
* @return the set
77+
*/
78+
@Query("SELECT rle FROM RecordLogEntity rle WHERE rle.recordId.datasetId = ?1 AND rle.step = ?2")
79+
Set<RecordLogEntity> findRecordLogByDatasetIdAndStep(String datasetId, Step step);
80+
7181
/**
7282
* Delete records that belong to the given dataset id
7383
*

‎src/main/java/eu/europeana/metis/sandbox/service/debias/DeBiasDetectService.java

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import eu.europeana.metis.sandbox.dto.debias.DetectionInfoDto;
44
import eu.europeana.metis.sandbox.entity.debias.DetectionEntity;
55
import eu.europeana.metis.sandbox.repository.DatasetRepository;
6+
import eu.europeana.metis.sandbox.repository.RecordLogRepository;
67
import eu.europeana.metis.sandbox.repository.debias.DetectRepository;
78
import java.time.ZonedDateTime;
89
import org.springframework.transaction.annotation.Transactional;
@@ -24,9 +25,15 @@ public class DeBiasDetectService implements DetectService {
2425
* Instantiates a new DeBias detect service.
2526
*
2627
* @param detectRepository the detect repository
28+
* @param datasetRepository the dataset repository
29+
* @param recordLogRepository the record log repository
30+
* @param recordPublishable the record publishable
2731
*/
28-
public DeBiasDetectService(DetectRepository detectRepository, DatasetRepository datasetRepository) {
29-
this.ready = new ReadyState(this, detectRepository, datasetRepository);
32+
public DeBiasDetectService(DetectRepository detectRepository,
33+
DatasetRepository datasetRepository,
34+
RecordLogRepository recordLogRepository,
35+
RecordPublishable recordPublishable) {
36+
this.ready = new ReadyState(this, detectRepository, datasetRepository, recordLogRepository, recordPublishable);
3037
this.processing = new ProcessingState(this, detectRepository);
3138
this.completed = new CompletedState(this, detectRepository);
3239
this.error = new ErrorState(this, detectRepository);

0 commit comments

Comments
 (0)