diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/logging/DataPrepperMarkers.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/logging/DataPrepperMarkers.java index 94cf06979e..a15dd52480 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/logging/DataPrepperMarkers.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/logging/DataPrepperMarkers.java @@ -6,6 +6,7 @@ public final class DataPrepperMarkers { public static final Marker EVENT = MarkerFactory.getMarker("EVENT"); public static final Marker SENSITIVE = MarkerFactory.getMarker("SENSITIVE"); + public static final Marker NOISY = MarkerFactory.getMarker("NOISY"); static { EVENT.add(SENSITIVE); diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/DelayProcessor.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/DelayProcessor.java index b6681fea19..471c7c5c8f 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/DelayProcessor.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/DelayProcessor.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.annotation.JsonClassDescription; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.processor.Processor; @@ -34,7 +35,7 @@ public Collection> execute(final Collection> records) { try { Thread.sleep(delayDuration.toMillis()); } catch (final InterruptedException ex) { - LOG.error("Interrupted during delay processor", ex); + LOG.error(NOISY, "Interrupted during delay processor", ex); } return records; } diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java index c3d908a8da..ff18a1e629 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/processor/StringProcessor.java @@ -8,6 +8,11 @@ import com.fasterxml.jackson.annotation.JsonClassDescription; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.configuration.PluginSetting; @@ -15,9 +20,6 @@ import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.processor.Processor; import org.opensearch.dataprepper.model.record.Record; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,8 +28,6 @@ import java.util.Map; import java.util.stream.Collectors; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - /** * A simple String implementation of {@link Processor} which generates new Records with uppercase or lowercase content. The current * simpler implementation does not handle errors (if any). @@ -84,7 +84,13 @@ public Collection> execute(final Collection> records .build(); modifiedRecords.add(new Record<>(newRecordEvent)); } catch (JsonProcessingException e) { - LOG.error(EVENT, "Unable to process Event data: {}", eventJson, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("Unable to process Event data: {}") + .addArgument(eventJson) + .setCause(e) + .log(); } } return modifiedRecords; diff --git a/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessor.java b/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessor.java index 3e8780a417..e9b4be67df 100644 --- a/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessor.java +++ b/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessor.java @@ -5,7 +5,14 @@ package org.opensearch.dataprepper.plugins.processor.csv; +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvParser; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import io.micrometer.core.instrument.Counter; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -14,11 +21,6 @@ import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.processor.Processor; import org.opensearch.dataprepper.model.record.Record; -import com.fasterxml.jackson.databind.MappingIterator; -import com.fasterxml.jackson.dataformat.csv.CsvMapper; -import com.fasterxml.jackson.dataformat.csv.CsvParser; -import com.fasterxml.jackson.dataformat.csv.CsvSchema; -import io.micrometer.core.instrument.Counter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,8 +30,6 @@ import java.util.List; import java.util.Objects; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - /** * Processor to parse CSV data in Events. * @@ -105,7 +105,13 @@ public Collection> doExecute(final Collection> recor } } catch (final IOException e) { csvInvalidEventsCounter.increment(); - LOG.error(EVENT, "An exception occurred while reading event [{}]", event, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("An exception occurred while reading event [{}]") + .addArgument(event) + .setCause(e) + .log(); } } return records; diff --git a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java index 328f8294bc..8c85250cb8 100644 --- a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java +++ b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java @@ -8,6 +8,7 @@ import io.micrometer.core.instrument.Counter; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -102,7 +103,7 @@ public Collection> doExecute(Collection> records) { record.getData().put(dateProcessorConfig.getDestination(), zonedDateTime); } } catch (final Exception e) { - LOG.error("An exception occurred while attempting to process Event: ", e); + LOG.error(NOISY,"An exception occurred while attempting to process Event: ", e); } } diff --git a/data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/DissectProcessor.java b/data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/DissectProcessor.java index cc93a9a4d1..50453c6131 100644 --- a/data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/DissectProcessor.java +++ b/data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/DissectProcessor.java @@ -6,6 +6,8 @@ package org.opensearch.dataprepper.plugins.processor.dissect; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -26,8 +28,6 @@ import java.util.Map; import java.util.Objects; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - @DataPrepperPlugin(name = "dissect", pluginType = Processor.class, pluginConfigurationType = DissectProcessorConfig.class) public class DissectProcessor extends AbstractProcessor, Record> { @@ -72,7 +72,13 @@ public Collection> doExecute(Collection> records) { } } } catch (Exception ex){ - LOG.error(EVENT, "Error dissecting the event [{}] ", record.getData(), ex); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("Error dissecting the event [{}]") + .addArgument(record.getData()) + .setCause(ex) + .log(); } } return records; diff --git a/data-prepper-plugins/flatten-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/flatten/FlattenProcessor.java b/data-prepper-plugins/flatten-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/flatten/FlattenProcessor.java index 0e22a6db10..24a8dbf2bc 100644 --- a/data-prepper-plugins/flatten-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/flatten/FlattenProcessor.java +++ b/data-prepper-plugins/flatten-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/flatten/FlattenProcessor.java @@ -7,6 +7,7 @@ import com.github.wnameless.json.flattener.JsonFlattener; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -87,7 +88,7 @@ public Collection> doExecute(final Collection> recor updateEvent(recordEvent, flattenedJson); } catch (Exception e) { - LOG.error("Fail to perform flatten operation", e); + LOG.error(NOISY, "Fail to perform flatten operation", e); recordEvent.getMetadata().addTags(config.getTagsOnFailure()); } finally { // Add temporarily deleted fields back diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessor.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessor.java index a264d7f66a..748ef2847a 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessor.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessor.java @@ -8,6 +8,7 @@ import io.micrometer.core.instrument.Counter; import org.opensearch.dataprepper.expression.ExpressionEvaluator; import org.opensearch.dataprepper.logging.DataPrepperMarkers; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -107,7 +108,7 @@ public Collection> doExecute(final Collection> recor try (final GeoIPDatabaseReader geoIPDatabaseReader = BatchGeoIPDatabaseReader.decorate(geoIPProcessorService.getGeoIPDatabaseReader())) { processRecords(records, geoIPDatabaseReader); } catch (final Exception e) { - LOG.error("Encountered exception in geoip processor.", e); + LOG.error(NOISY, "Encountered exception in geoip processor.", e); } return records; } diff --git a/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java index 6470830e49..5605fbee45 100644 --- a/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java +++ b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java @@ -12,6 +12,8 @@ import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Timer; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -21,6 +23,8 @@ import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.processor.Processor; import org.opensearch.dataprepper.model.record.Record; +import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig.TOTAL_PATTERNS_ATTEMPTED_METADATA_KEY; +import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig.TOTAL_TIME_SPENT_IN_GROK_METADATA_KEY; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,10 +58,6 @@ import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; -import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig.TOTAL_PATTERNS_ATTEMPTED_METADATA_KEY; -import static org.opensearch.dataprepper.plugins.processor.grok.GrokProcessorConfig.TOTAL_TIME_SPENT_IN_GROK_METADATA_KEY; - @SingleThread @DataPrepperPlugin(name = "grok", pluginType = Processor.class, pluginConfigurationType = GrokProcessorConfig.class) @@ -153,11 +153,25 @@ public Collection> doExecute(final Collection> recor } catch (final TimeoutException e) { event.getMetadata().addTags(tagsOnTimeout); - LOG.error(EVENT, "Matching on record [{}] took longer than [{}] and timed out", record.getData(), grokProcessorConfig.getTimeoutMillis()); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("Matching on record [{}] took longer than [{}] and timed out") + .addArgument(record.getData()) + .addArgument(grokProcessorConfig.getTimeoutMillis()) + .log(); + grokProcessingTimeoutsCounter.increment(); } catch (final ExecutionException | InterruptedException | RuntimeException e) { event.getMetadata().addTags(tagsOnMatchFailure); - LOG.error(EVENT, "An exception occurred when matching record [{}]", record.getData(), e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("An exception occurred when matching record [{}]") + .addArgument(record.getData()) + .setCause(e) + .log(); + grokProcessingErrorsCounter.increment(); } diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java index 816e03c211..321d4eebdc 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/AddEntryProcessor.java @@ -6,6 +6,8 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -26,8 +28,6 @@ import java.util.function.Consumer; import java.util.function.Supplier; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - @DataPrepperPlugin(name = "add_entries", pluginType = Processor.class, pluginConfigurationType = AddEntryProcessorConfig.class) public class AddEntryProcessor extends AbstractProcessor, Record> { private static final Logger LOG = LoggerFactory.getLogger(AddEntryProcessor.class); @@ -92,12 +92,28 @@ public Collection> doExecute(final Collection> recor } } } catch (Exception e) { - LOG.error(EVENT, "Error adding entry to record [{}] with key [{}], metadataKey [{}], value_expression [{}] format [{}], value [{}]", - recordEvent, entry.getKey(), entry.getMetadataKey(), entry.getValueExpression(), entry.getFormat(), entry.getValue(), e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("Error adding entry to record [{}] with key [{}], metadataKey [{}], value_expression [{}] format [{}], value [{}]") + .addArgument(recordEvent) + .addArgument(entry.getKey()) + .addArgument(entry.getMetadataKey()) + .addArgument(entry.getValueExpression()) + .addArgument(entry.getFormat()) + .addArgument(entry.getValue()) + .setCause(e) + .log(); } } } catch(final Exception e){ - LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("There was an exception while processing Event [{}]") + .addArgument(recordEvent) + .setCause(e) + .log(); } } diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java index 53ca543c17..2f71e6fc03 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessor.java @@ -6,6 +6,8 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -24,8 +26,6 @@ import java.util.List; import java.util.Objects; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - @DataPrepperPlugin(name = "convert_entry_type", pluginType = Processor.class, pluginConfigurationType = ConvertEntryTypeProcessorConfig.class) public class ConvertEntryTypeProcessor extends AbstractProcessor, Record> { private static final Logger LOG = LoggerFactory.getLogger(ConvertEntryTypeProcessor.class); @@ -91,7 +91,13 @@ public Collection> doExecute(final Collection> recor } } } catch (final Exception e) { - LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("There was an exception while processing Event [{}]") + .addArgument(recordEvent) + .setCause(e) + .log(); recordEvent.getMetadata().addTags(tagsOnFailure); } } diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessor.java index 0ed56b1b8d..fdaae6ab19 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessor.java @@ -6,6 +6,8 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -24,8 +26,6 @@ import java.util.Map; import java.util.Objects; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - @DataPrepperPlugin(name = "copy_values", pluginType = Processor.class, pluginConfigurationType = CopyValueProcessorConfig.class) public class CopyValueProcessor extends AbstractProcessor, Record> { private static final Logger LOG = LoggerFactory.getLogger(CopyValueProcessor.class); @@ -93,7 +93,13 @@ public Collection> doExecute(final Collection> recor } } } catch (final Exception e) { - LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("There was an exception while processing Event [{}]") + .addArgument(recordEvent) + .setCause(e) + .log(); } } diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessor.java index 5dc0ee8b42..8e483cfd05 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/DeleteEntryProcessor.java @@ -6,6 +6,8 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -22,8 +24,6 @@ import java.util.List; import java.util.Objects; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - @DataPrepperPlugin(name = "delete_entries", pluginType = Processor.class, pluginConfigurationType = DeleteEntryProcessorConfig.class) public class DeleteEntryProcessor extends AbstractProcessor, Record> { @@ -62,7 +62,13 @@ public Collection> doExecute(final Collection> recor recordEvent.delete(entry); } } catch (final Exception e) { - LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("There was an exception while processing Event [{}]") + .addArgument(recordEvent) + .setCause(e) + .log(); } } diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessor.java index 9c8655a4cb..7cc6ba237a 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ListToMapProcessor.java @@ -6,6 +6,8 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -24,8 +26,6 @@ import java.util.Map; import java.util.Objects; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - @DataPrepperPlugin(name = "list_to_map", pluginType = Processor.class, pluginConfigurationType = ListToMapProcessorConfig.class) public class ListToMapProcessor extends AbstractProcessor, Record> { @@ -78,7 +78,13 @@ public Collection> doExecute(final Collection> recor recordEvent.getMetadata().addTags(config.getTagsOnFailure()); continue; } catch (final Exception e) { - LOG.error(EVENT, "Error converting source list to map on record [{}]", recordEvent, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("Error converting source list to map on record [{}]") + .addArgument(recordEvent) + .setCause(e) + .log(); recordEvent.getMetadata().addTags(config.getTagsOnFailure()); continue; } @@ -86,11 +92,23 @@ public Collection> doExecute(final Collection> recor try { updateEvent(recordEvent, targetMap); } catch (final Exception e) { - LOG.error(EVENT, "Error updating record [{}] after converting source list to map", recordEvent, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("Error updating record [{}] after converting source list to map") + .addArgument(recordEvent) + .setCause(e) + .log(); recordEvent.getMetadata().addTags(config.getTagsOnFailure()); } } catch (final Exception e) { - LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("There was an exception while processing Event [{}]") + .addArgument(recordEvent) + .setCause(e) + .log(); recordEvent.getMetadata().addTags(config.getTagsOnFailure()); } } diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java index 38b5c3cf82..af291ebc05 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java @@ -8,6 +8,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -28,8 +30,6 @@ import java.util.Objects; import java.util.Set; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - @DataPrepperPlugin(name = "map_to_list", pluginType = Processor.class, pluginConfigurationType = MapToListProcessorConfig.class) public class MapToListProcessor extends AbstractProcessor, Record> { private static final Logger LOG = LoggerFactory.getLogger(MapToListProcessor.class); @@ -92,11 +92,22 @@ public Collection> doExecute(final Collection> recor recordEvent.put(config.getTarget(), targetList); } } catch (Exception e) { - LOG.error("Fail to perform Map to List operation", e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("Fail to perform Map to List operation") + .setCause(e) + .log(); recordEvent.getMetadata().addTags(config.getTagsOnFailure()); } } catch (final Exception e) { - LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("There was an exception while processing Event [{}]") + .addArgument(recordEvent) + .setCause(e) + .log(); recordEvent.getMetadata().addTags(config.getTagsOnFailure()); } } diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java index 25fb27a9fe..b75aef2f52 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/RenameKeyProcessor.java @@ -6,6 +6,8 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -21,8 +23,6 @@ import java.util.List; import java.util.Objects; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - @DataPrepperPlugin(name = "rename_keys", pluginType = Processor.class, pluginConfigurationType = RenameKeyProcessorConfig.class) public class RenameKeyProcessor extends AbstractProcessor, Record> { @@ -70,7 +70,13 @@ public Collection> doExecute(final Collection> recor } } } catch (final Exception e) { - LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("There was an exception while processing Event [{}]") + .addArgument(recordEvent) + .setCause(e) + .log(); } } diff --git a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/AbstractStringProcessor.java b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/AbstractStringProcessor.java index ae7a242da3..5ea2abcd57 100644 --- a/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/AbstractStringProcessor.java +++ b/data-prepper-plugins/mutate-string-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutatestring/AbstractStringProcessor.java @@ -5,21 +5,20 @@ package org.opensearch.dataprepper.plugins.processor.mutatestring; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.record.Record; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Collection; import java.util.List; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - public abstract class AbstractStringProcessor extends AbstractProcessor, Record> { private final List entries; static final Logger LOG = LoggerFactory.getLogger(AbstractStringProcessor.class); @@ -37,7 +36,14 @@ public Collection> doExecute(final Collection> recor try { performStringAction(recordEvent); } catch (final Exception e) { - LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); + LOG.error(NOISY, "There was an exception while processing Event [{}]", recordEvent, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("There was an exception while processing Event [{}]") + .addArgument(recordEvent) + .setCause(e) + .log(); } } diff --git a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java index 472ffec940..50a05d86cb 100644 --- a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java +++ b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessor.java @@ -6,6 +6,8 @@ package org.opensearch.dataprepper.plugins.processor.obfuscation; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -29,8 +31,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - @DataPrepperPlugin(name = "obfuscate", pluginType = Processor.class, pluginConfigurationType = ObfuscationProcessorConfig.class) public class ObfuscationProcessor extends AbstractProcessor, Record> { @@ -99,7 +99,12 @@ public ObfuscationProcessor(final PluginMetrics pluginMetrics, Pattern p = Pattern.compile(rawPattern); patterns.add(p); } catch (Exception e) { - LOG.error(e.getMessage()); + LOG.error(NOISY,e.getMessage()); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage(e.getMessage()) + .log(); throw new InvalidPluginConfigurationException("Invalid Pattern: \"" + rawPattern + "\" for source field " + this.source); } } diff --git a/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessor.java b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessor.java index d4c69bb331..682936c5b7 100644 --- a/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessor.java +++ b/data-prepper-plugins/translate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/translate/TranslateProcessor.java @@ -8,6 +8,8 @@ import org.apache.commons.lang3.Range; import org.apache.commons.lang3.math.NumberUtils; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; @@ -32,8 +34,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - @DataPrepperPlugin(name = "translate", pluginType = Processor.class, pluginConfigurationType = TranslateProcessorConfig.class) public class TranslateProcessor extends AbstractProcessor, Record> { @@ -67,8 +67,14 @@ public Collection> doExecute(Collection> records) { translateSource(sourceObject, recordEvent, targetConfig); } } catch (Exception ex) { - LOG.error(EVENT, "Error mapping the source [{}] of entry [{}]", mappingConfig.getSource(), - record.getData(), ex); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("Error mapping the source [{}] of entry [{}]") + .addArgument(mappingConfig.getSource()) + .addArgument(record.getData()) + .setCause(ex) + .log(); } } } diff --git a/data-prepper-plugins/truncate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/truncate/TruncateProcessor.java b/data-prepper-plugins/truncate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/truncate/TruncateProcessor.java index 6d29082bd6..6b5e4dd9fb 100644 --- a/data-prepper-plugins/truncate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/truncate/TruncateProcessor.java +++ b/data-prepper-plugins/truncate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/truncate/TruncateProcessor.java @@ -6,24 +6,24 @@ package org.opensearch.dataprepper.plugins.processor.truncate; import org.opensearch.dataprepper.expression.ExpressionEvaluator; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.plugin.InvalidPluginConfigurationException; -import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.processor.AbstractProcessor; import org.opensearch.dataprepper.model.processor.Processor; +import org.opensearch.dataprepper.model.record.Record; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collection; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; - /** * This processor takes in a key and truncates its value to a string with * characters from the front or at the end or at both removed. @@ -115,7 +115,13 @@ public Collection> doExecute(final Collection> recor } } } catch (final Exception e) { - LOG.error(EVENT, "There was an exception while processing Event [{}]", recordEvent, e); + LOG.atError() + .addMarker(EVENT) + .addMarker(NOISY) + .setMessage("There was an exception while processing Event [{}]") + .addArgument(recordEvent) + .setCause(e) + .log(); } } diff --git a/data-prepper-plugins/write-json-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/write_json/WriteJsonProcessor.java b/data-prepper-plugins/write-json-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/write_json/WriteJsonProcessor.java index 8e9ce379ce..51902aeb6d 100644 --- a/data-prepper-plugins/write-json-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/write_json/WriteJsonProcessor.java +++ b/data-prepper-plugins/write-json-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/write_json/WriteJsonProcessor.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import io.micrometer.core.instrument.Counter; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.metrics.PluginMetrics; @@ -52,7 +53,7 @@ public Collection> doExecute(Collection> records) { try { event.put(target, objectMapper.writeValueAsString(value)); } catch (Exception e) { - LOG.error("Failed to convert source to json string", e); + LOG.error(NOISY, "Failed to convert source to json string", e); writeJsonFailedCounter.increment(); } }