From 78fd1c3ec228debad049f60e5dbc3a6e17c1b656 Mon Sep 17 00:00:00 2001 From: Dinu John <86094133+dinujoh@users.noreply.github.com> Date: Wed, 11 Sep 2024 18:11:17 -0500 Subject: [PATCH 1/2] Move CsvMapper and Schema creation to constructor Signed-off-by: Dinu John <86094133+dinujoh@users.noreply.github.com> --- .../dataprepper/plugins/processor/csv/CsvProcessor.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessor.java b/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessor.java index 3e8780a417..405b165ac5 100644 --- a/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessor.java +++ b/data-prepper-plugins/csv-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessor.java @@ -46,6 +46,9 @@ public class CsvProcessor extends AbstractProcessor, Record private final ExpressionEvaluator expressionEvaluator; + private final CsvMapper mapper; + private final CsvSchema schema; + @DataPrepperPluginConstructor public CsvProcessor(final PluginMetrics pluginMetrics, final CsvProcessorConfig config, @@ -61,13 +64,12 @@ public CsvProcessor(final PluginMetrics pluginMetrics, String.format("csv_when value of %s is not a valid expression statement. " + "See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax.", config.getCsvWhen())); } + this.mapper = createCsvMapper(); + this.schema = createCsvSchema(); } @Override public Collection> doExecute(final Collection> records) { - final CsvMapper mapper = createCsvMapper(); - final CsvSchema schema = createCsvSchema(); - for (final Record record : records) { final Event event = record.getData(); From 5c11fb94cdc22d77d64ab03070ab86f004c892c4 Mon Sep 17 00:00:00 2001 From: Dinu John <86094133+dinujoh@users.noreply.github.com> Date: Wed, 11 Sep 2024 23:07:31 -0500 Subject: [PATCH 2/2] Fix CsvProcessor unit test Signed-off-by: Dinu John <86094133+dinujoh@users.noreply.github.com> --- .../dataprepper/plugins/processor/csv/CsvProcessorTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data-prepper-plugins/csv-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessorTest.java b/data-prepper-plugins/csv-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessorTest.java index cacfedb609..0e36d8ebea 100644 --- a/data-prepper-plugins/csv-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessorTest.java +++ b/data-prepper-plugins/csv-processor/src/test/java/org/opensearch/dataprepper/plugins/processor/csv/CsvProcessorTest.java @@ -117,6 +117,7 @@ void test_when_messageIsEmpty_then_notParsed() { @Test void test_when_delimiterIsTab_then_parsedCorrectly() { when(processorConfig.getDelimiter()).thenReturn("\t"); + csvProcessor = createObjectUnderTest(); Record eventUnderTest = createMessageEvent("1\t2\t3"); final List> editedEvents = (List>) csvProcessor.doExecute(Collections.singletonList(eventUnderTest)); @@ -293,6 +294,7 @@ void test_when_twoEventsAndNoHeaderSourceOnOne_then_ChooseUserColumns() { @Test void test_when_differentQuoteCharacter_then_parsesCorrectly() { when(processorConfig.getQuoteCharacter()).thenReturn("\'"); + csvProcessor = createObjectUnderTest(); final Record eventUnderTest = createMessageEvent("'1','2','3'"); final List> editedEvents = (List>) csvProcessor.doExecute(Collections.singletonList(eventUnderTest));