From 52d2f0ed1b48b3bfeef9d5eeb573f047215a9782 Mon Sep 17 00:00:00 2001 From: David Venable Date: Mon, 17 Jun 2024 10:40:28 -0500 Subject: [PATCH] Adds the EventKey and EventKeyFactory. (#4627) Adds the EventKey and EventKeyFactory. Resolves #1916. Signed-off-by: David Venable --- .../dataprepper/model/event/Event.java | 66 ++++++ .../dataprepper/model/event/EventKey.java | 21 ++ .../model/event/EventKeyFactory.java | 71 ++++++ .../dataprepper/model/event/JacksonEvent.java | 206 +++++++++--------- .../model/event/JacksonEventKey.java | 137 ++++++++++++ .../model/event/EventActionTest.java | 69 ++++++ .../model/event/EventKeyFactoryTest.java | 47 ++++ .../model/event/JacksonEventKeyTest.java | 179 +++++++++++++++ .../model/event/JacksonEventTest.java | 137 +++++++++++- .../integration/ProcessorPipelineIT.java | 118 ++++++++++ .../plugins/InMemorySourceAccessor.java | 17 +- .../dataprepper/plugins/SimpleProcessor.java | 56 +++++ .../plugins/SimpleProcessorConfig.java | 19 ++ .../pipeline/processor-pipeline.yaml | 14 ++ .../core/event/DefaultEventKeyFactory.java | 20 ++ .../event/InternalOnlyEventKeyBridge.java | 17 ++ .../event/DefaultEventKeyFactoryTest.java | 52 +++++ .../ApplicationContextToTypedSuppliers.java | 3 + ...pplicationContextToTypedSuppliersTest.java | 17 +- .../dataprepper/event/TestEventContext.java | 24 ++ .../dataprepper/event/TestEventFactory.java | 10 +- .../event/TestEventKeyFactory.java | 30 +++ .../event/TestEventKeyFactoryTest.java | 56 +++++ 23 files changed, 1268 insertions(+), 118 deletions(-) create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/EventKey.java create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/EventKeyFactory.java create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEventKey.java create mode 100644 data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/EventActionTest.java create mode 100644 data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/EventKeyFactoryTest.java create mode 100644 data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventKeyTest.java create mode 100644 data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/integration/ProcessorPipelineIT.java create mode 100644 data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/SimpleProcessor.java create mode 100644 data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/SimpleProcessorConfig.java create mode 100644 data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/processor-pipeline.yaml create mode 100644 data-prepper-event/src/main/java/org/opensearch/dataprepper/core/event/DefaultEventKeyFactory.java create mode 100644 data-prepper-event/src/main/java/org/opensearch/dataprepper/model/event/InternalOnlyEventKeyBridge.java create mode 100644 data-prepper-event/src/test/java/org/opensearch/dataprepper/core/event/DefaultEventKeyFactoryTest.java create mode 100644 data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventContext.java create mode 100644 data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventKeyFactory.java create mode 100644 data-prepper-test-event/src/test/java/org/opensearch/dataprepper/event/TestEventKeyFactoryTest.java diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java index 740447ecc0..e0e36d9237 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java @@ -26,6 +26,15 @@ */ public interface Event extends Serializable { + /** + * Adds or updates the key with a given value in the Event + * + * @param key where the value will be set + * @param value value to set the key to + * @since 2.8 + */ + void put(EventKey key, Object value); + /** * Adds or updates the key with a given value in the Event * @@ -35,6 +44,17 @@ public interface Event extends Serializable { */ void put(String key, Object value); + /** + * Retrieves the given key from the Event + * + * @param key the value to retrieve from + * @param clazz the return type of the value + * @param The type + * @return T a clazz object from the key + * @since 2.8 + */ + T get(EventKey key, Class clazz); + /** * Retrieves the given key from the Event * @@ -46,6 +66,17 @@ public interface Event extends Serializable { */ T get(String key, Class clazz); + /** + * Retrieves the given key from the Event as a List + * + * @param key the value to retrieve from + * @param clazz the return type of elements in the list + * @param The type + * @return {@literal List} a list of clazz elements + * @since 2.8 + */ + List getList(EventKey key, Class clazz); + /** * Retrieves the given key from the Event as a List * @@ -57,6 +88,14 @@ public interface Event extends Serializable { */ List getList(String key, Class clazz); + /** + * Deletes the given key from the Event + * + * @param key the field to be deleted + * @since 2.8 + */ + void delete(EventKey key); + /** * Deletes the given key from the Event * @@ -87,6 +126,15 @@ public interface Event extends Serializable { */ JsonNode getJsonNode(); + /** + * Gets a serialized Json string of the specific key in the Event + * + * @param key the field to be returned + * @return Json string of the field + * @since 2.8 + */ + String getAsJsonString(EventKey key); + /** * Gets a serialized Json string of the specific key in the Event * @@ -104,6 +152,15 @@ public interface Event extends Serializable { */ EventMetadata getMetadata(); + /** + * Checks if the key exists. + * + * @param key name of the key to look for + * @return returns true if the key exists, otherwise false + * @since 2.8 + */ + boolean containsKey(EventKey key); + /** * Checks if the key exists. * @@ -113,6 +170,15 @@ public interface Event extends Serializable { */ boolean containsKey(String key); + /** + * Checks if the value stored for the key is list + * + * @param key name of the key to look for + * @return returns true if the key is a list, otherwise false + * @since 2.8 + */ + boolean isValueAList(EventKey key); + /** * Checks if the value stored for the key is list * diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/EventKey.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/EventKey.java new file mode 100644 index 0000000000..9086f0f641 --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/EventKey.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.model.event; + +/** + * Model class to represent a key into a Data Prepper {@link Event}. + * + * @since 2.9 + */ +public interface EventKey { + /** + * The original key provided as a string. + * + * @return The key as a string + * @since 2.9 + */ + String getKey(); +} diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/EventKeyFactory.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/EventKeyFactory.java new file mode 100644 index 0000000000..e7cbc25463 --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/EventKeyFactory.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.model.event; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +/** + * A factory for producing {@link EventKey} objects. + * + * @since 2.9 + */ +public interface EventKeyFactory { + /** + * Creates an {@link EventKey} with given actions. + * + * @param key The key + * @param forActions Actions to support + * @return The EventKey + * @since 2.9 + */ + EventKey createEventKey(String key, EventAction... forActions); + + /** + * Creates an {@link EventKey} for the default actions, which are all. + * + * @param key The key + * @return The EventKey + * @since 2.9 + */ + default EventKey createEventKey(final String key) { + return createEventKey(key, EventAction.ALL); + } + + /** + * An action on an Event. + * + * @since 2.9 + */ + enum EventAction { + GET, + DELETE, + PUT, + ALL(GET, DELETE, PUT); + + private final List includedActions; + + EventAction(EventAction... eventActions) { + includedActions = Arrays.asList(eventActions); + + } + + boolean isMutableAction() { + return this != GET; + } + + Set getSupportedActions() { + final EnumSet supportedActions = EnumSet.noneOf(EventAction.class); + supportedActions.add(this); + supportedActions.addAll(includedActions); + + return Collections.unmodifiableSet(supportedActions); + } + } +} diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java index 9ef34bb82c..35e0dd863b 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java @@ -28,8 +28,8 @@ import java.io.ObjectInputStream; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; +import java.util.Deque; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; @@ -38,8 +38,8 @@ import java.util.Objects; import java.util.StringJoiner; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static org.opensearch.dataprepper.model.event.JacksonEventKey.trimTrailingSlashInKey; /** * A Jackson Implementation of {@link Event} interface. This implementation relies heavily on JsonNode to manage the keys of the event. @@ -137,20 +137,15 @@ public JsonNode getJsonNode() { return jsonNode; } - /** - * Adds or updates the key with a given value in the Event. - * - * @param key where the value will be set - * @param value value to set the key to - * @since 1.2 - */ @Override - public void put(final String key, final Object value) { - checkArgument(!key.isEmpty(), "key cannot be an empty string for put method"); + public void put(EventKey key, Object value) { + final JacksonEventKey jacksonEventKey = asJacksonEventKey(key); - final String trimmedKey = checkAndTrimKey(key); + if(!jacksonEventKey.supports(EventKeyFactory.EventAction.PUT)) { + throw new IllegalArgumentException("key cannot be an empty string for put method"); + } - final LinkedList keys = new LinkedList<>(Arrays.asList(trimmedKey.split(SEPARATOR, -1))); + final Deque keys = new LinkedList<>(jacksonEventKey.getKeyPathList()); JsonNode parentNode = jsonNode; @@ -166,6 +161,19 @@ public void put(final String key, final Object value) { } } + /** + * Adds or updates the key with a given value in the Event. + * + * @param key where the value will be set + * @param value value to set the key to + * @since 1.2 + */ + @Override + public void put(final String key, final Object value) { + final JacksonEventKey jacksonEventKey = new JacksonEventKey(key, EventKeyFactory.EventAction.PUT); + put(jacksonEventKey, value); + } + @Override public EventHandle getEventHandle() { return eventHandle; @@ -189,6 +197,27 @@ private JsonNode getOrCreateNode(final JsonNode node, final String key) { return childNode; } + @Override + public T get(EventKey key, Class clazz) { + JacksonEventKey jacksonEventKey = asJacksonEventKey(key); + + final JsonNode node = getNode(jacksonEventKey); + if (node.isMissingNode()) { + return null; + } + + return mapNodeToObject(key.getKey(), node, clazz); + } + + private static JacksonEventKey asJacksonEventKey(EventKey key) { + if(!(key instanceof JacksonEventKey)) { + throw new IllegalArgumentException("The key provided must be obtained through the EventKeyFactory."); + } + + JacksonEventKey jacksonEventKey = (JacksonEventKey) key; + return jacksonEventKey; + } + /** * Retrieves the value of type clazz from the key. * @@ -200,15 +229,8 @@ private JsonNode getOrCreateNode(final JsonNode node, final String key) { */ @Override public T get(final String key, final Class clazz) { - - final String trimmedKey = checkAndTrimKey(key); - - final JsonNode node = getNode(trimmedKey); - if (node.isMissingNode()) { - return null; - } - - return mapNodeToObject(key, node, clazz); + final JacksonEventKey jacksonEventKey = new JacksonEventKey(key, EventKeyFactory.EventAction.GET); + return get(jacksonEventKey, clazz); } private JsonNode getNode(final String key) { @@ -216,6 +238,10 @@ private JsonNode getNode(final String key) { return jsonNode.at(jsonPointer); } + private JsonNode getNode(final JacksonEventKey key) { + return jsonNode.at(key.getJsonPointer()); + } + private T mapNodeToObject(final String key, final JsonNode node, final Class clazz) { try { return mapper.treeToValue(node, clazz); @@ -225,6 +251,18 @@ private T mapNodeToObject(final String key, final JsonNode node, final Class } } + @Override + public List getList(EventKey key, Class clazz) { + JacksonEventKey jacksonEventKey = asJacksonEventKey(key); + + final JsonNode node = getNode(jacksonEventKey); + if (node.isMissingNode()) { + return null; + } + + return mapNodeToList(jacksonEventKey.getKey(), node, clazz); + } + /** * Retrieves the given key from the Event as a List * @@ -236,15 +274,8 @@ private T mapNodeToObject(final String key, final JsonNode node, final Class */ @Override public List getList(final String key, final Class clazz) { - - final String trimmedKey = checkAndTrimKey(key); - - final JsonNode node = getNode(trimmedKey); - if (node.isMissingNode()) { - return null; - } - - return mapNodeToList(key, node, clazz); + JacksonEventKey jacksonEventKey = new JacksonEventKey(key, EventKeyFactory.EventAction.GET); + return getList(jacksonEventKey, clazz); } private List mapNodeToList(final String key, final JsonNode node, final Class clazz) { @@ -267,16 +298,15 @@ private JsonPointer toJsonPointer(final String key) { return JsonPointer.compile(jsonPointerExpression); } - /** - * Deletes the key from the event. - * - * @param key the field to be deleted - */ @Override - public void delete(final String key) { + public void delete(final EventKey key) { + final JacksonEventKey jacksonEventKey = asJacksonEventKey(key); + + if(!jacksonEventKey.supports(EventKeyFactory.EventAction.DELETE)) { + throw new IllegalArgumentException("key cannot be an empty string for delete method"); + } - checkArgument(!key.isEmpty(), "key cannot be an empty string for delete method"); - final String trimmedKey = checkAndTrimKey(key); + final String trimmedKey = jacksonEventKey.getTrimmedKey(); final int index = trimmedKey.lastIndexOf(SEPARATOR); JsonNode baseNode = jsonNode; @@ -293,6 +323,17 @@ public void delete(final String key) { } } + /** + * Deletes the key from the event. + * + * @param key the field to be deleted + */ + @Override + public void delete(final String key) { + final JacksonEventKey jacksonEventKey = new JacksonEventKey(key, EventKeyFactory.EventAction.DELETE); + delete(jacksonEventKey); + } + @Override public void clear() { // Delete all entries from the event @@ -309,16 +350,22 @@ public String toJsonString() { } @Override - public String getAsJsonString(final String key) { - final String trimmedKey = checkAndTrimKey(key); + public String getAsJsonString(EventKey key) { - final JsonNode node = getNode(trimmedKey); + JacksonEventKey jacksonEventKey = asJacksonEventKey(key); + final JsonNode node = getNode(jacksonEventKey); if (node.isMissingNode()) { return null; } return node.toString(); } + @Override + public String getAsJsonString(final String key) { + JacksonEventKey jacksonEventKey = new JacksonEventKey(key, EventKeyFactory.EventAction.GET); + return getAsJsonString(jacksonEventKey); + } + /** * returns a string with formatted parts replaced by their values. The input * string may contain parts with format "${.../.../...}" which are replaced @@ -402,24 +449,35 @@ public EventMetadata getMetadata() { } @Override - public boolean containsKey(final String key) { - - final String trimmedKey = checkAndTrimKey(key); + public boolean containsKey(EventKey key) { + JacksonEventKey jacksonEventKey = asJacksonEventKey(key); - final JsonNode node = getNode(trimmedKey); + final JsonNode node = getNode(jacksonEventKey); return !node.isMissingNode(); } @Override - public boolean isValueAList(final String key) { - final String trimmedKey = checkAndTrimKey(key); + public boolean containsKey(final String key) { + JacksonEventKey jacksonEventKey = new JacksonEventKey(key, EventKeyFactory.EventAction.GET); + return containsKey(jacksonEventKey); + } - final JsonNode node = getNode(trimmedKey); + @Override + public boolean isValueAList(EventKey key) { + JacksonEventKey jacksonEventKey = asJacksonEventKey(key); + + final JsonNode node = getNode(jacksonEventKey); return node.isArray(); } + @Override + public boolean isValueAList(final String key) { + JacksonEventKey jacksonEventKey = new JacksonEventKey(key, EventKeyFactory.EventAction.GET); + return isValueAList(jacksonEventKey); + } + @Override public Map toMap() { return mapper.convertValue(jsonNode, MAP_TYPE_REFERENCE); @@ -427,30 +485,7 @@ public Map toMap() { public static boolean isValidEventKey(final String key) { - try { - checkKey(key); - return true; - } catch (final Exception e) { - return false; - } - } - private String checkAndTrimKey(final String key) { - checkKey(key); - return trimTrailingSlashInKey(key); - } - - private static void checkKey(final String key) { - checkNotNull(key, "key cannot be null"); - if (key.isEmpty()) { - // Empty string key is valid - return; - } - if (key.length() > MAX_KEY_LENGTH) { - throw new IllegalArgumentException("key cannot be longer than " + MAX_KEY_LENGTH + " characters"); - } - if (!isValidKey(key)) { - throw new IllegalArgumentException("key " + key + " must contain only alphanumeric chars with .-_@/ and must follow JsonPointer (ie. 'field/to/key')"); - } + return JacksonEventKey.isValidEventKey(key); } private String trimKey(final String key) { @@ -459,31 +494,6 @@ private String trimKey(final String key) { return trimTrailingSlashInKey(trimmedLeadingSlash); } - private String trimTrailingSlashInKey(final String key) { - return key.length() > 1 && key.endsWith(SEPARATOR) ? key.substring(0, key.length() - 1) : key; - } - - private static boolean isValidKey(final String key) { - for (int i = 0; i < key.length(); i++) { - char c = key.charAt(i); - - if (!(c >= 48 && c <= 57 - || c >= 65 && c <= 90 - || c >= 97 && c <= 122 - || c == '.' - || c == '-' - || c == '_' - || c == '@' - || c == '/' - || c == '[' - || c == ']')) { - - return false; - } - } - return true; - } - /** * Constructs an empty builder. * diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEventKey.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEventKey.java new file mode 100644 index 0000000000..172e6b1254 --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEventKey.java @@ -0,0 +1,137 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.model.event; + +import com.fasterxml.jackson.core.JsonPointer; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkNotNull; + +class JacksonEventKey implements EventKey { + private static final String SEPARATOR = "/"; + private static final int MAX_KEY_LENGTH = 2048; + private final String key; + private final EventKeyFactory.EventAction[] eventActions; + private final String trimmedKey; + private final List keyPathList; + private final JsonPointer jsonPointer; + private final Set supportedActions; + + JacksonEventKey(final String key, final EventKeyFactory.EventAction... eventActions) { + this.key = Objects.requireNonNull(key, "Parameter key cannot be null for EventKey."); + this.eventActions = eventActions.length == 0 ? new EventKeyFactory.EventAction[] { EventKeyFactory.EventAction.ALL } : eventActions; + + if(key.isEmpty()) { + for (final EventKeyFactory.EventAction action : this.eventActions) { + if (action.isMutableAction()) { + throw new IllegalArgumentException("Event key cannot be an empty string for " + action + " actions."); + } + } + } + + trimmedKey = checkAndTrimKey(key); + + keyPathList = Collections.unmodifiableList(Arrays.asList(trimmedKey.split(SEPARATOR, -1))); + jsonPointer = toJsonPointer(trimmedKey); + + supportedActions = EnumSet.noneOf(EventKeyFactory.EventAction.class); + for (final EventKeyFactory.EventAction eventAction : this.eventActions) { + supportedActions.addAll(eventAction.getSupportedActions()); + } + + } + + @Override + public String getKey() { + return key; + } + + String getTrimmedKey() { + return trimmedKey; + } + + List getKeyPathList() { + return keyPathList; + } + + JsonPointer getJsonPointer() { + return jsonPointer; + } + + boolean supports(final EventKeyFactory.EventAction eventAction) { + return supportedActions.contains(eventAction); + } + + private String checkAndTrimKey(final String key) { + checkKey(key); + return trimTrailingSlashInKey(key); + } + + private static void checkKey(final String key) { + checkNotNull(key, "key cannot be null"); + if (key.isEmpty()) { + // Empty string key is valid + return; + } + if (key.length() > MAX_KEY_LENGTH) { + throw new IllegalArgumentException("key cannot be longer than " + MAX_KEY_LENGTH + " characters"); + } + if (!isValidKey(key)) { + throw new IllegalArgumentException("key " + key + " must contain only alphanumeric chars with .-_@/ and must follow JsonPointer (ie. 'field/to/key')"); + } + } + + + static String trimTrailingSlashInKey(final String key) { + return key.length() > 1 && key.endsWith(SEPARATOR) ? key.substring(0, key.length() - 1) : key; + } + + private static boolean isValidKey(final String key) { + for (int i = 0; i < key.length(); i++) { + char c = key.charAt(i); + + if (!(c >= 48 && c <= 57 + || c >= 65 && c <= 90 + || c >= 97 && c <= 122 + || c == '.' + || c == '-' + || c == '_' + || c == '@' + || c == '/' + || c == '[' + || c == ']')) { + + return false; + } + } + return true; + } + + private JsonPointer toJsonPointer(final String key) { + final String jsonPointerExpression; + if (key.isEmpty() || key.startsWith("/")) { + jsonPointerExpression = key; + } else { + jsonPointerExpression = SEPARATOR + key; + } + return JsonPointer.compile(jsonPointerExpression); + } + + static boolean isValidEventKey(final String key) { + try { + checkKey(key); + return true; + } catch (final Exception e) { + return false; + } + } +} diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/EventActionTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/EventActionTest.java new file mode 100644 index 0000000000..edb63fa663 --- /dev/null +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/EventActionTest.java @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.model.event; + +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.junit.jupiter.params.provider.EnumSource; + +import java.util.stream.Stream; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +class EventActionTest { + @ParameterizedTest + @EnumSource(value = EventKeyFactory.EventAction.class, mode = EnumSource.Mode.EXCLUDE, names = {"GET"}) + void isMutableAction_is_true_for_mutable_actions(final EventKeyFactory.EventAction eventAction) { + assertThat(eventAction.isMutableAction(), equalTo(true)); + } + + @ParameterizedTest + @EnumSource(value = EventKeyFactory.EventAction.class, mode = EnumSource.Mode.INCLUDE, names = {"GET"}) + void isMutableAction_is_false_for_mutable_actions(final EventKeyFactory.EventAction eventAction) { + assertThat(eventAction.isMutableAction(), equalTo(false)); + } + + @ParameterizedTest + @EnumSource(value = EventKeyFactory.EventAction.class) + void getSupportedActions_includes_self(final EventKeyFactory.EventAction eventAction) { + assertThat(eventAction.getSupportedActions(), hasItem(eventAction)); + } + + @ParameterizedTest + @EnumSource(value = EventKeyFactory.EventAction.class) + void getSupportedActions_includes_for_all_actions_when_ALL(final EventKeyFactory.EventAction eventAction) { + assertThat(EventKeyFactory.EventAction.ALL.getSupportedActions(), hasItem(eventAction)); + } + + @ParameterizedTest + @ArgumentsSource(SupportsArgumentsProvider.class) + void supports_returns_expected_value(final EventKeyFactory.EventAction eventAction, final EventKeyFactory.EventAction otherAction, final boolean expectedSupports) { + assertThat(eventAction.getSupportedActions().contains(otherAction), equalTo(expectedSupports)); + } + + static class SupportsArgumentsProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext extensionContext) throws Exception { + return Stream.of( + arguments(EventKeyFactory.EventAction.GET, EventKeyFactory.EventAction.PUT, false), + arguments(EventKeyFactory.EventAction.GET, EventKeyFactory.EventAction.DELETE, false), + arguments(EventKeyFactory.EventAction.GET, EventKeyFactory.EventAction.ALL, false), + arguments(EventKeyFactory.EventAction.PUT, EventKeyFactory.EventAction.GET, false), + arguments(EventKeyFactory.EventAction.PUT, EventKeyFactory.EventAction.DELETE, false), + arguments(EventKeyFactory.EventAction.PUT, EventKeyFactory.EventAction.ALL, false), + arguments(EventKeyFactory.EventAction.DELETE, EventKeyFactory.EventAction.GET, false), + arguments(EventKeyFactory.EventAction.DELETE, EventKeyFactory.EventAction.PUT, false), + arguments(EventKeyFactory.EventAction.DELETE, EventKeyFactory.EventAction.ALL, false) + ); + } + } +} \ No newline at end of file diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/EventKeyFactoryTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/EventKeyFactoryTest.java new file mode 100644 index 0000000000..c2ed2d56f3 --- /dev/null +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/EventKeyFactoryTest.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.model.event; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class EventKeyFactoryTest { + + private String keyPath; + + @Mock + private EventKey eventKey; + + @BeforeEach + void setUp() { + keyPath = UUID.randomUUID().toString(); + } + + private EventKeyFactory createObjectUnderTest() { + return mock(EventKeyFactory.class); + } + + @Test + void createEventKey_calls_with_ALL_action() { + final EventKeyFactory objectUnderTest = createObjectUnderTest(); + when(objectUnderTest.createEventKey(anyString())).thenCallRealMethod(); + when(objectUnderTest.createEventKey(keyPath, EventKeyFactory.EventAction.ALL)).thenReturn(eventKey); + + assertThat(objectUnderTest.createEventKey(keyPath), equalTo(eventKey)); + } +} \ No newline at end of file diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventKeyTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventKeyTest.java new file mode 100644 index 0000000000..359db06278 --- /dev/null +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventKeyTest.java @@ -0,0 +1,179 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.model.event; + +import com.fasterxml.jackson.core.JsonPointer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.sameInstance; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +class JacksonEventKeyTest { + @Test + void constructor_throws_with_null_key() { + assertThrows(NullPointerException.class, () -> new JacksonEventKey(null)); + } + + @Test + void getKey_with_empty_string_for_GET() { + final JacksonEventKey objectUnderTest = new JacksonEventKey("", EventKeyFactory.EventAction.GET); + assertThat(objectUnderTest.getKey(), equalTo("")); + assertThat(objectUnderTest.getTrimmedKey(), equalTo("")); + assertThat(objectUnderTest.getKeyPathList(), notNullValue()); + assertThat(objectUnderTest.getKeyPathList(), equalTo(List.of(""))); + assertThat(objectUnderTest.getJsonPointer(), notNullValue()); + } + + @ParameterizedTest + @EnumSource(value = EventKeyFactory.EventAction.class, mode = EnumSource.Mode.EXCLUDE, names = {"GET"}) + void constructor_throws_with_empty_string_for_unsupported_actions(final EventKeyFactory.EventAction eventAction) { + assertThrows(IllegalArgumentException.class, () -> new JacksonEventKey("", eventAction)); + } + + + @ParameterizedTest + @ValueSource(strings = { + "inv(alid", + "getMetadata(\"test_key\")" + }) + void constructor_throws_with_invalid_key(final String key) { + assertThrows(IllegalArgumentException.class, () -> new JacksonEventKey(key)); + } + + @ParameterizedTest + @ValueSource(strings = { + "test_key", + "/test_key", + "key.with.dot", + "key-with-hyphen", + "key_with_underscore", + "key@with@at", + "key[with]brackets" + }) + void getKey_returns_expected_result(final String key) { + assertThat(new JacksonEventKey(key).getKey(), equalTo(key)); + } + + @ParameterizedTest + @CsvSource(value = { + "test_key, test_key", + "/test_key, /test_key", + "/test_key/, /test_key", + "key.with.dot, key.with.dot", + "key-with-hyphen, key-with-hyphen", + "key_with_underscore, key_with_underscore", + "key@with@at, key@with@at", + "key[with]brackets, key[with]brackets" + }) + void getTrimmedKey_returns_expected_result(final String key, final String expectedTrimmedKey) { + assertThat(new JacksonEventKey(key).getTrimmedKey(), equalTo(expectedTrimmedKey)); + } + + @ParameterizedTest + @ArgumentsSource(KeyPathListArgumentsProvider.class) + void getKeyPathList_returns_expected_value(final String key, final List expectedKeyPathList) { + assertThat(new JacksonEventKey(key).getKeyPathList(), equalTo(expectedKeyPathList)); + } + + @Test + void getJsonPointer_returns_a_valid_JsonPointer() { + final String testKey = UUID.randomUUID().toString(); + final JacksonEventKey objectUnderTest = new JacksonEventKey(testKey); + + final JsonPointer jsonPointer = objectUnderTest.getJsonPointer(); + assertThat(jsonPointer, notNullValue()); + assertThat(jsonPointer.toString(), equalTo("/" + testKey)); + } + + @Test + void getJsonPointer_returns_the_same_instance_for_multiple_calls() { + final String testKey = UUID.randomUUID().toString(); + final JacksonEventKey objectUnderTest = new JacksonEventKey(testKey); + + final JsonPointer jsonPointer = objectUnderTest.getJsonPointer(); + assertThat(objectUnderTest.getJsonPointer(), sameInstance(jsonPointer)); + assertThat(objectUnderTest.getJsonPointer(), sameInstance(jsonPointer)); + } + + @ParameterizedTest + @ArgumentsSource(SupportsArgumentsProvider.class) + void supports_returns_true_if_any_supports(final List eventActionsList, final EventKeyFactory.EventAction otherAction, final boolean expectedSupports) { + final String testKey = UUID.randomUUID().toString(); + final EventKeyFactory.EventAction[] eventActions = new EventKeyFactory.EventAction[eventActionsList.size()]; + eventActionsList.toArray(eventActions); + assertThat(new JacksonEventKey(testKey, eventActions).supports(otherAction), equalTo(expectedSupports)); + } + + @ParameterizedTest + @CsvSource(value = { + "test_key, true", + "/test_key, true", + "inv(alid, false", + "getMetadata(\"test_key\"), false", + "key.with.dot, true", + "key-with-hyphen, true", + "key_with_underscore, true", + "key@with@at, true", + "key[with]brackets, true" + }) + void isValidEventKey_returns_expected_result(final String key, final boolean isValid) { + assertThat(JacksonEventKey.isValidEventKey(key), equalTo(isValid)); + } + + + static class KeyPathListArgumentsProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext extensionContext) { + return Stream.of( + arguments("test_key", List.of("test_key")), + arguments("a/b", List.of("a", "b")), + arguments("a/b/", List.of("a", "b")), + arguments("a/b/c", List.of("a", "b", "c")), + arguments("a/b/c/", List.of("a", "b", "c")) + ); + } + } + + static class SupportsArgumentsProvider implements ArgumentsProvider { + @Override + public Stream provideArguments(final ExtensionContext extensionContext) throws Exception { + return Stream.of( + arguments(List.of(), EventKeyFactory.EventAction.GET, true), + arguments(List.of(), EventKeyFactory.EventAction.PUT, true), + arguments(List.of(), EventKeyFactory.EventAction.DELETE, true), + arguments(List.of(), EventKeyFactory.EventAction.ALL, true), + arguments(List.of(EventKeyFactory.EventAction.GET), EventKeyFactory.EventAction.GET, true), + arguments(List.of(EventKeyFactory.EventAction.PUT), EventKeyFactory.EventAction.PUT, true), + arguments(List.of(EventKeyFactory.EventAction.DELETE), EventKeyFactory.EventAction.DELETE, true), + arguments(List.of(EventKeyFactory.EventAction.GET), EventKeyFactory.EventAction.PUT, false), + arguments(List.of(EventKeyFactory.EventAction.GET, EventKeyFactory.EventAction.PUT), EventKeyFactory.EventAction.PUT, true), + arguments(List.of(EventKeyFactory.EventAction.PUT, EventKeyFactory.EventAction.GET), EventKeyFactory.EventAction.PUT, true), + arguments(List.of(EventKeyFactory.EventAction.DELETE), EventKeyFactory.EventAction.PUT, false), + arguments(List.of(EventKeyFactory.EventAction.DELETE, EventKeyFactory.EventAction.GET), EventKeyFactory.EventAction.PUT, false), + arguments(List.of(EventKeyFactory.EventAction.DELETE, EventKeyFactory.EventAction.GET, EventKeyFactory.EventAction.PUT), EventKeyFactory.EventAction.PUT, true), + arguments(List.of(EventKeyFactory.EventAction.ALL), EventKeyFactory.EventAction.GET, true), + arguments(List.of(EventKeyFactory.EventAction.ALL), EventKeyFactory.EventAction.PUT, true), + arguments(List.of(EventKeyFactory.EventAction.ALL), EventKeyFactory.EventAction.DELETE, true) + ); + } + } +} \ No newline at end of file diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java index 1a7efb7467..90645d2961 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java @@ -74,6 +74,53 @@ public void testPutAndGet_withRandomString() { assertThat(result, is(equalTo(value))); } + @Test + public void testPutAndGet_withRandomString_eventKey() { + final EventKey key = new JacksonEventKey("aRandomKey" + UUID.randomUUID()); + final UUID value = UUID.randomUUID(); + + event.put(key, value); + final UUID result = event.get(key, UUID.class); + + assertThat(result, is(notNullValue())); + assertThat(result, is(equalTo(value))); + } + + @Test + public void testPutAndGet_withRandomString_eventKey_multiple_events() { + final EventKey key = new JacksonEventKey("aRandomKey" + UUID.randomUUID()); + final UUID value = UUID.randomUUID(); + + for(int i = 0; i < 10; i++) { + event = JacksonEvent.builder() + .withEventType(eventType) + .build(); + + event.put(key, value); + final UUID result = event.get(key, UUID.class); + + assertThat(result, is(notNullValue())); + assertThat(result, is(equalTo(value))); + } + } + + @Test + public void testPutAndGet_eventKey_with_non_JacksonEventKey_throws() { + final EventKey key = mock(EventKey.class); + final UUID value = UUID.randomUUID(); + + assertThrows(IllegalArgumentException.class, () -> event.put(key, value)); + assertThrows(IllegalArgumentException.class, () -> event.get(key, UUID.class)); + } + + @Test + public void testPut_eventKey_with_immutable_action() { + final EventKey key = new JacksonEventKey("aRandomKey" + UUID.randomUUID(), EventKeyFactory.EventAction.GET); + final UUID value = UUID.randomUUID(); + + assertThrows(IllegalArgumentException.class, () -> event.put(key, value)); + } + @ParameterizedTest @ValueSource(strings = {"/", "foo", "foo-bar", "foo_bar", "foo.bar", "/foo", "/foo/", "a1K.k3-01_02", "keyWithBrackets[]"}) void testPutAndGet_withStrings(final String key) { @@ -86,6 +133,19 @@ void testPutAndGet_withStrings(final String key) { assertThat(result, is(equalTo(value))); } + @ParameterizedTest + @ValueSource(strings = {"/", "foo", "foo-bar", "foo_bar", "foo.bar", "/foo", "/foo/", "a1K.k3-01_02", "keyWithBrackets[]"}) + void testPutAndGet_withStrings_eventKey(final String key) { + final UUID value = UUID.randomUUID(); + + final EventKey eventKey = new JacksonEventKey(key); + event.put(eventKey, value); + final UUID result = event.get(eventKey, UUID.class); + + assertThat(result, is(notNullValue())); + assertThat(result, is(equalTo(value))); + } + @Test public void testPutKeyCannotBeEmptyString() { Throwable exception = assertThrows(IllegalArgumentException.class, () -> event.put("", "value")); @@ -93,7 +153,7 @@ public void testPutKeyCannotBeEmptyString() { } @Test - public void testPutAndGet_withMultLevelKey() { + public void testPutAndGet_withMultiLevelKey() { final String key = "foo/bar"; final UUID value = UUID.randomUUID(); @@ -104,6 +164,18 @@ public void testPutAndGet_withMultLevelKey() { assertThat(result, is(equalTo(value))); } + @Test + public void testPutAndGet_withMultiLevelKey_eventKey() { + final EventKey key = new JacksonEventKey("foo/bar"); + final UUID value = UUID.randomUUID(); + + event.put(key, value); + final UUID result = event.get(key, UUID.class); + + assertThat(result, is(notNullValue())); + assertThat(result, is(equalTo(value))); + } + @Test public void testPutAndGet_withMultiLevelKeyTwice() { final String key = "foo/bar"; @@ -125,6 +197,27 @@ public void testPutAndGet_withMultiLevelKeyTwice() { assertThat(result2, is(equalTo(value2))); } + @Test + public void testPutAndGet_withMultiLevelKeyTwice_eventKey() { + final EventKey key = new JacksonEventKey("foo/bar"); + final UUID value = UUID.randomUUID(); + + event.put(key, value); + final UUID result = event.get(key, UUID.class); + + assertThat(result, is(notNullValue())); + assertThat(result, is(equalTo(value))); + + final EventKey key2 = new JacksonEventKey("foo/fizz"); + final UUID value2 = UUID.randomUUID(); + + event.put(key2, value2); + final UUID result2 = event.get(key2, UUID.class); + + assertThat(result2, is(notNullValue())); + assertThat(result2, is(equalTo(value2))); + } + @Test public void testPutAndGet_withMultiLevelKeyWithADash() { final String key = "foo/bar-bar"; @@ -137,6 +230,18 @@ public void testPutAndGet_withMultiLevelKeyWithADash() { assertThat(result, is(equalTo(value))); } + @Test + public void testPutAndGet_withMultiLevelKeyWithADash_eventKey() { + final EventKey key = new JacksonEventKey("foo/bar-bar"); + final UUID value = UUID.randomUUID(); + + event.put(key, value); + final UUID result = event.get(key, UUID.class); + + assertThat(result, is(notNullValue())); + assertThat(result, is(equalTo(value))); + } + @ParameterizedTest @ValueSource(strings = {"foo", "/foo", "/foo/", "foo/"}) void testGetAtRootLevel(final String key) { @@ -148,6 +253,17 @@ void testGetAtRootLevel(final String key) { assertThat(result, is(Map.of("foo", value))); } + @ParameterizedTest + @ValueSource(strings = {"foo", "/foo", "/foo/", "foo/"}) + void testGetAtRootLevel_eventKey(final String key) { + final String value = UUID.randomUUID().toString(); + + event.put(new JacksonEventKey(key), value); + final Map result = event.get(new JacksonEventKey("", EventKeyFactory.EventAction.GET), Map.class); + + assertThat(result, is(Map.of("foo", value))); + } + @ParameterizedTest @ValueSource(strings = {"/foo/bar", "foo/bar", "foo/bar/"}) void testGetAtRootLevelWithMultiLevelKey(final String key) { @@ -159,6 +275,17 @@ void testGetAtRootLevelWithMultiLevelKey(final String key) { assertThat(result, is(Map.of("foo", Map.of("bar", value)))); } + @ParameterizedTest + @ValueSource(strings = {"/foo/bar", "foo/bar", "foo/bar/"}) + void testGetAtRootLevelWithMultiLevelKey_eventKey(final String key) { + final String value = UUID.randomUUID().toString(); + + event.put(new JacksonEventKey(key), value); + final Map result = event.get( new JacksonEventKey("", EventKeyFactory.EventAction.GET), Map.class); + + assertThat(result, is(Map.of("foo", Map.of("bar", value)))); + } + @Test public void testPutUpdateAndGet_withPojo() { final String key = "foo/bar"; @@ -293,6 +420,14 @@ public void testDeleteKey(final String key) { assertThat(result, is(nullValue())); } + @Test + public void testDelete_eventKey_with_immutable_action() { + final EventKey key = new JacksonEventKey("aRandomKey" + UUID.randomUUID(), EventKeyFactory.EventAction.GET); + final UUID value = UUID.randomUUID(); + + assertThrows(IllegalArgumentException.class, () -> event.delete(key)); + } + @Test public void testClear() { event.put("key1", UUID.randomUUID()); diff --git a/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/integration/ProcessorPipelineIT.java b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/integration/ProcessorPipelineIT.java new file mode 100644 index 0000000000..8aaf401e15 --- /dev/null +++ b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/integration/ProcessorPipelineIT.java @@ -0,0 +1,118 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.integration; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.plugins.InMemorySinkAccessor; +import org.opensearch.dataprepper.plugins.InMemorySourceAccessor; +import org.opensearch.dataprepper.test.framework.DataPrepperTestRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.empty; + +class ProcessorPipelineIT { + private static final Logger LOG = LoggerFactory.getLogger(ProcessorPipelineIT.class); + private static final String IN_MEMORY_IDENTIFIER = "ProcessorPipelineIT"; + private static final String PIPELINE_CONFIGURATION_UNDER_TEST = "processor-pipeline.yaml"; + private DataPrepperTestRunner dataPrepperTestRunner; + private InMemorySourceAccessor inMemorySourceAccessor; + private InMemorySinkAccessor inMemorySinkAccessor; + + @BeforeEach + void setUp() { + dataPrepperTestRunner = DataPrepperTestRunner.builder() + .withPipelinesDirectoryOrFile(PIPELINE_CONFIGURATION_UNDER_TEST) + .build(); + + inMemorySourceAccessor = dataPrepperTestRunner.getInMemorySourceAccessor(); + inMemorySinkAccessor = dataPrepperTestRunner.getInMemorySinkAccessor(); + dataPrepperTestRunner.start(); + LOG.info("Started test runner."); + } + + @AfterEach + void tearDown() { + LOG.info("Test tear down. Stop the test runner."); + dataPrepperTestRunner.stop(); + } + + @Test + void run_with_single_record() { + final String messageValue = UUID.randomUUID().toString(); + final Event event = JacksonEvent.fromMessage(messageValue); + final Record eventRecord = new Record<>(event); + + LOG.info("Submitting a single record."); + inMemorySourceAccessor.submit(IN_MEMORY_IDENTIFIER, Collections.singletonList(eventRecord)); + + await().atMost(400, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + assertThat(inMemorySinkAccessor.get(IN_MEMORY_IDENTIFIER), not(empty())); + }); + + final List> records = inMemorySinkAccessor.get(IN_MEMORY_IDENTIFIER); + + assertThat(records.size(), equalTo(1)); + + assertThat(records.get(0), notNullValue()); + assertThat(records.get(0).getData(), notNullValue()); + assertThat(records.get(0).getData().get("message", String.class), equalTo(messageValue)); + assertThat(records.get(0).getData().get("test1", String.class), equalTo("knownPrefix10")); + } + + @Test + void pipeline_with_single_batch_of_records() { + final int recordsToCreate = 200; + final List> inputRecords = IntStream.range(0, recordsToCreate) + .mapToObj(i -> UUID.randomUUID().toString()) + .map(JacksonEvent::fromMessage) + .map(Record::new) + .collect(Collectors.toList()); + + LOG.info("Submitting a batch of record."); + inMemorySourceAccessor.submit(IN_MEMORY_IDENTIFIER, inputRecords); + + await().atMost(400, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + assertThat(inMemorySinkAccessor.get(IN_MEMORY_IDENTIFIER), not(empty())); + }); + + assertThat(inMemorySinkAccessor.get(IN_MEMORY_IDENTIFIER).size(), equalTo(recordsToCreate)); + + final List> sinkRecords = inMemorySinkAccessor.get(IN_MEMORY_IDENTIFIER); + + for (int i = 0; i < sinkRecords.size(); i++) { + final Record inputRecord = inputRecords.get(i); + final Record sinkRecord = sinkRecords.get(i); + assertThat(sinkRecord, notNullValue()); + final Event recordData = sinkRecord.getData(); + assertThat(recordData, notNullValue()); + assertThat( + recordData.get("message", String.class), + equalTo(inputRecord.getData().get("message", String.class))); + assertThat(recordData.get("test1", String.class), + equalTo("knownPrefix1" + i)); + } + } +} diff --git a/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/InMemorySourceAccessor.java b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/InMemorySourceAccessor.java index 71151be22e..3957d259a9 100644 --- a/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/InMemorySourceAccessor.java +++ b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/InMemorySourceAccessor.java @@ -6,20 +6,19 @@ package org.opensearch.dataprepper.plugins; import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.JacksonEvent; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.model.event.EventFactory; import org.opensearch.dataprepper.model.event.EventBuilder; +import org.opensearch.dataprepper.model.event.EventFactory; +import org.opensearch.dataprepper.model.record.Record; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.UUID; import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.atomic.AtomicBoolean; /** * Provides a mechanism to write records to an in_memory source. This allows the pipeline to execute @@ -62,8 +61,8 @@ public void submit(final String testingKey, int numRecords) { for (int i = 0; i < numRecords; i++) { Map eventMap = Map.of("message", UUID.randomUUID().toString()); EventBuilder eventBuilder = (EventBuilder) eventFactory.eventBuilder(EventBuilder.class).withData(eventMap); - JacksonEvent event = (JacksonEvent) eventBuilder.build(); - records.add(new Record(event)); + Event event = eventBuilder.build(); + records.add(new Record<>(event)); } submit(testingKey, records); } @@ -79,8 +78,8 @@ public void submitWithStatus(final String testingKey, int numRecords) { int status = (int)(Math.random() * (max - min + 1) + min); Map eventMap = Map.of("message", UUID.randomUUID().toString(), "status", status); EventBuilder eventBuilder = (EventBuilder) eventFactory.eventBuilder(EventBuilder.class).withData(eventMap); - JacksonEvent event = (JacksonEvent) eventBuilder.build(); - records.add(new Record(event)); + Event event = eventBuilder.build(); + records.add(new Record<>(event)); } submit(testingKey, records); } diff --git a/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/SimpleProcessor.java b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/SimpleProcessor.java new file mode 100644 index 0000000000..bc59deb138 --- /dev/null +++ b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/SimpleProcessor.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins; + +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyFactory; +import org.opensearch.dataprepper.model.processor.Processor; +import org.opensearch.dataprepper.model.record.Record; + +import java.util.Collection; + +@DataPrepperPlugin(name = "simple_test", pluginType = Processor.class, pluginConfigurationType = SimpleProcessorConfig.class) +public class SimpleProcessor implements Processor, Record> { + private final EventKey eventKey1; + private final String valuePrefix1; + int count = 0; + + @DataPrepperPluginConstructor + public SimpleProcessor( + final SimpleProcessorConfig simpleProcessorConfig, + final EventKeyFactory eventKeyFactory) { + eventKey1 = eventKeyFactory.createEventKey(simpleProcessorConfig.getKey1()); + valuePrefix1 = simpleProcessorConfig.getValuePrefix1(); + } + + @Override + public Collection> execute(final Collection> records) { + for (final Record record : records) { + record.getData().put(eventKey1, valuePrefix1 + count); + count++; + } + + return records; + } + + @Override + public void prepareForShutdown() { + + } + + @Override + public boolean isReadyForShutdown() { + return false; + } + + @Override + public void shutdown() { + + } +} diff --git a/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/SimpleProcessorConfig.java b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/SimpleProcessorConfig.java new file mode 100644 index 0000000000..957202431f --- /dev/null +++ b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugins/SimpleProcessorConfig.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins; + +public class SimpleProcessorConfig { + private String key1; + private String valuePrefix1; + + public String getKey1() { + return key1; + } + + public String getValuePrefix1() { + return valuePrefix1; + } +} diff --git a/data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/processor-pipeline.yaml b/data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/processor-pipeline.yaml new file mode 100644 index 0000000000..2223a07c3e --- /dev/null +++ b/data-prepper-core/src/integrationTest/resources/org/opensearch/dataprepper/pipeline/processor-pipeline.yaml @@ -0,0 +1,14 @@ +processor-pipeline: + delay: 10 + source: + in_memory: + testing_key: ProcessorPipelineIT + + processor: + - simple_test: + key1: /test1 + value_prefix1: knownPrefix1 + + sink: + - in_memory: + testing_key: ProcessorPipelineIT diff --git a/data-prepper-event/src/main/java/org/opensearch/dataprepper/core/event/DefaultEventKeyFactory.java b/data-prepper-event/src/main/java/org/opensearch/dataprepper/core/event/DefaultEventKeyFactory.java new file mode 100644 index 0000000000..605b5bcb41 --- /dev/null +++ b/data-prepper-event/src/main/java/org/opensearch/dataprepper/core/event/DefaultEventKeyFactory.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.core.event; + +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyFactory; +import org.opensearch.dataprepper.model.event.InternalOnlyEventKeyBridge; + +import javax.inject.Named; + +@Named +public class DefaultEventKeyFactory implements EventKeyFactory { + @Override + public EventKey createEventKey(final String key, final EventAction... forActions) { + return InternalOnlyEventKeyBridge.createEventKey(key, forActions); + } +} diff --git a/data-prepper-event/src/main/java/org/opensearch/dataprepper/model/event/InternalOnlyEventKeyBridge.java b/data-prepper-event/src/main/java/org/opensearch/dataprepper/model/event/InternalOnlyEventKeyBridge.java new file mode 100644 index 0000000000..130b94db0e --- /dev/null +++ b/data-prepper-event/src/main/java/org/opensearch/dataprepper/model/event/InternalOnlyEventKeyBridge.java @@ -0,0 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.model.event; + +/** + * Until we remove {@link JacksonEvent} from data-prepper-api, + * we will need this class to give us access to the package-protected + * {@link JacksonEventKey}. + */ +public class InternalOnlyEventKeyBridge { + public static EventKey createEventKey(final String key, final EventKeyFactory.EventAction... forAction) { + return new JacksonEventKey(key, forAction); + } +} diff --git a/data-prepper-event/src/test/java/org/opensearch/dataprepper/core/event/DefaultEventKeyFactoryTest.java b/data-prepper-event/src/test/java/org/opensearch/dataprepper/core/event/DefaultEventKeyFactoryTest.java new file mode 100644 index 0000000000..8d034fcc83 --- /dev/null +++ b/data-prepper-event/src/test/java/org/opensearch/dataprepper/core/event/DefaultEventKeyFactoryTest.java @@ -0,0 +1,52 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.core.event; + +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyFactory; + +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +class DefaultEventKeyFactoryTest { + + private DefaultEventKeyFactory createObjectUnderTest() { + return new DefaultEventKeyFactory(); + } + + @Test + void createEventKey_returns_correct_EventKey() { + final String keyPath = UUID.randomUUID().toString(); + final EventKey eventKey = createObjectUnderTest().createEventKey(keyPath); + + assertThat(eventKey, notNullValue()); + assertThat(eventKey.getKey(), equalTo(keyPath)); + } + + @Test + void createEventKey_with_EventAction_returns_correct_EventKey() { + final String keyPath = UUID.randomUUID().toString(); + final EventKey eventKey = createObjectUnderTest().createEventKey(keyPath, EventKeyFactory.EventAction.GET); + + assertThat(eventKey, notNullValue()); + assertThat(eventKey.getKey(), equalTo(keyPath)); + } + + @Test + void createEventKey_returns_JacksonEventKey() { + final String keyPath = UUID.randomUUID().toString(); + final EventKey eventKey = createObjectUnderTest().createEventKey(keyPath); + + assertThat(eventKey, notNullValue()); + assertThat(eventKey.getClass().getSimpleName(), equalTo("JacksonEventKey")); + + assertThat(eventKey.getKey(), equalTo(keyPath)); + } +} \ No newline at end of file diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ApplicationContextToTypedSuppliers.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ApplicationContextToTypedSuppliers.java index f5ceebbde6..f9e1abddb7 100644 --- a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ApplicationContextToTypedSuppliers.java +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ApplicationContextToTypedSuppliers.java @@ -8,6 +8,7 @@ import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; import org.opensearch.dataprepper.model.breaker.CircuitBreaker; import org.opensearch.dataprepper.model.event.EventFactory; +import org.opensearch.dataprepper.model.event.EventKeyFactory; import org.springframework.beans.factory.annotation.Autowired; import javax.inject.Inject; @@ -31,6 +32,7 @@ class ApplicationContextToTypedSuppliers { @Inject ApplicationContextToTypedSuppliers( final EventFactory eventFactory, + final EventKeyFactory eventKeyFactory, final AcknowledgementSetManager acknowledgementSetManager, @Autowired(required = false) final CircuitBreaker circuitBreaker ) { @@ -39,6 +41,7 @@ class ApplicationContextToTypedSuppliers { typedSuppliers = Map.of( EventFactory.class, () -> eventFactory, + EventKeyFactory.class, () -> eventKeyFactory, AcknowledgementSetManager.class, () -> acknowledgementSetManager, CircuitBreaker.class, () -> circuitBreaker ); diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ApplicationContextToTypedSuppliersTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ApplicationContextToTypedSuppliersTest.java index 0cd008559a..a12540a46a 100644 --- a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ApplicationContextToTypedSuppliersTest.java +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ApplicationContextToTypedSuppliersTest.java @@ -12,6 +12,7 @@ import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; import org.opensearch.dataprepper.model.breaker.CircuitBreaker; import org.opensearch.dataprepper.model.event.EventFactory; +import org.opensearch.dataprepper.model.event.EventKeyFactory; import java.util.Map; import java.util.function.Supplier; @@ -28,6 +29,9 @@ class ApplicationContextToTypedSuppliersTest { @Mock private EventFactory eventFactory; + @Mock + private EventKeyFactory eventKeyFactory; + @Mock private AcknowledgementSetManager acknowledgementSetManager; @@ -37,6 +41,7 @@ class ApplicationContextToTypedSuppliersTest { private ApplicationContextToTypedSuppliers createObjectUnderTest() { return new ApplicationContextToTypedSuppliers( eventFactory, + eventKeyFactory, acknowledgementSetManager, circuitBreaker ); @@ -58,12 +63,16 @@ void constructor_throws_with_null_AcknowledgementSetManager() { void getArgumentsSuppliers_returns_map_with_expected_classes() { final Map, Supplier> argumentsSuppliers = createObjectUnderTest().getArgumentsSuppliers(); - assertThat(argumentsSuppliers.size(), equalTo(3)); + assertThat(argumentsSuppliers.size(), equalTo(4)); assertThat(argumentsSuppliers, hasKey(EventFactory.class)); assertThat(argumentsSuppliers.get(EventFactory.class), notNullValue()); assertThat(argumentsSuppliers.get(EventFactory.class).get(), equalTo(eventFactory)); + assertThat(argumentsSuppliers, hasKey(EventKeyFactory.class)); + assertThat(argumentsSuppliers.get(EventKeyFactory.class), notNullValue()); + assertThat(argumentsSuppliers.get(EventKeyFactory.class).get(), equalTo(eventKeyFactory)); + assertThat(argumentsSuppliers, hasKey(AcknowledgementSetManager.class)); assertThat(argumentsSuppliers.get(AcknowledgementSetManager.class), notNullValue()); assertThat(argumentsSuppliers.get(AcknowledgementSetManager.class).get(), equalTo(acknowledgementSetManager)); @@ -79,12 +88,16 @@ void getArgumentsSuppliers_returns_map_with_null_optional_CircuitBreaker() { final Map, Supplier> argumentsSuppliers = createObjectUnderTest().getArgumentsSuppliers(); - assertThat(argumentsSuppliers.size(), equalTo(3)); + assertThat(argumentsSuppliers.size(), equalTo(4)); assertThat(argumentsSuppliers, hasKey(EventFactory.class)); assertThat(argumentsSuppliers.get(EventFactory.class), notNullValue()); assertThat(argumentsSuppliers.get(EventFactory.class).get(), equalTo(eventFactory)); + assertThat(argumentsSuppliers, hasKey(EventKeyFactory.class)); + assertThat(argumentsSuppliers.get(EventKeyFactory.class), notNullValue()); + assertThat(argumentsSuppliers.get(EventKeyFactory.class).get(), equalTo(eventKeyFactory)); + assertThat(argumentsSuppliers, hasKey(AcknowledgementSetManager.class)); assertThat(argumentsSuppliers.get(AcknowledgementSetManager.class), notNullValue()); assertThat(argumentsSuppliers.get(AcknowledgementSetManager.class).get(), equalTo(acknowledgementSetManager)); diff --git a/data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventContext.java b/data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventContext.java new file mode 100644 index 0000000000..6c5b001129 --- /dev/null +++ b/data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventContext.java @@ -0,0 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.event; + +import org.opensearch.dataprepper.core.event.EventFactoryApplicationContextMarker; +import org.springframework.context.annotation.AnnotationConfigApplicationContext; + +class TestEventContext { + private static AnnotationConfigApplicationContext APPLICATION_CONTEXT; + + private TestEventContext() {} + + static T getFromContext(final Class targetClass) { + if(APPLICATION_CONTEXT == null) { + APPLICATION_CONTEXT = new AnnotationConfigApplicationContext(); + APPLICATION_CONTEXT.scan(EventFactoryApplicationContextMarker.class.getPackageName()); + APPLICATION_CONTEXT.refresh(); + } + return APPLICATION_CONTEXT.getBean(targetClass); + } +} diff --git a/data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventFactory.java b/data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventFactory.java index 932c9ca66a..08a2cd2f29 100644 --- a/data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventFactory.java +++ b/data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventFactory.java @@ -5,18 +5,15 @@ package org.opensearch.dataprepper.event; -import org.opensearch.dataprepper.core.event.EventFactoryApplicationContextMarker; import org.opensearch.dataprepper.model.event.BaseEventBuilder; import org.opensearch.dataprepper.model.event.Event; import org.opensearch.dataprepper.model.event.EventFactory; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; /** * An implementation of {@link EventFactory} that is useful for integration and unit tests * in other projects. */ public class TestEventFactory implements EventFactory { - private static AnnotationConfigApplicationContext APPLICATION_CONTEXT; private static EventFactory DEFAULT_EVENT_FACTORY; private final EventFactory innerEventFactory; @@ -25,11 +22,8 @@ public class TestEventFactory implements EventFactory { } public static EventFactory getTestEventFactory() { - if(APPLICATION_CONTEXT == null) { - APPLICATION_CONTEXT = new AnnotationConfigApplicationContext(); - APPLICATION_CONTEXT.scan(EventFactoryApplicationContextMarker.class.getPackageName()); - APPLICATION_CONTEXT.refresh(); - DEFAULT_EVENT_FACTORY = APPLICATION_CONTEXT.getBean(EventFactory.class); + if(DEFAULT_EVENT_FACTORY == null) { + DEFAULT_EVENT_FACTORY = TestEventContext.getFromContext(EventFactory.class); } return new TestEventFactory(DEFAULT_EVENT_FACTORY); } diff --git a/data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventKeyFactory.java b/data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventKeyFactory.java new file mode 100644 index 0000000000..0cec742924 --- /dev/null +++ b/data-prepper-test-event/src/main/java/org/opensearch/dataprepper/event/TestEventKeyFactory.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.event; + +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyFactory; + +public class TestEventKeyFactory implements EventKeyFactory { + private static EventKeyFactory DEFAULT_EVENT_KEY_FACTORY; + private final EventKeyFactory innerEventKeyFactory; + + TestEventKeyFactory(final EventKeyFactory innerEventKeyFactory) { + this.innerEventKeyFactory = innerEventKeyFactory; + } + + public static EventKeyFactory getTestEventFactory() { + if(DEFAULT_EVENT_KEY_FACTORY == null) { + DEFAULT_EVENT_KEY_FACTORY = TestEventContext.getFromContext(EventKeyFactory.class); + } + return new TestEventKeyFactory(DEFAULT_EVENT_KEY_FACTORY); + } + + @Override + public EventKey createEventKey(final String key, final EventAction... forActions) { + return innerEventKeyFactory.createEventKey(key, forActions); + } +} diff --git a/data-prepper-test-event/src/test/java/org/opensearch/dataprepper/event/TestEventKeyFactoryTest.java b/data-prepper-test-event/src/test/java/org/opensearch/dataprepper/event/TestEventKeyFactoryTest.java new file mode 100644 index 0000000000..65b17819b8 --- /dev/null +++ b/data-prepper-test-event/src/test/java/org/opensearch/dataprepper/event/TestEventKeyFactoryTest.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.event; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.event.EventKey; +import org.opensearch.dataprepper.model.event.EventKeyFactory; + +import java.util.UUID; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class TestEventKeyFactoryTest { + + @Mock + private EventKeyFactory innerEventKeyFactory; + + @Mock + private EventKey eventKey; + + private TestEventKeyFactory createObjectUnderTest() { + return new TestEventKeyFactory(innerEventKeyFactory); + } + + @Test + void createEventKey_returns_from_inner_EventKeyFactory() { + final String keyPath = UUID.randomUUID().toString(); + when(innerEventKeyFactory.createEventKey(keyPath, EventKeyFactory.EventAction.ALL)) + .thenReturn(eventKey); + + assertThat(createObjectUnderTest().createEventKey(keyPath), + equalTo(eventKey)); + } + + @ParameterizedTest + @EnumSource(EventKeyFactory.EventAction.class) + void createEventKey_with_Actions_returns_from_inner_EventKeyFactory(final EventKeyFactory.EventAction eventAction) { + final String keyPath = UUID.randomUUID().toString(); + when(innerEventKeyFactory.createEventKey(keyPath, eventAction)) + .thenReturn(eventKey); + + assertThat(createObjectUnderTest().createEventKey(keyPath, eventAction), + equalTo(eventKey)); + } +} \ No newline at end of file