Skip to content

Commit

Permalink
Normalize the include/exclude keys in the JacksonEvent implementation…
Browse files Browse the repository at this point in the history 
… in order to fix a problem where the ndjson codec was not correctly including/excluding keys. (opensearch-project#3209)

Signed-off-by: David Venable <dlv@amazon.com>
  • Loading branch information
@dlvenable
dlvenable authored Aug 22, 2023
1 parent d2bdb38 commit fecb842
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 15 deletions.
3 changes: 2 additions & 1 deletion data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -607,11 +607,12 @@ String searchAndFilter(JsonNode node, String path, final List<String> filterKeys
List<String> valueList = new ArrayList<>();

node.properties().forEach(entry -> {
String keyPath = path + SEPARATOR + entry.getKey();
String keyPath = trimKey(path + SEPARATOR + entry.getKey());
// Track whether the key is found in the filter list.
// Different behaviours between include and exclude action.
boolean found = false;
for (String key : filterKeys) {
key = trimKey(key);
if (keyPath.equals(key)) {
found = true;
// To keep the order.
Expand Down
28 changes: 14 additions & 14 deletions data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -691,37 +691,37 @@ void testJsonStringBuilderWithIncludeKeys() {
.build();

// Include Keys must start with / and also ordered, This is pre-processed in SinkModel
List<String> includeKeys1 = Arrays.asList("/foo", "/info");
List<String> includeKeys1 = Arrays.asList("foo", "info");
final String expectedJsonString1 = "{\"foo\":\"bar\",\"info\":{\"name\":\"hello\",\"foo\":\"bar\"}}";
assertThat(event.jsonBuilder().rootKey(null).includeKeys(includeKeys1).toJsonString(), equalTo(expectedJsonString1));

// Test child node
List<String> includeKeys2 = Arrays.asList("/foo", "/info/name");
List<String> includeKeys2 = Arrays.asList("foo", "info/name");
final String expectedJsonString2 = "{\"foo\":\"bar\",\"info\":{\"name\":\"hello\"}}";
assertThat(event.jsonBuilder().includeKeys(includeKeys2).toJsonString(), equalTo(expectedJsonString2));

// Test array node.
List<String> includeKeys3 = Arrays.asList("/foo", "/tags/key");
List<String> includeKeys3 = Arrays.asList("foo", "tags/key");
final String expectedJsonString3 = "{\"foo\":\"bar\",\"tags\":[{\"key\":\"a\"},{\"key\":\"c\"}]}";
assertThat(event.jsonBuilder().includeKeys(includeKeys3).toJsonString(), equalTo(expectedJsonString3));

// Test some keys not found
List<String> includeKeys4 = Arrays.asList("/foo", "/info/age");
List<String> includeKeys4 = Arrays.asList("foo", "info/age");
final String expectedJsonString4 = "{\"foo\":\"bar\",\"info\":{}}";
assertThat(event.jsonBuilder().includeKeys(includeKeys4).toJsonString(), equalTo(expectedJsonString4));

// Test all keys not found
List<String> includeKeys5 = List.of("/hello");
List<String> includeKeys5 = List.of("/ello");
final String expectedJsonString5 = "{}";
assertThat(event.jsonBuilder().includeKeys(includeKeys5).toJsonString(), equalTo(expectedJsonString5));

// Test working with root node
List<String> includeKeys6 = List.of("/name");
List<String> includeKeys6 = List.of("name");
final String expectedJsonString6 = "{\"name\":\"hello\"}";
assertThat(event.jsonBuilder().rootKey("info").includeKeys(includeKeys6).toJsonString(), equalTo(expectedJsonString6));

// Test working with unknown root node
List<String> includeKeys7 = List.of("/name");
List<String> includeKeys7 = List.of("name");
final String expectedJsonString7 = "{}";
assertThat(event.jsonBuilder().rootKey("hello").includeKeys(includeKeys7).toJsonString(), equalTo(expectedJsonString7));

Expand All @@ -738,37 +738,37 @@ void testJsonStringBuilderWithExcludeKeys() {
.build();

// Include Keys must start with / and also ordered, This is pre-processed in SinkModel
List<String> excludeKeys1 = Arrays.asList("/foo", "/info");
List<String> excludeKeys1 = Arrays.asList("foo", "info");
final String expectedJsonString1 = "{\"id\":1,\"tags\":[{\"key\":\"a\",\"value\":\"b\"},{\"key\":\"c\",\"value\":\"d\"}]}";
assertThat(event.jsonBuilder().rootKey(null).excludeKeys(excludeKeys1).toJsonString(), equalTo(expectedJsonString1));

// Test child node
List<String> excludeKeys2 = Arrays.asList("/foo", "/info/name");
List<String> excludeKeys2 = Arrays.asList("foo", "info/name");
final String expectedJsonString2 = "{\"id\":1,\"info\":{\"foo\":\"bar\"},\"tags\":[{\"key\":\"a\",\"value\":\"b\"},{\"key\":\"c\",\"value\":\"d\"}]}";
assertThat(event.jsonBuilder().excludeKeys(excludeKeys2).toJsonString(), equalTo(expectedJsonString2));

// Test array node.
List<String> excludeKeys3 = Arrays.asList("/foo", "/tags/key");
List<String> excludeKeys3 = Arrays.asList("foo", "tags/key");
final String expectedJsonString3 = "{\"id\":1,\"info\":{\"name\":\"hello\",\"foo\":\"bar\"},\"tags\":[{\"value\":\"b\"},{\"value\":\"d\"}]}";
assertThat(event.jsonBuilder().excludeKeys(excludeKeys3).toJsonString(), equalTo(expectedJsonString3));

// Test some keys not found
List<String> excludeKeys4 = Arrays.asList("/foo", "/info/age");
List<String> excludeKeys4 = Arrays.asList("foo", "info/age");
final String expectedJsonString4 = "{\"id\":1,\"info\":{\"name\":\"hello\",\"foo\":\"bar\"},\"tags\":[{\"key\":\"a\",\"value\":\"b\"},{\"key\":\"c\",\"value\":\"d\"}]}";
assertThat(event.jsonBuilder().excludeKeys(excludeKeys4).toJsonString(), equalTo(expectedJsonString4));

// Test all keys not found
List<String> excludeKeys5 = List.of("/hello");
List<String> excludeKeys5 = List.of("hello");
final String expectedJsonString5 = "{\"id\":1,\"foo\":\"bar\",\"info\":{\"name\":\"hello\",\"foo\":\"bar\"},\"tags\":[{\"key\":\"a\",\"value\":\"b\"},{\"key\":\"c\",\"value\":\"d\"}]}";
assertThat(event.jsonBuilder().excludeKeys(excludeKeys5).toJsonString(), equalTo(expectedJsonString5));

// Test working with root node
List<String> excludeKeys6 = List.of("/name");
List<String> excludeKeys6 = List.of("name");
final String expectedJsonString6 = "{\"foo\":\"bar\"}";
assertThat(event.jsonBuilder().rootKey("info").excludeKeys(excludeKeys6).toJsonString(), equalTo(expectedJsonString6));

// Test working with unknown root node
List<String> includeKeys7 = List.of("/name");
List<String> includeKeys7 = List.of("name");
final String expectedJsonString7 = "{}";
assertThat(event.jsonBuilder().rootKey("hello").includeKeys(includeKeys7).toJsonString(), equalTo(expectedJsonString7));

Expand Down

0 comments on commit fecb842

Please sign in to comment.