Skip to content

Commit

Permalink
Documentation improvements for a number of Data Prepper processors: R…
Browse files Browse the repository at this point in the history 
…e-ordering properties; adding missing documentation; missed enums. (opensearch-project#5026)

Signed-off-by: David Venable <dlv@amazon.com>
  • Loading branch information
@dlvenable
dlvenable authored Oct 8, 2024
1 parent 27337ea commit 0e3b852
Show file tree
Hide file tree
Showing 24 changed files with 247 additions and 169 deletions.
10 changes: 5 additions & 5 deletions .../src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,6 @@ public static boolean isValidPattern(final String pattern) {
"This option cannot be defined at the same time as <code>match</code>. Default is <code>false</code>.")
private Boolean fromTimeReceived = DEFAULT_FROM_TIME_RECEIVED;

@JsonProperty("to_origination_metadata")
@JsonPropertyDescription("When <code>true</code>, the matched time is also added to the event's metadata as an instance of " +
"<code>Instant</code>. Default is <code>false</code>.")
private Boolean toOriginationMetadata = DEFAULT_TO_ORIGINATION_METADATA;

@JsonProperty("match")
@JsonPropertyDescription("The date match configuration. " +
"This option cannot be defined at the same time as <code>from_time_received</code>. " +
Expand All @@ -128,6 +123,11 @@ public static boolean isValidPattern(final String pattern) {
"Default is <code>yyyy-MM-dd'T'HH:mm:ss.SSSXXX</code>.")
private String outputFormat = DEFAULT_OUTPUT_FORMAT;

@JsonProperty("to_origination_metadata")
@JsonPropertyDescription("When <code>true</code>, the matched time is also added to the event's metadata as an instance of " +
"<code>Instant</code>. Default is <code>false</code>.")
private Boolean toOriginationMetadata = DEFAULT_TO_ORIGINATION_METADATA;

@JsonProperty("source_timezone")
@JsonPropertyDescription("The time zone used to parse dates, including when the zone or offset cannot be extracted " +
"from the value. If the zone or offset are part of the value, then the time zone is ignored. " +
Expand Down
8 changes: 4 additions & 4 deletions ...va/org/opensearch/dataprepper/plugins/processor/decompress/DecompressProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,14 @@ public class DecompressProcessorConfig {
@NotNull
private DecompressionType decompressionType;

@JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, such as <code>'/is_compressed == true'</code>, that determines when the decompress processor will run on certain events.")
@JsonProperty("decompress_when")
private String decompressWhen;

@JsonPropertyDescription("A list of strings with which to tag events when the processor fails to decompress the keys inside an event. Defaults to <code>_decompression_failure</code>.")
@JsonProperty("tags_on_failure")
private List<String> tagsOnFailure = List.of("_decompression_failure");

@JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a>, such as <code>'/is_compressed == true'</code>, that determines when the decompress processor will run on certain events.")
@JsonProperty("decompress_when")
private String decompressWhen;

@JsonIgnore
private final EncodingType encodingType = EncodingType.BASE64;

Expand Down
8 changes: 4 additions & 4 deletions ...ain/java/org/opensearch/dataprepper/plugins/processor/flatten/FlattenProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,15 @@ public class FlattenProcessorConfig {
"By default no keys are excluded.")
private List<String> excludeKeys = DEFAULT_EXCLUDE_KEYS;

@JsonProperty("tags_on_failure")
@JsonPropertyDescription("A list of tags to add to the event metadata when the event fails to process.")
private List<String> tagsOnFailure;

@JsonProperty("flatten_when")
@JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> such as <code>/some_key == \"test\"</code>. " +
"If specified, the <code>flatten</code> processor will only run on events when the expression evaluates to true. ")
private String flattenWhen;

@JsonProperty("tags_on_failure")
@JsonPropertyDescription("A list of tags to add to the event metadata when the event fails to process.")
private List<String> tagsOnFailure;

public String getSource() {
return source;
}
Expand Down
4 changes: 4 additions & 0 deletions ...ocessor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/EntryConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@

package org.opensearch.dataprepper.plugins.geoip.processor;

import com.fasterxml.jackson.annotation.JsonClassDescription;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonPropertyDescription;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import jakarta.validation.constraints.AssertTrue;
import jakarta.validation.constraints.NotEmpty;
import org.opensearch.dataprepper.plugins.geoip.GeoIPField;
Expand All @@ -15,6 +17,8 @@
import java.util.EnumSet;
import java.util.List;

@JsonPropertyOrder
@JsonClassDescription("Defines a single entry for geolocation.")
public class EntryConfig {
static final String DEFAULT_TARGET = "geo";

Expand Down
2 changes: 1 addition & 1 deletion ...rc/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class GeoIPProcessorConfig {
@NotNull
@Size(min = 1)
@JsonProperty("entries")
@JsonPropertyDescription("The list of entries marked for enrichment.")
@JsonPropertyDescription("The list of entries for enrichment. Each entry provides a source field with an IP address along with a target for the enriched geolocation data.")
private List<EntryConfig> entries;

@JsonProperty("tags_on_engine_failure")
Expand Down
42 changes: 21 additions & 21 deletions .../src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,17 @@ public class GrokProcessorConfig {
static final int DEFAULT_TIMEOUT_MILLIS = 30000;
static final String DEFAULT_TARGET_KEY = null;

@JsonProperty(MATCH)
@JsonPropertyDescription("Specifies which keys should match specific patterns. " +
"Each key is a source field. The value is a list of possible grok patterns to match on. " +
"The <code>grok</code> processor will extract values from the first match for each field. " +
"Default is an empty response body.")
private Map<String, List<String>> match = Collections.emptyMap();

@JsonProperty(TARGET_KEY)
@JsonPropertyDescription("Specifies a parent-level key used to store all captures. Default value is <code>null</code> which will write captures into the root of the event.")
private String targetKey = DEFAULT_TARGET_KEY;

@JsonProperty(BREAK_ON_MATCH)
@JsonPropertyDescription("Specifies whether to match all patterns (<code>false</code>) or stop once the first successful " +
"match is found (<code>true</code>). Default is <code>true</code>.")
Expand All @@ -55,13 +66,6 @@ public class GrokProcessorConfig {
@JsonPropertyDescription("Enables the preservation of <code>null</code> captures from the processed output. Default is <code>false</code>.")
private boolean keepEmptyCaptures = DEFAULT_KEEP_EMPTY_CAPTURES;

@JsonProperty(MATCH)
@JsonPropertyDescription("Specifies which keys should match specific patterns. " +
"Each key is a source field. The value is a list of possible grok patterns to match on. " +
"The <code>grok</code> processor will extract values from the first match for each field. " +
"Default is an empty response body.")
private Map<String, List<String>> match = Collections.emptyMap();

@JsonProperty(NAMED_CAPTURES_ONLY)
@JsonPropertyDescription("Specifies whether to keep only named captures. Default is <code>true</code>.")
private boolean namedCapturesOnly = DEFAULT_NAMED_CAPTURES_ONLY;
Expand All @@ -71,6 +75,11 @@ public class GrokProcessorConfig {
"Default is an empty list.")
private List<String> keysToOverwrite = Collections.emptyList();

@JsonProperty(PATTERN_DEFINITIONS)
@JsonPropertyDescription("Allows for a custom pattern that can be used inline inside the response body. " +
"Default is an empty response body.")
private Map<String, String> patternDefinitions = Collections.emptyMap();

@JsonProperty(PATTERNS_DIRECTORIES)
@JsonPropertyDescription("Specifies which directory paths contain the custom pattern files. Default is an empty list.")
private List<String> patternsDirectories = Collections.emptyList();
Expand All @@ -80,25 +89,11 @@ public class GrokProcessorConfig {
"<code>pattern_directories</code>. Default is <code>*</code>.")
private String patternsFilesGlob = DEFAULT_PATTERNS_FILES_GLOB;

@JsonProperty(PATTERN_DEFINITIONS)
@JsonPropertyDescription("Allows for a custom pattern that can be used inline inside the response body. " +
"Default is an empty response body.")
private Map<String, String> patternDefinitions = Collections.emptyMap();

@JsonProperty(TIMEOUT_MILLIS)
@JsonPropertyDescription("The maximum amount of time during which matching occurs. " +
"Setting to <code>0</code> prevents any matching from occurring. Default is <code>30000</code>.")
private int timeoutMillis = DEFAULT_TIMEOUT_MILLIS;

@JsonProperty(TARGET_KEY)
@JsonPropertyDescription("Specifies a parent-level key used to store all captures. Default value is <code>null</code> which will write captures into the root of the event.")
private String targetKey = DEFAULT_TARGET_KEY;

@JsonProperty(GROK_WHEN)
@JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> such as <code>'/test != false'</code>. " +
"If specified, the <code>grok</code> processor will only run on events when the expression evaluates to true. ")
private String grokWhen;

@JsonProperty(TAGS_ON_MATCH_FAILURE)
@JsonPropertyDescription("A <code>List</code> of <code>String</code>s that specifies the tags to be set in the event when grok fails to " +
"match or an unknown exception occurs while matching. This tag may be used in conditional expressions in " +
Expand All @@ -109,6 +104,11 @@ public class GrokProcessorConfig {
@JsonPropertyDescription("The tags to add to the event metadata if the grok match times out.")
private List<String> tagsOnTimeout = Collections.emptyList();

@JsonProperty(GROK_WHEN)
@JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> such as <code>'/test != false'</code>. " +
"If specified, the <code>grok</code> processor will only run on events when the expression evaluates to true. ")
private String grokWhen;

@JsonProperty(INCLUDE_PERFORMANCE_METADATA)
@JsonPropertyDescription("A boolean value to determine whether to include performance metadata into event metadata. " +
"If set to true, the events coming out of grok will have new fields such as <code>_total_grok_patterns_attempted</code> and <code>_total_grok_processing_time</code>." +
Expand Down
80 changes: 40 additions & 40 deletions ...n/java/org/opensearch/dataprepper/plugins/processor/keyvalue/KeyValueProcessorConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,31 +47,33 @@ public class KeyValueProcessorConfig {
"The default value is <code>parsed_message</code>.")
private String destination = DEFAULT_DESTINATION;

@JsonProperty("field_split_characters")
@JsonPropertyDescription("A string of characters specifying the delimiter that separates key-value pairs. " +
"Special regular expression characters such as <code>[</code> and <code>]</code> must be escaped with <code>\\\\</code>. " +
"This field cannot be defined along with <code>field_delimiter_regex</code>. " +
"The default value is <code>&amp;</code>.")
private String fieldSplitCharacters = DEFAULT_FIELD_SPLIT_CHARACTERS;

@JsonProperty("field_delimiter_regex")
@JsonPropertyDescription("A regular expression specifying the delimiter that separates key-value pairs. " +
"Special regular expression characters such as <code>[</code> and <code>]</code> must be escaped with <code>\\\\</code>. " +
"This field cannot be defined along with <code>field_split_characters</code>. " +
"If this option is not defined, the <code>key_value</code> processor will parse the source using <code>field_split_characters</code>.")
private String fieldDelimiterRegex;

@JsonProperty("field_split_characters")
@JsonPropertyDescription("A string of characters specifying the delimiter that separates key-value pairs. " +
@JsonProperty("value_split_characters")
@JsonPropertyDescription("A string of characters specifying the delimiter that separates keys from their values within a key-value pair. " +
"Special regular expression characters such as <code>[</code> and <code>]</code> must be escaped with <code>\\\\</code>. " +
"This field cannot be defined along with <code>field_delimiter_regex</code>. " +
"The default value is <code>&amp;</code>.")
private String fieldSplitCharacters = DEFAULT_FIELD_SPLIT_CHARACTERS;

@JsonProperty("include_keys")
@JsonPropertyDescription("An array specifying the keys that should be included in the destination field. " +
"By default, all keys will be added.")
@NotNull
private List<String> includeKeys = DEFAULT_INCLUDE_KEYS;
"This field cannot be defined along with <code>key_value_delimiter_regex</code>. " +
"The default value is <code>=</code>.")
private String valueSplitCharacters = DEFAULT_VALUE_SPLIT_CHARACTERS;

@JsonProperty("exclude_keys")
@JsonPropertyDescription("An array specifying the parsed keys that should be excluded from the destination field. " +
"By default, no keys will be excluded.")
@NotNull
private List<String> excludeKeys = DEFAULT_EXCLUDE_KEYS;
@JsonProperty("key_value_delimiter_regex")
@JsonPropertyDescription("A regular expression specifying the delimiter that separates keys from their values within a key-value pair. " +
"Special regular expression characters such as <code>[</code> and <code>]</code> must be escaped with <code>\\\\</code>. " +
"This field cannot be defined along with <code>value_split_characters</code>. " +
"If this option is not defined, the <code>key_value</code> processor will parse the source using <code>value_split_characters</code>.")
private String keyValueDelimiterRegex;

@JsonProperty("default_values")
@JsonPropertyDescription("A map specifying the default keys and their values that should be added " +
Expand All @@ -81,26 +83,24 @@ public class KeyValueProcessorConfig {
@NotNull
private Map<String, Object> defaultValues = DEFAULT_DEFAULT_VALUES;

@JsonProperty("key_value_delimiter_regex")
@JsonPropertyDescription("A regular expression specifying the delimiter that separates keys from their values within a key-value pair. " +
"Special regular expression characters such as <code>[</code> and <code>]</code> must be escaped with <code>\\\\</code>. " +
"This field cannot be defined along with <code>value_split_characters</code>. " +
"If this option is not defined, the <code>key_value</code> processor will parse the source using <code>value_split_characters</code>.")
private String keyValueDelimiterRegex;

@JsonProperty("value_split_characters")
@JsonPropertyDescription("A string of characters specifying the delimiter that separates keys from their values within a key-value pair. " +
"Special regular expression characters such as <code>[</code> and <code>]</code> must be escaped with <code>\\\\</code>. " +
"This field cannot be defined along with <code>key_value_delimiter_regex</code>. " +
"The default value is <code>=</code>.")
private String valueSplitCharacters = DEFAULT_VALUE_SPLIT_CHARACTERS;

@JsonProperty("non_match_value")
@JsonPropertyDescription("Configures a value to use when the processor cannot split a key-value pair. " +
"The value specified in this configuration is the value used in <code>destination</code> map. " +
"The default behavior is to drop the key-value pair.")
private Object nonMatchValue = DEFAULT_NON_MATCH_VALUE;

@JsonProperty("include_keys")
@JsonPropertyDescription("An array specifying the keys that should be included in the destination field. " +
"By default, all keys will be added.")
@NotNull
private List<String> includeKeys = DEFAULT_INCLUDE_KEYS;

@JsonProperty("exclude_keys")
@JsonPropertyDescription("An array specifying the parsed keys that should be excluded from the destination field. " +
"By default, no keys will be excluded.")
@NotNull
private List<String> excludeKeys = DEFAULT_EXCLUDE_KEYS;

@JsonPropertyDescription("A prefix to append before all keys. By default no prefix is added.")
@NotNull
private String prefix = DEFAULT_PREFIX;
Expand Down Expand Up @@ -168,11 +168,7 @@ public class KeyValueProcessorConfig {
"<code>whitespace</code> will always be <code>\"strict\"</code>.")
@NotNull
private boolean recursive = DEFAULT_RECURSIVE;

@JsonProperty("tags_on_failure")
@JsonPropertyDescription("The tags to add to the event metadata if the <code>key_value</code> processor fails to parse the source string.")
private List<String> tagsOnFailure;


@JsonProperty("overwrite_if_destination_exists")
@JsonPropertyDescription("Specifies whether to overwrite existing fields if there are key conflicts " +
"when writing parsed fields to the event. Default is <code>true</code>.")
Expand All @@ -184,11 +180,6 @@ public class KeyValueProcessorConfig {
"then <code>{\"key1=value1&amp;key2\"}</code> parses to <code>{\"key1\": \"value1\"}</code>.")
private boolean dropKeysWithNoValue = false;

@JsonProperty("key_value_when")
@JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> such as <code>/some_key == \"test\"</code>. " +
"If specified, the <code>key_value</code> processor will only run on events when the expression evaluates to true. ")
private String keyValueWhen;

@JsonProperty("strict_grouping")
@JsonPropertyDescription("When enabled, groups with unmatched end characters yield errors. " +
"The event is ignored after the errors are logged. " +
Expand All @@ -204,6 +195,15 @@ public class KeyValueProcessorConfig {
@Size(min = 0, max = 1, message = "string_literal_character may only have character")
private String stringLiteralCharacter = null;

@JsonProperty("tags_on_failure")
@JsonPropertyDescription("The tags to add to the event metadata if the <code>key_value</code> processor fails to parse the source string.")
private List<String> tagsOnFailure;

@JsonProperty("key_value_when")
@JsonPropertyDescription("A <a href=\"https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/\">conditional expression</a> such as <code>/some_key == \"test\"</code>. " +
"If specified, the <code>key_value</code> processor will only run on events when the expression evaluates to true. ")
private String keyValueWhen;

@AssertTrue(message = "Invalid Configuration. value_grouping option and field_delimiter_regex are mutually exclusive")
boolean isValidValueGroupingAndFieldDelimiterRegex() {
return (!valueGrouping || fieldDelimiterRegex == null);
Expand Down
Loading

0 comments on commit 0e3b852

Please sign in to comment.