diff --git a/assets/docs/configuration/overview-full-example.hcl b/assets/docs/configuration/overview-full-example.hcl index a8f28749..a2e13703 100644 --- a/assets/docs/configuration/overview-full-example.hcl +++ b/assets/docs/configuration/overview-full-example.hcl @@ -90,16 +90,20 @@ stats_receiver { buffer_sec = 20 } -// log level configuration (default: "info") +# log level configuration (default: "info") log_level = "info" -// Specifies how failed writes to the target should be retried, depending on an error type +# Specifies how failed writes to the target should be retried, depending on an error type retry { transient { + # Initial delay (before first retry) for transient errors delay_ms = 1000 + + # Maximum number of retries for transient errors max_attempts = 5 } setup { + # Initial delay (before first retry) for setup errors delay_ms = 20000 } } diff --git a/assets/docs/configuration/retry-example.hcl b/assets/docs/configuration/retry-example.hcl index 2f0d5299..17bcc981 100644 --- a/assets/docs/configuration/retry-example.hcl +++ b/assets/docs/configuration/retry-example.hcl @@ -1,9 +1,13 @@ retry { transient { + # Initial delay (before first retry) for transient errors delay_ms = 5000 + + # Maximum number of retries for transient errors max_attempts = 10 } setup { + # Initial delay (before first retry) for setup errors delay_ms = 30000 } } diff --git a/assets/docs/configuration/targets/http-full-example.hcl b/assets/docs/configuration/targets/http-full-example.hcl index 9a8f8a64..05ccb254 100644 --- a/assets/docs/configuration/targets/http-full-example.hcl +++ b/assets/docs/configuration/targets/http-full-example.hcl @@ -66,7 +66,8 @@ target { # Optional path to the file containing template which is used to build HTTP request based on a batch of input data template_file = "myTemplate.file" - # 2 invalid + 1 setup error rules + # Optional HTTP response rules which are used to match HTTP response code/body and categorize it as either invalid data or target setup error. + # For example, we can have 2 invalid + 1 setup error rules: response_rules { # This one is a match when... invalid { diff --git a/assets/docs/configuration/targets/http-template-full-example.file b/assets/docs/configuration/targets/http-template-full-example.file new file mode 100644 index 00000000..52222c24 --- /dev/null +++ b/assets/docs/configuration/targets/http-template-full-example.file @@ -0,0 +1,6 @@ +{ + "secret_key": "{{ env "SECRET_KEY" }}", + "data": [ + {{ range $i, $data := . }} {{if $i}},{{end}} {{ prettyPrint .foo }} {{ end }} + ] +} diff --git a/assets/docs/configuration/targets/http-template-unwrap-example.file b/assets/docs/configuration/targets/http-template-unwrap-example.file new file mode 100644 index 00000000..0aa31145 --- /dev/null +++ b/assets/docs/configuration/targets/http-template-unwrap-example.file @@ -0,0 +1 @@ +{{ prettyPrint (index . 0)}} diff --git a/assets/docs/configuration/transformations/builtin/jq-full-example.hcl b/assets/docs/configuration/transformations/builtin/jq-full-example.hcl index 411da2c1..f9c6dd0f 100644 --- a/assets/docs/configuration/transformations/builtin/jq-full-example.hcl +++ b/assets/docs/configuration/transformations/builtin/jq-full-example.hcl @@ -1,5 +1,6 @@ transform { use "jq" { + # Full JQ command which will be used to transform input data. jq_command = < 18.194.133.57 68220ade-307b-4898-8e25-c4c8ac92f1d7 transaction item58 35.87 1 python-requests/2.21.0 2019-05-10 14:40:30.000 {"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1","data":[{"schema":"iglu:nl.basjes/yauaa_context/jsonschema/1-0-0","data":{"deviceBrand":"Unknown","deviceName":"Unknown","operatingSystemName":"Unknown","agentVersionMajor":"2","layoutEngineVersionMajor":"??","deviceClass":"Unknown","agentNameVersionMajor":"python-requests 2","operatingSystemClass":"Unknown","layoutEngineName":"Unknown","agentName":"python-requests","agentVersion":"2.21.0","layoutEngineClass":"Unknown","agentNameVersion":"python-requests 2.21.0","operatingSystemVersion":"??","agentClass":"Special","layoutEngineVersion":"??"}}]} 2019-05-10 14:40:31.105 com.snowplowanalytics.snowplow transaction_item jsonschema 1-0-0 `) diff --git a/pkg/transform/transformconfig/transform_config.go b/pkg/transform/transformconfig/transform_config.go index 580fedb3..ae6b76fc 100644 --- a/pkg/transform/transformconfig/transform_config.go +++ b/pkg/transform/transformconfig/transform_config.go @@ -25,6 +25,7 @@ var SupportedTransformations = []config.ConfigurationPair{ filter.AtomicFilterConfigPair, filter.UnstructFilterConfigPair, filter.ContextFilterConfigPair, + filter.JQFilterConfigPair, transform.SetPkConfigPair, transform.EnrichedToJSONConfigPair, transform.Base64DecodeConfigPair,