Skip to content

Commit

Permalink
Add expiry config + base64 validation to the GTM SS preview header tr…
Browse files Browse the repository at this point in the history
…ansformation
  • Loading branch information
pondzix committed Sep 17, 2024
1 parent 45c34bb commit 4fbefc7
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 18 deletions.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
transform {
use "spGtmssPreview" {
# Message expiry time in seconds (comparing current time to the message's collector timestamp). If message is expired, it's sent to failure target.
expiry_seconds = 600
}
}
2 changes: 1 addition & 1 deletion docs/configuration_transformations_docs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func TestBuiltinTransformationDocumentation(t *testing.T) {
}

func TestBuiltinSnowplowTransformationDocumentation(t *testing.T) {
transformationsToTest := []string{"spEnrichedFilter", "spEnrichedFilterContext", "spEnrichedFilterUnstructEvent", "spEnrichedSetPk", "spEnrichedToJson"}
transformationsToTest := []string{"spEnrichedFilter", "spEnrichedFilterContext", "spEnrichedFilterUnstructEvent", "spEnrichedSetPk", "spEnrichedToJson", "spGtmssPreview"}

for _, tfm := range transformationsToTest {

Expand Down
37 changes: 30 additions & 7 deletions pkg/transform/snowplow_gtmss_preview.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package transform

import (
"encoding/base64"
"errors"
"time"

"github.com/snowplow/snowbridge/config"
"github.com/snowplow/snowbridge/pkg/models"
Expand All @@ -11,14 +13,16 @@ import (

// GTMSSPreviewConfig is a configuration object for the spEnrichedToJson transformation
type GTMSSPreviewConfig struct {
Expiry int `hcl:"expiry_seconds,optional"`
}

// The gtmssPreviewAdapter implements the Pluggable interface
type gtmssPreviewAdapter func(i interface{}) (interface{}, error)

// ProvideDefault implements the ComponentConfigurable interface
func (f gtmssPreviewAdapter) ProvideDefault() (interface{}, error) {
return nil, nil
cfg := &GTMSSPreviewConfig{Expiry: 300} // seconds -> 5 minutes
return cfg, nil
}

// Create implements the ComponentCreator interface.
Expand All @@ -27,22 +31,24 @@ func (f gtmssPreviewAdapter) Create(i interface{}) (interface{}, error) {
}

// gtmssPreviewAdapterGenerator returns a gtmssPreviewAdapter
func gtmssPreviewAdapterGenerator(f func() (TransformationFunction, error)) gtmssPreviewAdapter {
func gtmssPreviewAdapterGenerator(f func(cfg *GTMSSPreviewConfig) (TransformationFunction, error)) gtmssPreviewAdapter {
return func(i interface{}) (interface{}, error) {
if i != nil {
cfg, ok := i.(*GTMSSPreviewConfig)
if !ok {
return nil, errors.New("unexpected configuration input for gtmssPreview transformation")
}

return f()
return f(cfg)
}
}

// gtmssPreviewConfigFunction returns a transformation function
func gtmssPreviewConfigFunction() (TransformationFunction, error) {
func gtmssPreviewConfigFunction(cfg *GTMSSPreviewConfig) (TransformationFunction, error) {
ctx := "contexts_com_google_tag-manager_server-side_preview_mode_1"
property := "x-gtm-server-preview"
header := "x-gtm-server-preview"
return gtmssPreviewTransformation(ctx, property, header), nil
expiry := time.Duration(cfg.Expiry) * time.Second
return gtmssPreviewTransformation(ctx, property, header, expiry), nil
}

// GTMSSPreviewConfigPair is the configuration pair for the gtmss preview transformation
Expand All @@ -52,14 +58,27 @@ var GTMSSPreviewConfigPair = config.ConfigurationPair{
}

// gtmssPreviewTransformation returns a transformation function
func gtmssPreviewTransformation(ctx, property, headerKey string) TransformationFunction {
func gtmssPreviewTransformation(ctx, property, headerKey string, expiry time.Duration) TransformationFunction {
return func(message *models.Message, interState interface{}) (*models.Message, *models.Message, *models.Message, interface{}) {
parsedEvent, err := IntermediateAsSpEnrichedParsed(interState, message)
if err != nil {
message.SetError(err)
return nil, nil, message, nil
}

tstamp, err := parsedEvent.GetValue("collector_tstamp")
if err != nil {
message.SetError(err)
return nil, nil, message, nil
}

if collectorTstamp, ok := tstamp.(time.Time); ok {
if time.Now().UTC().After(collectorTstamp.Add(expiry)) {
message.SetError(errors.New("Message has expired"))
return nil, nil, message, nil
}
}

headerVal, err := extractHeaderValue(parsedEvent, ctx, property)
if err != nil {
message.SetError(err)
Expand Down Expand Up @@ -96,6 +115,10 @@ func extractHeaderValue(parsedEvent analytics.ParsedEvent, ctx, prop string) (*s
return nil, errors.New("invalid header value")
}

_, err = base64.StdEncoding.DecodeString(headerVal)
if err != nil {
return nil, err
}
return &headerVal, nil
}

Expand Down
57 changes: 51 additions & 6 deletions pkg/transform/snowplow_gtmss_preview_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"reflect"
"strings"
"testing"
"time"

"github.com/davecgh/go-spew/spew"
"github.com/stretchr/testify/assert"
Expand All @@ -13,12 +14,15 @@ import (
"github.com/snowplow/snowplow-golang-analytics-sdk/analytics"
)

const fiveYears = time.Hour * 24 * 365 * 5

func TestGTMSSPreview(t *testing.T) {
testCases := []struct {
Scenario string
Ctx string
Property string
HeaderKey string
Expiry time.Duration
InputMsg *models.Message
InputInterState interface{}
Expected map[string]*models.Message
Expand All @@ -30,6 +34,7 @@ func TestGTMSSPreview(t *testing.T) {
Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1",
Property: "x-gtm-server-preview",
HeaderKey: "x-gtm-server-preview",
Expiry: fiveYears,
InputMsg: &models.Message{
Data: spTsvWithGtmss,
PartitionKey: "pk",
Expand All @@ -54,6 +59,7 @@ func TestGTMSSPreview(t *testing.T) {
Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1",
Property: "x-gtm-server-preview",
HeaderKey: "x-gtm-server-preview",
Expiry: fiveYears,
InputMsg: &models.Message{
Data: spTsvNoGtmss,
PartitionKey: "pk",
Expand All @@ -76,6 +82,7 @@ func TestGTMSSPreview(t *testing.T) {
Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1",
Property: "x-gtm-server-preview",
HeaderKey: "x-gtm-server-preview",
Expiry: fiveYears,
InputMsg: &models.Message{
Data: []byte(`asdf`),
PartitionKey: "pk",
Expand All @@ -98,6 +105,7 @@ func TestGTMSSPreview(t *testing.T) {
Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1",
Property: "x-gtm-server-preview",
HeaderKey: "x-gtm-server-preview",
Expiry: fiveYears,
InputMsg: &models.Message{
Data: spTsvWithGtmss,
PartitionKey: "pk",
Expand Down Expand Up @@ -126,6 +134,7 @@ func TestGTMSSPreview(t *testing.T) {
Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1",
Property: "x-gtm-server-preview",
HeaderKey: "x-gtm-server-preview",
Expiry: fiveYears,
InputMsg: &models.Message{
Data: spTsvWithGtmss,
PartitionKey: "pk",
Expand Down Expand Up @@ -155,6 +164,7 @@ func TestGTMSSPreview(t *testing.T) {
Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1",
Property: "x-gtm-server-preview",
HeaderKey: "x-gtm-server-preview",
Expiry: fiveYears,
InputMsg: &models.Message{
Data: spTsvWithGtmss,
PartitionKey: "pk",
Expand All @@ -180,6 +190,7 @@ func TestGTMSSPreview(t *testing.T) {
Ctx: "app_id",
Property: "x-gtm-server-preview",
HeaderKey: "x-gtm-server-preview",
Expiry: fiveYears,
InputMsg: &models.Message{
Data: spTsvWithGtmss,
PartitionKey: "pk",
Expand All @@ -200,11 +211,34 @@ func TestGTMSSPreview(t *testing.T) {
ExpInterState: spTsvWithGtmssParsed,
Error: nil,
},
{
Scenario: "expired_message",
Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1",
Property: "x-gtm-server-preview",
HeaderKey: "x-gtm-server-preview",
Expiry: 1 * time.Hour,
InputMsg: &models.Message{
Data: spTsvWithGtmss,
PartitionKey: "pk",
},
InputInterState: nil,
Expected: map[string]*models.Message{
"success": nil,
"filtered": nil,
"failed": {
Data: []byte(spTsvWithGtmss),
PartitionKey: "pk",
HTTPHeaders: nil,
},
},
ExpInterState: nil,
Error: errors.New("Message has expired"),
},
}

for _, tt := range testCases {
t.Run(tt.Scenario, func(t *testing.T) {
transFunction := gtmssPreviewTransformation(tt.Ctx, tt.Property, tt.HeaderKey)
transFunction := gtmssPreviewTransformation(tt.Ctx, tt.Property, tt.HeaderKey, tt.Expiry)
s, f, e, i := transFunction(tt.InputMsg, tt.InputInterState)

if !reflect.DeepEqual(i, tt.ExpInterState) {
Expand Down Expand Up @@ -281,13 +315,21 @@ func TestExtractHeaderValue(t *testing.T) {
Error: nil,
},
{
Scenario: "invalid_header_value",
Scenario: "invalid_header_value (no string)",
Event: fakeSpTsvParsed,
Ctx: "contexts_com_snowplowanalytics_snowplow_web_page_1",
Prop: "id",
Expected: nil,
Error: errors.New("invalid header value"),
},
{
Scenario: "invalid_header_value (no base64)",
Event: gtmssInvalidNoB64Parsed,
Ctx: "contexts_com_google_tag-manager_server-side_preview_mode_1",
Prop: "x-gtm-server-preview",
Expected: nil,
Error: errors.New("illegal base64 data at input"),
},
{
Scenario: "event_without_contexts",
Event: spTsvNoCtxParsed,
Expand Down Expand Up @@ -343,7 +385,7 @@ func Benchmark_GTMSSPreview_With_Preview_Ctx_no_intermediate(b *testing.B) {
prop := "x-gtm-server-preview"
header := "x-gtm-server-preview"

transFunction := gtmssPreviewTransformation(ctx, prop, header)
transFunction := gtmssPreviewTransformation(ctx, prop, header, fiveYears)

for n := 0; n < b.N; n++ {
transFunction(inputMsg, nil)
Expand All @@ -362,7 +404,7 @@ func Benchmark_GTMSSPreview_With_Preview_Ctx_With_intermediate(b *testing.B) {
prop := "x-gtm-server-preview"
header := "x-gtm-server-preview"

transFunction := gtmssPreviewTransformation(ctx, prop, header)
transFunction := gtmssPreviewTransformation(ctx, prop, header, fiveYears)

for n := 0; n < b.N; n++ {
transFunction(inputMsg, interState)
Expand All @@ -380,7 +422,7 @@ func Benchmark_GTMSSPreview_No_Preview_Ctx_no_intermediate(b *testing.B) {
prop := "x-gtm-server-preview"
header := "x-gtm-server-preview"

transFunction := gtmssPreviewTransformation(ctx, prop, header)
transFunction := gtmssPreviewTransformation(ctx, prop, header, fiveYears)

for n := 0; n < b.N; n++ {
transFunction(inputMsg, nil)
Expand All @@ -399,7 +441,7 @@ func Benchmark_GTMSSPreview_No_Preview_Ctx_With_intermediate(b *testing.B) {
prop := "x-gtm-server-preview"
header := "x-gtm-server-preview"

transFunction := gtmssPreviewTransformation(ctx, prop, header)
transFunction := gtmssPreviewTransformation(ctx, prop, header, fiveYears)

for n := 0; n < b.N; n++ {
transFunction(inputMsg, interState)
Expand Down Expand Up @@ -449,6 +491,9 @@ var spTsvNoGtmssParsed, _ = analytics.ParseEvent(string(spTsvNoGtmss))
var spTsvWithGtmss = []byte(`media-test web 2024-03-12 04:27:01.760 2024-03-12 04:27:01.755 2024-03-12 04:27:01.743 unstruct 9be3afe8-8a62-41ac-93db-12f425d82ac9 spTest js-3.17.0 snowplow-micro-2.0.0-stdout$ snowplow-micro-2.0.0 media_tester 172.17.0.1 23a0eb65-83f6-4957-839e-f3044bfefb99 1 a2f53212-26a3-4781-81d6-f14aa8d4552b http://localhost:8000/?sgtm-preview-header=ZW52LTcyN3wtMkMwR084ekptbWxiZmpkcHNIRENBfDE4ZTJkYzgxMDc2NDg1MjVmMzI2Mw== http localhost 8000 / sgtm-preview-header=ZW52LTcyN3wtMkMwR084ekptbWxiZmpkcHNIRENBfDE4ZTJkYzgxMDc2NDg1MjVmMzI2Mw== {"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0","data":[{"schema":"iglu:org.whatwg/media_element/jsonschema/1-0-0","data":{"htmlId":"bunny-mp4","mediaType":"VIDEO","autoPlay":false,"buffered":[{"start":0,"end":1.291666}],"controls":true,"currentSrc":"https://archive.org/download/BigBuckBunny_124/Content/big_buck_bunny_720p_surround.mp4","defaultMuted":false,"defaultPlaybackRate":1,"error":null,"networkState":"NETWORK_LOADING","preload":"","readyState":"HAVE_ENOUGH_DATA","seekable":[{"start":0,"end":596.503219}],"seeking":false,"src":"https://archive.org/download/BigBuckBunny_124/Content/big_buck_bunny_720p_surround.mp4","textTracks":[],"fileExtension":"mp4","fullscreen":false,"pictureInPicture":false}},{"schema":"iglu:com.snowplowanalytics.snowplow/media_player/jsonschema/1-0-0","data":{"currentTime":0,"duration":596.503219,"ended":false,"loop":false,"muted":false,"paused":false,"playbackRate":1,"volume":100}},{"schema":"iglu:org.whatwg/video_element/jsonschema/1-0-0","data":{"poster":"","videoHeight":360,"videoWidth":640}},{"schema":"iglu:com.snowplowanalytics.snowplow/web_page/jsonschema/1-0-0","data":{"id":"021c4d09-e502-4562-8182-5ac7247125ec"}},{"schema":"iglu:com.google.tag-manager.server-side/user_data/jsonschema/1-0-0","data":{"email_address":"foo@example.com","phone_number":"+15551234567","address":{"first_name":"Jane","last_name":"Doe","street":"123 Fake St","city":"San Francisco","region":"CA","postal_code":"94016","country":"US"}}},{"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-2","data":{"osType":"testOsType","osVersion":"testOsVersion","deviceManufacturer":"testDevMan","deviceModel":"testDevModel"}},{"schema":"iglu:com.google.tag-manager.server-side/preview_mode/jsonschema/1-0-0","data":{"x-gtm-server-preview":"ZW52LTcyN3wtMkMwR084ekptbWxiZmpkcHNIRENBfDE4ZTJkYzgxMDc2NDg1MjVmMzI2Mw=="}},{"schema":"iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-0-2","data":{"userId":"23a0eb65-83f6-4957-839e-f3044bfefb99","sessionId":"73fcdaa3-0164-41ce-a336-fb00c4ebf68c","eventIndex":7,"sessionIndex":1,"previousSessionId":null,"storageMechanism":"COOKIE_1","firstEventId":"327b9ff9-ed5f-40cf-918a-1b1a775ae347","firstEventTimestamp":"2024-03-12T04:25:36.684Z"}}]} {"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.snowplowanalytics.snowplow/media_player_event/jsonschema/1-0-0","data":{"type":"play"}}} Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0 en-US 1 24 1920 935 Europe/Athens 1920 1080 windows-1252 1920 935 2024-03-12 04:27:01.745 73fcdaa3-0164-41ce-a336-fb00c4ebf68c 2024-03-12 04:27:01.753 com.snowplowanalytics.snowplow media_player_event jsonschema 1-0-0 `)
var spTsvWithGtmssParsed, _ = analytics.ParseEvent(string(spTsvWithGtmss))

var gtmssInvalidNoB64 = []byte(`media-test web 2024-03-12 04:27:01.760 2024-03-12 04:27:01.755 2024-03-12 04:27:01.743 unstruct 9be3afe8-8a62-41ac-93db-12f425d82ac9 spTest js-3.17.0 snowplow-micro-2.0.0-stdout$ snowplow-micro-2.0.0 media_tester 172.17.0.1 23a0eb65-83f6-4957-839e-f3044bfefb99 1 a2f53212-26a3-4781-81d6-f14aa8d4552b http://localhost:8000/?sgtm-preview-header=ZW52LTcyN3wtMkMwR084ekptbWxiZmpkcHNIRENBfDE4ZTJkYzgxMDc2NDg1MjVmMzI2Mw== http localhost 8000 / sgtm-preview-header=ZW52LTcyN3wtMkMwR084ekptbWxiZmpkcHNIRENBfDE4ZTJkYzgxMDc2NDg1MjVmMzI2Mw== {"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0","data":[{"schema":"iglu:org.whatwg/media_element/jsonschema/1-0-0","data":{"htmlId":"bunny-mp4","mediaType":"VIDEO","autoPlay":false,"buffered":[{"start":0,"end":1.291666}],"controls":true,"currentSrc":"https://archive.org/download/BigBuckBunny_124/Content/big_buck_bunny_720p_surround.mp4","defaultMuted":false,"defaultPlaybackRate":1,"error":null,"networkState":"NETWORK_LOADING","preload":"","readyState":"HAVE_ENOUGH_DATA","seekable":[{"start":0,"end":596.503219}],"seeking":false,"src":"https://archive.org/download/BigBuckBunny_124/Content/big_buck_bunny_720p_surround.mp4","textTracks":[],"fileExtension":"mp4","fullscreen":false,"pictureInPicture":false}},{"schema":"iglu:com.snowplowanalytics.snowplow/media_player/jsonschema/1-0-0","data":{"currentTime":0,"duration":596.503219,"ended":false,"loop":false,"muted":false,"paused":false,"playbackRate":1,"volume":100}},{"schema":"iglu:org.whatwg/video_element/jsonschema/1-0-0","data":{"poster":"","videoHeight":360,"videoWidth":640}},{"schema":"iglu:com.snowplowanalytics.snowplow/web_page/jsonschema/1-0-0","data":{"id":"021c4d09-e502-4562-8182-5ac7247125ec"}},{"schema":"iglu:com.google.tag-manager.server-side/user_data/jsonschema/1-0-0","data":{"email_address":"foo@example.com","phone_number":"+15551234567","address":{"first_name":"Jane","last_name":"Doe","street":"123 Fake St","city":"San Francisco","region":"CA","postal_code":"94016","country":"US"}}},{"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-2","data":{"osType":"testOsType","osVersion":"testOsVersion","deviceManufacturer":"testDevMan","deviceModel":"testDevModel"}},{"schema":"iglu:com.google.tag-manager.server-side/preview_mode/jsonschema/1-0-0","data":{"x-gtm-server-preview":"this is not valid base64"}},{"schema":"iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-0-2","data":{"userId":"23a0eb65-83f6-4957-839e-f3044bfefb99","sessionId":"73fcdaa3-0164-41ce-a336-fb00c4ebf68c","eventIndex":7,"sessionIndex":1,"previousSessionId":null,"storageMechanism":"COOKIE_1","firstEventId":"327b9ff9-ed5f-40cf-918a-1b1a775ae347","firstEventTimestamp":"2024-03-12T04:25:36.684Z"}}]} {"schema":"iglu:com.snowplowanalytics.snowplow/unstruct_event/jsonschema/1-0-0","data":{"schema":"iglu:com.snowplowanalytics.snowplow/media_player_event/jsonschema/1-0-0","data":{"type":"play"}}} Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0 en-US 1 24 1920 935 Europe/Athens 1920 1080 windows-1252 1920 935 2024-03-12 04:27:01.745 73fcdaa3-0164-41ce-a336-fb00c4ebf68c 2024-03-12 04:27:01.753 com.snowplowanalytics.snowplow media_player_event jsonschema 1-0-0 `)
var gtmssInvalidNoB64Parsed, _ = analytics.ParseEvent(string(gtmssInvalidNoB64))

var fakeSpTsv = []byte(`media-test web 2024-03-12 04:25:40.277 2024-03-12 04:25:40.272 2024-03-12 04:25:36.685 page_view 1313411b-282f-4aa9-b37c-c60d4723cf47 spTest js-3.17.0 snowplow-micro-2.0.0-stdout$ snowplow-micro-2.0.0 media_tester 172.17.0.1 23a0eb65-83f6-4957-839e-f3044bfefb99 1 a2f53212-26a3-4781-81d6-f14aa8d4552b http://localhost:8000/ Test Media Tracking http localhost 8000 / {"schema":"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-0","data":[{"schema":"iglu:com.snowplowanalytics.snowplow/web_page/jsonschema/1-0-0","data":{"id":["FAILS"]}},{"schema":"iglu:com.google.tag-manager.server-side/user_data/jsonschema/1-0-0","data":{"email_address":"foo@example.com","phone_number":"+15551234567","address":{"first_name":"Jane","last_name":"Doe","street":"123 Fake St","city":"San Francisco","region":"CA","postal_code":"94016","country":"US"}}},{"schema":"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-2","data":{"osType":"testOsType","osVersion":"testOsVersion","deviceManufacturer":"testDevMan","deviceModel":"testDevModel"}},{"schema":"iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-0-2","data":{"userId":"23a0eb65-83f6-4957-839e-f3044bfefb99","sessionId":"73fcdaa3-0164-41ce-a336-fb00c4ebf68c","eventIndex":2,"sessionIndex":1,"previousSessionId":null,"storageMechanism":"COOKIE_1","firstEventId":"327b9ff9-ed5f-40cf-918a-1b1a775ae347","firstEventTimestamp":"2024-03-12T04:25:36.684Z"}}]} Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0 en-US 1 24 1920 935 Europe/Athens 1920 1080 windows-1252 1920 935 2024-03-12 04:25:40.268 73fcdaa3-0164-41ce-a336-fb00c4ebf68c 2024-03-12 04:25:36.689 com.snowplowanalytics.snowplow page_view jsonschema 1-0-0 `)
var fakeSpTsvParsed, _ = analytics.ParseEvent(string(fakeSpTsv))

Expand Down

0 comments on commit 4fbefc7

Please sign in to comment.