From f8ba0fddae373ad2354050d68b8a633e6b3cdd12 Mon Sep 17 00:00:00 2001 From: Thomas Poignant Date: Tue, 14 Jan 2025 14:42:03 +0100 Subject: [PATCH] feat: Add metadata in the exporters (#2930) * Fix doc Signed-off-by: Thomas Poignant * Add metadata field to exporter Signed-off-by: Thomas Poignant * Add swagger metadata Signed-off-by: Thomas Poignant * Add test metadata Signed-off-by: Thomas Poignant --------- Signed-off-by: Thomas Poignant --- .../controller/collect_eval_data.go | 5 +- .../controller/collect_eval_data_test.go | 11 +++ cmd/relayproxy/docs/docs.go | 21 ++++- cmd/relayproxy/docs/swagger.json | 21 ++++- cmd/relayproxy/docs/swagger.yaml | 13 ++- .../model/collect_eval_data_request.go | 2 +- .../valid_collected_data_metadata.json | 18 ++++ .../valid_request_metadata.json | 21 +++++ .../valid_response_metadata.json | 3 + exporter/feature_event.go | 5 ++ exporter/feature_event_test.go | 86 +++++++++++++++++++ exporter/fileexporter/exporter_test.go | 12 +-- .../export-evaluation-data/file.mdx | 1 - .../export-evaluation-data/file.mdx | 1 - 14 files changed, 199 insertions(+), 21 deletions(-) create mode 100644 cmd/relayproxy/testdata/controller/collect_eval_data/valid_collected_data_metadata.json create mode 100644 cmd/relayproxy/testdata/controller/collect_eval_data/valid_request_metadata.json create mode 100644 cmd/relayproxy/testdata/controller/collect_eval_data/valid_response_metadata.json diff --git a/cmd/relayproxy/controller/collect_eval_data.go b/cmd/relayproxy/controller/collect_eval_data.go index 0225a3e8221..6ce38f9ccec 100644 --- a/cmd/relayproxy/controller/collect_eval_data.go +++ b/cmd/relayproxy/controller/collect_eval_data.go @@ -74,11 +74,12 @@ func (h *collectEvalData) Handler(c echo.Context) error { event.CreationDate, _ = strconv.ParseInt( strconv.FormatInt(event.CreationDate, 10)[:10], 10, 64) } + if reqBody.Meta != nil { + event.Metadata = reqBody.Meta + } h.goFF.CollectEventData(event) } - h.metrics.IncCollectEvalData(float64(len(reqBody.Events))) - return c.JSON(http.StatusOK, model.CollectEvalDataResponse{ IngestedContentCount: len(reqBody.Events), }) diff --git a/cmd/relayproxy/controller/collect_eval_data_test.go b/cmd/relayproxy/controller/collect_eval_data_test.go index f4042732eb3..d31350e7363 100644 --- a/cmd/relayproxy/controller/collect_eval_data_test.go +++ b/cmd/relayproxy/controller/collect_eval_data_test.go @@ -99,6 +99,17 @@ func Test_collect_eval_data_Handler(t *testing.T) { collectedDataFile: "../testdata/controller/collect_eval_data/valid_collected_data_with_timestamp_ms.json", }, }, + { + name: "should have the metadata in the exporter", + args: args{ + "../testdata/controller/collect_eval_data/valid_request_metadata.json", + }, + want: want{ + httpCode: http.StatusOK, + bodyFile: "../testdata/controller/collect_eval_data/valid_response_metadata.json", + collectedDataFile: "../testdata/controller/collect_eval_data/valid_collected_data_metadata.json", + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/cmd/relayproxy/docs/docs.go b/cmd/relayproxy/docs/docs.go index 4cc0559e49a..47f898072c4 100644 --- a/cmd/relayproxy/docs/docs.go +++ b/cmd/relayproxy/docs/docs.go @@ -651,6 +651,14 @@ const docTemplate = `{ "type": "string", "example": "feature" }, + "metadata": { + "description": "Metadata are static information added in the providers to give context about the events generated.", + "allOf": [ + { + "$ref": "#/definitions/exporter.FeatureEventMetadata" + } + ] + }, "source": { "description": "Source indicates where the event was generated.\nThis is set to SERVER when the event was evaluated in the relay-proxy and PROVIDER_CACHE when it is evaluated from the cache.", "type": "string", @@ -676,6 +684,10 @@ const docTemplate = `{ } } }, + "exporter.FeatureEventMetadata": { + "type": "object", + "additionalProperties": true + }, "flag.ErrorCode": { "type": "string", "enum": [ @@ -744,10 +756,11 @@ const docTemplate = `{ }, "meta": { "description": "Meta are the extra information added during the configuration", - "type": "object", - "additionalProperties": { - "type": "string" - } + "allOf": [ + { + "$ref": "#/definitions/exporter.FeatureEventMetadata" + } + ] } } }, diff --git a/cmd/relayproxy/docs/swagger.json b/cmd/relayproxy/docs/swagger.json index ed79c4a0454..d033f1e4eed 100644 --- a/cmd/relayproxy/docs/swagger.json +++ b/cmd/relayproxy/docs/swagger.json @@ -643,6 +643,14 @@ "type": "string", "example": "feature" }, + "metadata": { + "description": "Metadata are static information added in the providers to give context about the events generated.", + "allOf": [ + { + "$ref": "#/definitions/exporter.FeatureEventMetadata" + } + ] + }, "source": { "description": "Source indicates where the event was generated.\nThis is set to SERVER when the event was evaluated in the relay-proxy and PROVIDER_CACHE when it is evaluated from the cache.", "type": "string", @@ -668,6 +676,10 @@ } } }, + "exporter.FeatureEventMetadata": { + "type": "object", + "additionalProperties": true + }, "flag.ErrorCode": { "type": "string", "enum": [ @@ -736,10 +748,11 @@ }, "meta": { "description": "Meta are the extra information added during the configuration", - "type": "object", - "additionalProperties": { - "type": "string" - } + "allOf": [ + { + "$ref": "#/definitions/exporter.FeatureEventMetadata" + } + ] } } }, diff --git a/cmd/relayproxy/docs/swagger.yaml b/cmd/relayproxy/docs/swagger.yaml index 580b62ffff1..2b32513b837 100644 --- a/cmd/relayproxy/docs/swagger.yaml +++ b/cmd/relayproxy/docs/swagger.yaml @@ -43,6 +43,11 @@ definitions: A feature event will only be generated if the trackEvents attribute of the flag is set to true. example: feature type: string + metadata: + allOf: + - $ref: '#/definitions/exporter.FeatureEventMetadata' + description: Metadata are static information added in the providers to give + context about the events generated. source: description: |- Source indicates where the event was generated. @@ -71,6 +76,9 @@ definitions: example: v1.0.0 type: string type: object + exporter.FeatureEventMetadata: + additionalProperties: true + type: object flag.ErrorCode: enum: - PROVIDER_NOT_READY @@ -121,10 +129,9 @@ definitions: $ref: '#/definitions/exporter.FeatureEvent' type: array meta: - additionalProperties: - type: string + allOf: + - $ref: '#/definitions/exporter.FeatureEventMetadata' description: Meta are the extra information added during the configuration - type: object type: object model.CollectEvalDataResponse: properties: diff --git a/cmd/relayproxy/model/collect_eval_data_request.go b/cmd/relayproxy/model/collect_eval_data_request.go index d27c5486102..9891aa7503e 100644 --- a/cmd/relayproxy/model/collect_eval_data_request.go +++ b/cmd/relayproxy/model/collect_eval_data_request.go @@ -7,7 +7,7 @@ import ( // CollectEvalDataRequest is the request to collect data in type CollectEvalDataRequest struct { // Meta are the extra information added during the configuration - Meta map[string]string `json:"meta"` + Meta exporter.FeatureEventMetadata `json:"meta"` // Events is the list of the event we send in the payload Events []exporter.FeatureEvent `json:"events"` diff --git a/cmd/relayproxy/testdata/controller/collect_eval_data/valid_collected_data_metadata.json b/cmd/relayproxy/testdata/controller/collect_eval_data/valid_collected_data_metadata.json new file mode 100644 index 00000000000..02b2554d347 --- /dev/null +++ b/cmd/relayproxy/testdata/controller/collect_eval_data/valid_collected_data_metadata.json @@ -0,0 +1,18 @@ +{ + "kind": "feature", + "contextKind": "user", + "userKey": "94a25909-20d8-40cc-8500-fee99b569345", + "creationDate": 1680246000, + "key": "my-feature-flag", + "variation": "admin-variation", + "value": "string", + "default": false, + "version": "v1.0.0", + "source": "PROVIDER_CACHE", + "metadata": { + "environment": "production", + "sdkVersion": "v1.0.0", + "source": "my-source", + "timestamp": 1680246000 + } +} diff --git a/cmd/relayproxy/testdata/controller/collect_eval_data/valid_request_metadata.json b/cmd/relayproxy/testdata/controller/collect_eval_data/valid_request_metadata.json new file mode 100644 index 00000000000..13132c1b393 --- /dev/null +++ b/cmd/relayproxy/testdata/controller/collect_eval_data/valid_request_metadata.json @@ -0,0 +1,21 @@ +{ + "events": [ + { + "contextKind": "user", + "creationDate": 1680246000, + "default": false, + "key": "my-feature-flag", + "kind": "feature", + "userKey": "94a25909-20d8-40cc-8500-fee99b569345", + "value": "string", + "variation": "admin-variation", + "version": "v1.0.0" + } + ], + "meta": { + "environment": "production", + "sdkVersion": "v1.0.0", + "source": "my-source", + "timestamp": 1680246000 + } +} \ No newline at end of file diff --git a/cmd/relayproxy/testdata/controller/collect_eval_data/valid_response_metadata.json b/cmd/relayproxy/testdata/controller/collect_eval_data/valid_response_metadata.json new file mode 100644 index 00000000000..30653ead6b5 --- /dev/null +++ b/cmd/relayproxy/testdata/controller/collect_eval_data/valid_response_metadata.json @@ -0,0 +1,3 @@ +{ + "ingestedContentCount": 1 +} \ No newline at end of file diff --git a/exporter/feature_event.go b/exporter/feature_event.go index 08a68ed2524..b2e0a41a80a 100644 --- a/exporter/feature_event.go +++ b/exporter/feature_event.go @@ -7,6 +7,8 @@ import ( "github.com/thomaspoignant/go-feature-flag/ffcontext" ) +type FeatureEventMetadata = map[string]interface{} + func NewFeatureEvent( ctx ffcontext.Context, flagKey string, @@ -75,6 +77,9 @@ type FeatureEvent struct { // Source indicates where the event was generated. // This is set to SERVER when the event was evaluated in the relay-proxy and PROVIDER_CACHE when it is evaluated from the cache. Source string `json:"source" example:"SERVER" parquet:"name=source, type=BYTE_ARRAY, convertedtype=UTF8"` + + // Metadata are static information added in the providers to give context about the events generated. + Metadata FeatureEventMetadata `json:"metadata,omitempty" parquet:"name=metadata, type=MAP, keytype=BYTE_ARRAY, keyconvertedtype=UTF8, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"` } // MarshalInterface marshals all interface type fields in FeatureEvent into JSON-encoded string. diff --git a/exporter/feature_event_test.go b/exporter/feature_event_test.go index bc99f52e3f3..28b34cea101 100644 --- a/exporter/feature_event_test.go +++ b/exporter/feature_event_test.go @@ -1,6 +1,7 @@ package exporter_test import ( + "encoding/json" "testing" "time" @@ -114,3 +115,88 @@ func TestFeatureEvent_MarshalInterface(t *testing.T) { }) } } + +func TestFeatureEvent_MarshalJSON(t *testing.T) { + tests := []struct { + name string + featureEvent *exporter.FeatureEvent + want string + wantErr assert.ErrorAssertionFunc + }{ + { + name: "Should not return a metadata field if metadata is empty", + featureEvent: &exporter.FeatureEvent{ + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: map[string]interface{}{ + "string": "string", + "bool": true, + "float": 1.23, + "int": 1, + }, + Default: false, + Metadata: map[string]interface{}{}, + }, + want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"string":"string","bool":true,"float":1.23,"int":1},"default":false}`, + wantErr: assert.NoError, + }, + { + name: "Should not return a metadata field if metadata is nil", + featureEvent: &exporter.FeatureEvent{ + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: map[string]interface{}{ + "string": "string", + "bool": true, + "float": 1.23, + "int": 1, + }, + Default: false, + }, + want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"string":"string","bool":true,"float":1.23,"int":1},"default":false}`, + wantErr: assert.NoError, + }, + { + name: "Should return a metadata field if metadata is not empty", + featureEvent: &exporter.FeatureEvent{ + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: map[string]interface{}{ + "string": "string", + "bool": true, + "float": 1.23, + "int": 1, + }, + Default: false, + Metadata: map[string]interface{}{ + "metadata1": "metadata1", + "metadata2": 24, + "metadata3": true, + }, + }, + want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"string":"string","bool":true,"float":1.23,"int":1},"default":false,"metadata":{"metadata1":"metadata1","metadata2":24,"metadata3":true}}`, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := json.Marshal(tt.featureEvent) + tt.wantErr(t, err) + if err != nil { + assert.JSONEq(t, tt.want, string(got)) + } + }) + } +} diff --git a/exporter/fileexporter/exporter_test.go b/exporter/fileexporter/exporter_test.go index 6ab1e6791bb..2b00e74f805 100644 --- a/exporter/fileexporter/exporter_test.go +++ b/exporter/fileexporter/exporter_test.go @@ -107,7 +107,7 @@ func TestFile_Export(t *testing.T) { featureEvents: []exporter.FeatureEvent{ { Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", - Variation: "Default", Value: "YO", Default: false, Source: "SERVER", + Variation: "Default", Value: "YO", Default: false, Source: "SERVER", Metadata: map[string]interface{}{"test": "test"}, }, { Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", @@ -120,11 +120,11 @@ func TestFile_Export(t *testing.T) { featureEvents: []exporter.FeatureEvent{ { Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", - Variation: "Default", Value: `"YO"`, Default: false, Source: "SERVER", + Variation: "Default", Value: `"YO"`, Default: false, Source: "SERVER", Metadata: map[string]interface{}{"test": "test"}, }, { Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", - Variation: "Default", Value: `"YO2"`, Default: false, Version: "127", Source: "SERVER", + Variation: "Default", Value: `"YO2"`, Default: false, Version: "127", Source: "SERVER", Metadata: map[string]interface{}{}, }, }, }, @@ -175,8 +175,9 @@ func TestFile_Export(t *testing.T) { "float": 1.23, "int": 1, }, - Default: false, - Source: "SERVER", + Default: false, + Source: "SERVER", + Metadata: map[string]interface{}{"test": "test"}, }, }, }, @@ -193,6 +194,7 @@ func TestFile_Export(t *testing.T) { Value: `{"bool":true,"float":1.23,"int":1,"string":"string"}`, Default: false, Source: "SERVER", + Metadata: map[string]interface{}{"test": "test"}, }, }, }, diff --git a/website/docs/integrations/export-evaluation-data/file.mdx b/website/docs/integrations/export-evaluation-data/file.mdx index c4c77a3fcc0..852c0756282 100644 --- a/website/docs/integrations/export-evaluation-data/file.mdx +++ b/website/docs/integrations/export-evaluation-data/file.mdx @@ -35,7 +35,6 @@ exporter: |---------------------------|:----------------:|--------|------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `kind` | | string | **none** | **Value should be `file`**.
_This field is mandatory and describes which retriever you are using._ | | `outputDir` | | string | **none** | OutputDir is the location of the directory where to store the exported files. | -| `path` | | string | **bucket root level** | The location of the directory in Google Cloud Storage. | | `flushInterval` | | int | `60000` | The interval in millisecond between 2 calls to the webhook _(if the `maxEventInMemory` is reached before the flushInterval we will call the exporter before)_. | | `maxEventInMemory` | | int | `100000` | If we hit that limit we will call the exporter. | | `format` | | string | `JSON` | Format is the output format you want in your exported file. Available format: `JSON`, `CSV`, `Parquet`. | diff --git a/website/versioned_docs/version-v1.40.0/integrations/export-evaluation-data/file.mdx b/website/versioned_docs/version-v1.40.0/integrations/export-evaluation-data/file.mdx index c4c77a3fcc0..852c0756282 100644 --- a/website/versioned_docs/version-v1.40.0/integrations/export-evaluation-data/file.mdx +++ b/website/versioned_docs/version-v1.40.0/integrations/export-evaluation-data/file.mdx @@ -35,7 +35,6 @@ exporter: |---------------------------|:----------------:|--------|------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `kind` | | string | **none** | **Value should be `file`**.
_This field is mandatory and describes which retriever you are using._ | | `outputDir` | | string | **none** | OutputDir is the location of the directory where to store the exported files. | -| `path` | | string | **bucket root level** | The location of the directory in Google Cloud Storage. | | `flushInterval` | | int | `60000` | The interval in millisecond between 2 calls to the webhook _(if the `maxEventInMemory` is reached before the flushInterval we will call the exporter before)_. | | `maxEventInMemory` | | int | `100000` | If we hit that limit we will call the exporter. | | `format` | | string | `JSON` | Format is the output format you want in your exported file. Available format: `JSON`, `CSV`, `Parquet`. |