Skip to content

Commit

Permalink
feat: Add metadata in the exporters (#2930)
Browse files Browse the repository at this point in the history 
* Fix doc

Signed-off-by: Thomas Poignant <thomas.poignant@gofeatureflag.org>

* Add metadata field to exporter

Signed-off-by: Thomas Poignant <thomas.poignant@gofeatureflag.org>

* Add swagger metadata

Signed-off-by: Thomas Poignant <thomas.poignant@gofeatureflag.org>

* Add test metadata

Signed-off-by: Thomas Poignant <thomas.poignant@gofeatureflag.org>

---------

Signed-off-by: Thomas Poignant <thomas.poignant@gofeatureflag.org>
  • Loading branch information
@thomaspoignant
thomaspoignant authored Jan 14, 2025
1 parent caf3bfb commit f8ba0fd
Show file tree
Hide file tree
Showing 14 changed files with 199 additions and 21 deletions.
5 changes: 3 additions & 2 deletions cmd/relayproxy/controller/collect_eval_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,12 @@ func (h *collectEvalData) Handler(c echo.Context) error {
event.CreationDate, _ = strconv.ParseInt(
strconv.FormatInt(event.CreationDate, 10)[:10], 10, 64)
}
if reqBody.Meta != nil {
event.Metadata = reqBody.Meta
}
h.goFF.CollectEventData(event)
}

h.metrics.IncCollectEvalData(float64(len(reqBody.Events)))

return c.JSON(http.StatusOK, model.CollectEvalDataResponse{
IngestedContentCount: len(reqBody.Events),
})
Expand Down
11 changes: 11 additions & 0 deletions cmd/relayproxy/controller/collect_eval_data_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,17 @@ func Test_collect_eval_data_Handler(t *testing.T) {
collectedDataFile: "../testdata/controller/collect_eval_data/valid_collected_data_with_timestamp_ms.json",
},
},
{
name: "should have the metadata in the exporter",
args: args{
"../testdata/controller/collect_eval_data/valid_request_metadata.json",
},
want: want{
httpCode: http.StatusOK,
bodyFile: "../testdata/controller/collect_eval_data/valid_response_metadata.json",
collectedDataFile: "../testdata/controller/collect_eval_data/valid_collected_data_metadata.json",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
21 changes: 17 additions & 4 deletions cmd/relayproxy/docs/docs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -651,6 +651,14 @@ const docTemplate = `{
"type": "string",
"example": "feature"
},
"metadata": {
"description": "Metadata are static information added in the providers to give context about the events generated.",
"allOf": [
{
"$ref": "#/definitions/exporter.FeatureEventMetadata"
}
]
},
"source": {
"description": "Source indicates where the event was generated.\nThis is set to SERVER when the event was evaluated in the relay-proxy and PROVIDER_CACHE when it is evaluated from the cache.",
"type": "string",
Expand All @@ -676,6 +684,10 @@ const docTemplate = `{
}
}
},
"exporter.FeatureEventMetadata": {
"type": "object",
"additionalProperties": true
},
"flag.ErrorCode": {
"type": "string",
"enum": [
Expand Down Expand Up @@ -744,10 +756,11 @@ const docTemplate = `{
},
"meta": {
"description": "Meta are the extra information added during the configuration",
"type": "object",
"additionalProperties": {
"type": "string"
}
"allOf": [
{
"$ref": "#/definitions/exporter.FeatureEventMetadata"
}
]
}
}
},
Expand Down
21 changes: 17 additions & 4 deletions cmd/relayproxy/docs/swagger.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -643,6 +643,14 @@
"type": "string",
"example": "feature"
},
"metadata": {
"description": "Metadata are static information added in the providers to give context about the events generated.",
"allOf": [
{
"$ref": "#/definitions/exporter.FeatureEventMetadata"
}
]
},
"source": {
"description": "Source indicates where the event was generated.\nThis is set to SERVER when the event was evaluated in the relay-proxy and PROVIDER_CACHE when it is evaluated from the cache.",
"type": "string",
Expand All @@ -668,6 +676,10 @@
}
}
},
"exporter.FeatureEventMetadata": {
"type": "object",
"additionalProperties": true
},
"flag.ErrorCode": {
"type": "string",
"enum": [
Expand Down Expand Up @@ -736,10 +748,11 @@
},
"meta": {
"description": "Meta are the extra information added during the configuration",
"type": "object",
"additionalProperties": {
"type": "string"
}
"allOf": [
{
"$ref": "#/definitions/exporter.FeatureEventMetadata"
}
]
}
}
},
Expand Down
13 changes: 10 additions & 3 deletions cmd/relayproxy/docs/swagger.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ definitions:
A feature event will only be generated if the trackEvents attribute of the flag is set to true.
example: feature
type: string
metadata:
allOf:
- $ref: '#/definitions/exporter.FeatureEventMetadata'
description: Metadata are static information added in the providers to give
context about the events generated.
source:
description: |-
Source indicates where the event was generated.
Expand Down Expand Up @@ -71,6 +76,9 @@ definitions:
example: v1.0.0
type: string
type: object
exporter.FeatureEventMetadata:
additionalProperties: true
type: object
flag.ErrorCode:
enum:
- PROVIDER_NOT_READY
Expand Down Expand Up @@ -121,10 +129,9 @@ definitions:
$ref: '#/definitions/exporter.FeatureEvent'
type: array
meta:
additionalProperties:
type: string
allOf:
- $ref: '#/definitions/exporter.FeatureEventMetadata'
description: Meta are the extra information added during the configuration
type: object
type: object
model.CollectEvalDataResponse:
properties:
Expand Down
2 changes: 1 addition & 1 deletion cmd/relayproxy/model/collect_eval_data_request.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
// CollectEvalDataRequest is the request to collect data in
type CollectEvalDataRequest struct {
// Meta are the extra information added during the configuration
Meta map[string]string `json:"meta"`
Meta exporter.FeatureEventMetadata `json:"meta"`

// Events is the list of the event we send in the payload
Events []exporter.FeatureEvent `json:"events"`
Expand Down
18 changes: 18 additions & 0 deletions cmd/relayproxy/testdata/controller/collect_eval_data/valid_collected_data_metadata.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"kind": "feature",
"contextKind": "user",
"userKey": "94a25909-20d8-40cc-8500-fee99b569345",
"creationDate": 1680246000,
"key": "my-feature-flag",
"variation": "admin-variation",
"value": "string",
"default": false,
"version": "v1.0.0",
"source": "PROVIDER_CACHE",
"metadata": {
"environment": "production",
"sdkVersion": "v1.0.0",
"source": "my-source",
"timestamp": 1680246000
}
}
21 changes: 21 additions & 0 deletions cmd/relayproxy/testdata/controller/collect_eval_data/valid_request_metadata.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"events": [
{
"contextKind": "user",
"creationDate": 1680246000,
"default": false,
"key": "my-feature-flag",
"kind": "feature",
"userKey": "94a25909-20d8-40cc-8500-fee99b569345",
"value": "string",
"variation": "admin-variation",
"version": "v1.0.0"
}
],
"meta": {
"environment": "production",
"sdkVersion": "v1.0.0",
"source": "my-source",
"timestamp": 1680246000
}
}
3 changes: 3 additions & 0 deletions cmd/relayproxy/testdata/controller/collect_eval_data/valid_response_metadata.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"ingestedContentCount": 1
}
5 changes: 5 additions & 0 deletions exporter/feature_event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"github.com/thomaspoignant/go-feature-flag/ffcontext"
)

type FeatureEventMetadata = map[string]interface{}

func NewFeatureEvent(
ctx ffcontext.Context,
flagKey string,
Expand Down Expand Up @@ -75,6 +77,9 @@ type FeatureEvent struct {
// Source indicates where the event was generated.
// This is set to SERVER when the event was evaluated in the relay-proxy and PROVIDER_CACHE when it is evaluated from the cache.
Source string `json:"source" example:"SERVER" parquet:"name=source, type=BYTE_ARRAY, convertedtype=UTF8"`

// Metadata are static information added in the providers to give context about the events generated.
Metadata FeatureEventMetadata `json:"metadata,omitempty" parquet:"name=metadata, type=MAP, keytype=BYTE_ARRAY, keyconvertedtype=UTF8, valuetype=BYTE_ARRAY, valueconvertedtype=UTF8"`
}

// MarshalInterface marshals all interface type fields in FeatureEvent into JSON-encoded string.
Expand Down
86 changes: 86 additions & 0 deletions exporter/feature_event_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package exporter_test

import (
"encoding/json"
"testing"
"time"

Expand Down Expand Up @@ -114,3 +115,88 @@ func TestFeatureEvent_MarshalInterface(t *testing.T) {
})
}
}

func TestFeatureEvent_MarshalJSON(t *testing.T) {
tests := []struct {
name string
featureEvent *exporter.FeatureEvent
want string
wantErr assert.ErrorAssertionFunc
}{
{
name: "Should not return a metadata field if metadata is empty",
featureEvent: &exporter.FeatureEvent{
Kind: "feature",
ContextKind: "anonymousUser",
UserKey: "ABCD",
CreationDate: 1617970547,
Key: "random-key",
Variation: "Default",
Value: map[string]interface{}{
"string": "string",
"bool": true,
"float": 1.23,
"int": 1,
},
Default: false,
Metadata: map[string]interface{}{},
},
want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"string":"string","bool":true,"float":1.23,"int":1},"default":false}`,
wantErr: assert.NoError,
},
{
name: "Should not return a metadata field if metadata is nil",
featureEvent: &exporter.FeatureEvent{
Kind: "feature",
ContextKind: "anonymousUser",
UserKey: "ABCD",
CreationDate: 1617970547,
Key: "random-key",
Variation: "Default",
Value: map[string]interface{}{
"string": "string",
"bool": true,
"float": 1.23,
"int": 1,
},
Default: false,
},
want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"string":"string","bool":true,"float":1.23,"int":1},"default":false}`,
wantErr: assert.NoError,
},
{
name: "Should return a metadata field if metadata is not empty",
featureEvent: &exporter.FeatureEvent{
Kind: "feature",
ContextKind: "anonymousUser",
UserKey: "ABCD",
CreationDate: 1617970547,
Key: "random-key",
Variation: "Default",
Value: map[string]interface{}{
"string": "string",
"bool": true,
"float": 1.23,
"int": 1,
},
Default: false,
Metadata: map[string]interface{}{
"metadata1": "metadata1",
"metadata2": 24,
"metadata3": true,
},
},
want: `{"kind":"feature","contextKind":"anonymousUser","userKey":"ABCD","creationDate":1617970547,"key":"random-key","variation":"Default","value":{"string":"string","bool":true,"float":1.23,"int":1},"default":false,"metadata":{"metadata1":"metadata1","metadata2":24,"metadata3":true}}`,
wantErr: assert.NoError,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := json.Marshal(tt.featureEvent)
tt.wantErr(t, err)
if err != nil {
assert.JSONEq(t, tt.want, string(got))
}
})
}
}
12 changes: 7 additions & 5 deletions exporter/fileexporter/exporter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func TestFile_Export(t *testing.T) {
featureEvents: []exporter.FeatureEvent{
{
Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key",
Variation: "Default", Value: "YO", Default: false, Source: "SERVER",
Variation: "Default", Value: "YO", Default: false, Source: "SERVER", Metadata: map[string]interface{}{"test": "test"},
},
{
Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key",
Expand All @@ -120,11 +120,11 @@ func TestFile_Export(t *testing.T) {
featureEvents: []exporter.FeatureEvent{
{
Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key",
Variation: "Default", Value: `"YO"`, Default: false, Source: "SERVER",
Variation: "Default", Value: `"YO"`, Default: false, Source: "SERVER", Metadata: map[string]interface{}{"test": "test"},
},
{
Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key",
Variation: "Default", Value: `"YO2"`, Default: false, Version: "127", Source: "SERVER",
Variation: "Default", Value: `"YO2"`, Default: false, Version: "127", Source: "SERVER", Metadata: map[string]interface{}{},
},
},
},
Expand Down Expand Up @@ -175,8 +175,9 @@ func TestFile_Export(t *testing.T) {
"float": 1.23,
"int": 1,
},
Default: false,
Source: "SERVER",
Default: false,
Source: "SERVER",
Metadata: map[string]interface{}{"test": "test"},
},
},
},
Expand All @@ -193,6 +194,7 @@ func TestFile_Export(t *testing.T) {
Value: `{"bool":true,"float":1.23,"int":1,"string":"string"}`,
Default: false,
Source: "SERVER",
Metadata: map[string]interface{}{"test": "test"},
},
},
},
Expand Down
1 change: 0 additions & 1 deletion website/docs/integrations/export-evaluation-data/file.mdx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ exporter:
|---------------------------|:----------------:|--------|------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `kind` | <Mandatory /> | string | **none** | **Value should be `file`**.<br/>_This field is mandatory and describes which retriever you are using._ |
| `outputDir` | <Mandatory /> | string | **none** | OutputDir is the location of the directory where to store the exported files. |
| `path` | <NotMandatory /> | string | **bucket root level** | The location of the directory in Google Cloud Storage. |
| `flushInterval` | <NotMandatory /> | int | `60000` | The interval in millisecond between 2 calls to the webhook _(if the `maxEventInMemory` is reached before the flushInterval we will call the exporter before)_. |
| `maxEventInMemory` | <NotMandatory /> | int | `100000` | If we hit that limit we will call the exporter. |
| `format` | <NotMandatory /> | string | `JSON` | Format is the output format you want in your exported file. Available format: `JSON`, `CSV`, `Parquet`. |
Expand Down
Loading

0 comments on commit f8ba0fd

Please sign in to comment.