From 200dbad7dfab84fdd41fed1b4f5f815e1e9a8731 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Mon, 6 Jan 2025 15:21:08 +0000 Subject: [PATCH 1/8] Add four Murmur3 hash converters to convert the target to hash values - Murmur3Hash: Murmur3 32-bit hash represented as a signed integer - Murmur3Hash128: Murmur3 128-bit hash represented as two signed integers - Murmur3Hex: hexadecimal string in little-endian of the 32-bit Murmur3 hash - Murmur3Hex128: hexadecimal string in little-endian of the 128-bit Murmur3 hash --- connector/countconnector/go.mod | 1 + connector/countconnector/go.sum | 2 + connector/datadogconnector/go.mod | 1 + connector/datadogconnector/go.sum | 2 + connector/routingconnector/go.mod | 1 + connector/routingconnector/go.sum | 2 + connector/signaltometricsconnector/go.mod | 1 + connector/signaltometricsconnector/go.sum | 2 + connector/sumconnector/go.mod | 1 + connector/sumconnector/go.sum | 2 + exporter/datadogexporter/go.mod | 1 + exporter/datadogexporter/go.sum | 2 + .../datadogexporter/integrationtest/go.mod | 1 + .../datadogexporter/integrationtest/go.sum | 2 + .../integrationtest/go.mod | 1 + .../integrationtest/go.sum | 2 + exporter/honeycombmarkerexporter/go.mod | 1 + exporter/honeycombmarkerexporter/go.sum | 2 + internal/filter/go.mod | 1 + internal/filter/go.sum | 2 + pkg/ottl/e2e/e2e_test.go | 26 ++++ pkg/ottl/go.mod | 1 + pkg/ottl/go.sum | 2 + pkg/ottl/ottlfuncs/func_murmur3.go | 88 ++++++++++++ pkg/ottl/ottlfuncs/func_murmur3_test.go | 133 ++++++++++++++++++ pkg/ottl/ottlfuncs/functions.go | 4 + processor/attributesprocessor/go.mod | 1 + processor/attributesprocessor/go.sum | 2 + processor/filterprocessor/go.mod | 1 + processor/filterprocessor/go.sum | 2 + processor/logdedupprocessor/go.mod | 1 + processor/logdedupprocessor/go.sum | 2 + processor/routingprocessor/go.mod | 1 + processor/routingprocessor/go.sum | 2 + processor/spanprocessor/go.mod | 1 + processor/spanprocessor/go.sum | 2 + processor/tailsamplingprocessor/go.mod | 1 + processor/tailsamplingprocessor/go.sum | 2 + processor/transformprocessor/go.mod | 1 + processor/transformprocessor/go.sum | 2 + testbed/go.mod | 1 + testbed/go.sum | 2 + 42 files changed, 308 insertions(+) create mode 100644 pkg/ottl/ottlfuncs/func_murmur3.go create mode 100644 pkg/ottl/ottlfuncs/func_murmur3_test.go diff --git a/connector/countconnector/go.mod b/connector/countconnector/go.mod index 4b3c2841079d..d2584e205537 100644 --- a/connector/countconnector/go.mod +++ b/connector/countconnector/go.mod @@ -52,6 +52,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.116.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/connector/xconnector v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/connector/countconnector/go.sum b/connector/countconnector/go.sum index 9a83f2229fe1..314cc939e1cb 100644 --- a/connector/countconnector/go.sum +++ b/connector/countconnector/go.sum @@ -80,6 +80,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/connector/datadogconnector/go.mod b/connector/datadogconnector/go.mod index 6ee8e77a49b4..f66c25a5e58e 100644 --- a/connector/datadogconnector/go.mod +++ b/connector/datadogconnector/go.mod @@ -225,6 +225,7 @@ require ( github.com/tinylib/msgp v1.2.4 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/connector/datadogconnector/go.sum b/connector/datadogconnector/go.sum index 6037c05eb2d8..3c9d8d28d98f 100644 --- a/connector/datadogconnector/go.sum +++ b/connector/datadogconnector/go.sum @@ -823,6 +823,8 @@ github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYg github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= diff --git a/connector/routingconnector/go.mod b/connector/routingconnector/go.mod index 7bc004279abd..58131a0fc546 100644 --- a/connector/routingconnector/go.mod +++ b/connector/routingconnector/go.mod @@ -51,6 +51,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.116.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.116.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/connector/xconnector v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/connector/routingconnector/go.sum b/connector/routingconnector/go.sum index 9276cf0263de..994526242fad 100644 --- a/connector/routingconnector/go.sum +++ b/connector/routingconnector/go.sum @@ -80,6 +80,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/connector/signaltometricsconnector/go.mod b/connector/signaltometricsconnector/go.mod index 36e64e81ff3d..5bdc92ada121 100644 --- a/connector/signaltometricsconnector/go.mod +++ b/connector/signaltometricsconnector/go.mod @@ -48,6 +48,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.116.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/connector/xconnector v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/connector/signaltometricsconnector/go.sum b/connector/signaltometricsconnector/go.sum index 2812e9fe307d..318013dd2f6c 100644 --- a/connector/signaltometricsconnector/go.sum +++ b/connector/signaltometricsconnector/go.sum @@ -78,6 +78,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/connector/sumconnector/go.mod b/connector/sumconnector/go.mod index a218b5a893e3..04a1f308ccc8 100644 --- a/connector/sumconnector/go.mod +++ b/connector/sumconnector/go.mod @@ -52,6 +52,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.116.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/connector/xconnector v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/connector/sumconnector/go.sum b/connector/sumconnector/go.sum index 9a83f2229fe1..314cc939e1cb 100644 --- a/connector/sumconnector/go.sum +++ b/connector/sumconnector/go.sum @@ -80,6 +80,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/exporter/datadogexporter/go.mod b/exporter/datadogexporter/go.mod index d0a662307da5..1fd17486b189 100644 --- a/exporter/datadogexporter/go.mod +++ b/exporter/datadogexporter/go.mod @@ -321,6 +321,7 @@ require ( github.com/tinylib/msgp v1.2.4 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/vultr/govultr/v2 v2.17.2 // indirect diff --git a/exporter/datadogexporter/go.sum b/exporter/datadogexporter/go.sum index e20ef6e6bbd4..fb04cfa543dc 100644 --- a/exporter/datadogexporter/go.sum +++ b/exporter/datadogexporter/go.sum @@ -975,6 +975,8 @@ github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8t github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= diff --git a/exporter/datadogexporter/integrationtest/go.mod b/exporter/datadogexporter/integrationtest/go.mod index 63f74077a320..97f1cd6ef88e 100644 --- a/exporter/datadogexporter/integrationtest/go.mod +++ b/exporter/datadogexporter/integrationtest/go.mod @@ -287,6 +287,7 @@ require ( github.com/stretchr/objx v0.5.2 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect github.com/vultr/govultr/v2 v2.17.2 // indirect github.com/x448/float16 v0.8.4 // indirect diff --git a/exporter/datadogexporter/integrationtest/go.sum b/exporter/datadogexporter/integrationtest/go.sum index 04554a5c53d8..656e40fcd4ff 100644 --- a/exporter/datadogexporter/integrationtest/go.sum +++ b/exporter/datadogexporter/integrationtest/go.sum @@ -959,6 +959,8 @@ github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8t github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= diff --git a/exporter/elasticsearchexporter/integrationtest/go.mod b/exporter/elasticsearchexporter/integrationtest/go.mod index d2e7135958f6..f9194410eba1 100644 --- a/exporter/elasticsearchexporter/integrationtest/go.mod +++ b/exporter/elasticsearchexporter/integrationtest/go.mod @@ -129,6 +129,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/tklauser/go-sysconf v0.3.13 // indirect github.com/tklauser/numcpus v0.7.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect diff --git a/exporter/elasticsearchexporter/integrationtest/go.sum b/exporter/elasticsearchexporter/integrationtest/go.sum index 48eec690740a..cb8efdf54d25 100644 --- a/exporter/elasticsearchexporter/integrationtest/go.sum +++ b/exporter/elasticsearchexporter/integrationtest/go.sum @@ -261,6 +261,8 @@ github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08 github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= diff --git a/exporter/honeycombmarkerexporter/go.mod b/exporter/honeycombmarkerexporter/go.mod index c1c6408d24f8..ba3410c75419 100644 --- a/exporter/honeycombmarkerexporter/go.mod +++ b/exporter/honeycombmarkerexporter/go.mod @@ -62,6 +62,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rs/cors v1.11.1 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/client v1.22.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/config/configauth v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/exporter/honeycombmarkerexporter/go.sum b/exporter/honeycombmarkerexporter/go.sum index 6a1b12bb2386..1d6fa6596097 100644 --- a/exporter/honeycombmarkerexporter/go.sum +++ b/exporter/honeycombmarkerexporter/go.sum @@ -94,6 +94,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/internal/filter/go.mod b/internal/filter/go.mod index 4439f8984ad2..d2728f2d0e1b 100644 --- a/internal/filter/go.mod +++ b/internal/filter/go.mod @@ -48,6 +48,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.116.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/pdata/pprofile v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/internal/filter/go.sum b/internal/filter/go.sum index 7d214439419c..0056f3337c22 100644 --- a/internal/filter/go.sum +++ b/internal/filter/go.sum @@ -84,6 +84,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/ottl/e2e/e2e_test.go b/pkg/ottl/e2e/e2e_test.go index bb1dd43b5282..7ba571eeaedf 100644 --- a/pkg/ottl/e2e/e2e_test.go +++ b/pkg/ottl/e2e/e2e_test.go @@ -594,6 +594,32 @@ func Test_e2e_converters(t *testing.T) { tCtx.GetLogRecord().Attributes().PutDouble("test", 60) }, }, + { + statement: `set(attributes["test"], Murmur3Hash128("Hello World"))`, + want: func(tCtx ottllog.TransformContext) { + s := tCtx.GetLogRecord().Attributes().PutEmptySlice("test") + s.AppendEmpty().SetInt(1901405986810282715) + s.AppendEmpty().SetInt(-8942425033498643417) + }, + }, + { + statement: `set(attributes["test"], Murmur3Hash("Hello World"))`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutInt("test", int64(427197390)) + }, + }, + { + statement: `set(attributes["test"], Murmur3Hex128("Hello World"))`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("test", "dbc2a0c1ab26631a27b4c09fcf1fe683") + }, + }, + { + statement: `set(attributes["test"], Murmur3Hex("Hello World"))`, + want: func(tCtx ottllog.TransformContext) { + tCtx.GetLogRecord().Attributes().PutStr("test", "ce837619") + }, + }, { statement: `set(attributes["test"], Nanoseconds(Duration("1ms")))`, want: func(tCtx ottllog.TransformContext) { diff --git a/pkg/ottl/go.mod b/pkg/ottl/go.mod index 744aafc8b085..451624b12106 100644 --- a/pkg/ottl/go.mod +++ b/pkg/ottl/go.mod @@ -14,6 +14,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.116.0 github.com/stretchr/testify v1.10.0 + github.com/twmb/murmur3 v1.1.8 github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 go.opentelemetry.io/collector/component v0.116.1-0.20241220212031-7c2639723f67 go.opentelemetry.io/collector/component/componenttest v0.116.1-0.20241220212031-7c2639723f67 diff --git a/pkg/ottl/go.sum b/pkg/ottl/go.sum index d2c73f6f4f30..f74cb07bcad0 100644 --- a/pkg/ottl/go.sum +++ b/pkg/ottl/go.sum @@ -66,6 +66,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/pkg/ottl/ottlfuncs/func_murmur3.go b/pkg/ottl/ottlfuncs/func_murmur3.go new file mode 100644 index 000000000000..52ad7cd38405 --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_murmur3.go @@ -0,0 +1,88 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottlfuncs" + +import ( + "context" + "encoding/binary" + "encoding/hex" + "fmt" + + "github.com/twmb/murmur3" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +type murmur3Variant int + +const ( + Murmur3Sum32 murmur3Variant = iota + Murmur3Sum128 + Murmur3Hex32 + Murmur3Hex128 +) + +type Murmur3Arguments[K any] struct { + Target ottl.StringGetter[K] +} + +func NewMurmur3HashFactory[K any]() ottl.Factory[K] { + return NewMurmur3Factory[K]("Murmur3Hash", Murmur3Sum32) +} + +func NewMurmur3Hash128Factory[K any]() ottl.Factory[K] { + return NewMurmur3Factory[K]("Murmur3Hash128", Murmur3Sum128) +} + +func NewMurmur3HexFactory[K any]() ottl.Factory[K] { + return NewMurmur3Factory[K]("Murmur3Hex", Murmur3Hex32) +} + +func NewMurmur3Hex128Factory[K any]() ottl.Factory[K] { + return NewMurmur3Factory[K]("Murmur3Hex128", Murmur3Hex128) +} + +func NewMurmur3Factory[K any](name string, variant murmur3Variant) ottl.Factory[K] { + return ottl.NewFactory(name, &Murmur3Arguments[K]{}, func(_ ottl.FunctionContext, oArgs ottl.Arguments) (ottl.ExprFunc[K], error) { + args, ok := oArgs.(*Murmur3Arguments[K]) + if !ok { + return nil, fmt.Errorf("%s args must be of type *Murmur3Arguments[K]", name) + } + + return createMurmur3Function[K](args, variant) + }) +} + +func createMurmur3Function[K any](args *Murmur3Arguments[K], variant murmur3Variant) (ottl.ExprFunc[K], error) { + return func(ctx context.Context, tCtx K) (any, error) { + val, err := args.Target.Get(ctx, tCtx) + if err != nil { + return nil, err + } + + switch variant { + case Murmur3Sum32: + h := murmur3.Sum32([]byte(val)) + return int64(h), nil + case Murmur3Sum128: + h1, h2 := murmur3.Sum128([]byte(val)) + return []int64{int64(h1), int64(h2)}, nil + // MurmurHash3 is sensitive to endianness + // Hex returns the hexadecimal representation of the hash in little-endian + case Murmur3Hex32: + h := murmur3.Sum32([]byte(val)) + b := make([]byte, 4) + binary.LittleEndian.PutUint32(b, h) + return hex.EncodeToString(b), nil + case Murmur3Hex128: + h1, h2 := murmur3.Sum128([]byte(val)) + b := make([]byte, 16) + binary.LittleEndian.PutUint64(b[:8], h1) + binary.LittleEndian.PutUint64(b[8:], h2) + return hex.EncodeToString(b), nil + default: + return nil, fmt.Errorf("unknown murmur3 variant: %d", variant) + } + }, nil +} diff --git a/pkg/ottl/ottlfuncs/func_murmur3_test.go b/pkg/ottl/ottlfuncs/func_murmur3_test.go new file mode 100644 index 000000000000..e4cc364b8691 --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_murmur3_test.go @@ -0,0 +1,133 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ottlfuncs + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +func Test_Murmur3(t *testing.T) { + tests := []struct { + name string + oArgs *Murmur3Arguments[any] + variant murmur3Variant + expected any + funcError string + }{ + { + name: "string in Murmur3Hash", + oArgs: &Murmur3Arguments[any]{ + Target: ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "Hello World", nil + }, + }, + }, + variant: Murmur3Sum32, + expected: int64(427197390), + }, + { + name: "string in Murmur3Hash128", + oArgs: &Murmur3Arguments[any]{ + Target: ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "Hello World", nil + }, + }, + }, + variant: Murmur3Sum128, + expected: []int64{int64(1901405986810282715), int64(-8942425033498643417)}, + }, + { + name: "empty string", + oArgs: &Murmur3Arguments[any]{ + Target: ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "", nil + }, + }, + }, + variant: Murmur3Hex128, + expected: "00000000000000000000000000000000", + }, + { + name: "string in Murmur3Hex128", + oArgs: &Murmur3Arguments[any]{ + Target: ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "Hello World", nil + }, + }, + }, + variant: Murmur3Hex128, + expected: "dbc2a0c1ab26631a27b4c09fcf1fe683", + }, + { + name: "string in Murmur3Hex", + oArgs: &Murmur3Arguments[any]{ + Target: ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "Hello World", nil + }, + }, + }, + variant: Murmur3Hex32, + expected: "ce837619", + }, + { + name: "invalid variant", + oArgs: &Murmur3Arguments[any]{ + Target: ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "Hello World", nil + }, + }, + }, + variant: 66, + funcError: "unknown murmur3 variant: 66", + }, + { + name: "non-string", + oArgs: &Murmur3Arguments[any]{ + Target: ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return 10, nil + }, + }, + }, + variant: Murmur3Sum32, + funcError: "expected string but got int", + }, + { + name: "nil", + oArgs: &Murmur3Arguments[any]{ + Target: ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return nil, nil + }, + }, + }, + variant: Murmur3Sum32, + funcError: "expected string but got nil", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + exprFunc, err := createMurmur3Function[any](tt.oArgs, tt.variant) + assert.NoError(t, err) + result, err := exprFunc(context.Background(), nil) + if tt.funcError != "" { + require.ErrorContains(t, err, tt.funcError) + return + } + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/pkg/ottl/ottlfuncs/functions.go b/pkg/ottl/ottlfuncs/functions.go index efd8480fc691..7e05e4b168d0 100644 --- a/pkg/ottl/ottlfuncs/functions.go +++ b/pkg/ottl/ottlfuncs/functions.go @@ -68,6 +68,10 @@ func converters[K any]() []ottl.Factory[K] { NewMinuteFactory[K](), NewMinutesFactory[K](), NewMonthFactory[K](), + NewMurmur3HashFactory[K](), + NewMurmur3Hash128Factory[K](), + NewMurmur3HexFactory[K](), + NewMurmur3Hex128Factory[K](), NewNanosecondsFactory[K](), NewNowFactory[K](), NewParseCSVFactory[K](), diff --git a/processor/attributesprocessor/go.mod b/processor/attributesprocessor/go.mod index 2c7f178138bc..cf6e76ad7bcb 100644 --- a/processor/attributesprocessor/go.mod +++ b/processor/attributesprocessor/go.mod @@ -54,6 +54,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.116.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/client v1.22.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/component/componentstatus v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/processor/attributesprocessor/go.sum b/processor/attributesprocessor/go.sum index aefc1a04bdd5..0f503d8f3fb4 100644 --- a/processor/attributesprocessor/go.sum +++ b/processor/attributesprocessor/go.sum @@ -84,6 +84,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/processor/filterprocessor/go.mod b/processor/filterprocessor/go.mod index 8d3a2c0232dd..8fdb73f71eb4 100644 --- a/processor/filterprocessor/go.mod +++ b/processor/filterprocessor/go.mod @@ -59,6 +59,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.116.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/component/componentstatus v0.116.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/consumer/xconsumer v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/processor/filterprocessor/go.sum b/processor/filterprocessor/go.sum index 9192dc48b4ec..a52634639698 100644 --- a/processor/filterprocessor/go.sum +++ b/processor/filterprocessor/go.sum @@ -84,6 +84,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/processor/logdedupprocessor/go.mod b/processor/logdedupprocessor/go.mod index a157e20d9c65..5c0e8ccf38fb 100644 --- a/processor/logdedupprocessor/go.mod +++ b/processor/logdedupprocessor/go.mod @@ -59,6 +59,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/component/componentstatus v0.116.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/consumer/xconsumer v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/processor/logdedupprocessor/go.sum b/processor/logdedupprocessor/go.sum index 3d66631d748e..8cba76543c87 100644 --- a/processor/logdedupprocessor/go.sum +++ b/processor/logdedupprocessor/go.sum @@ -80,6 +80,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/processor/routingprocessor/go.mod b/processor/routingprocessor/go.mod index def9585dbccb..b4e37347c399 100644 --- a/processor/routingprocessor/go.mod +++ b/processor/routingprocessor/go.mod @@ -64,6 +64,7 @@ require ( github.com/mostynb/go-grpc-compression v1.2.3 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.116.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/component/componentstatus v0.116.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/config/configauth v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/processor/routingprocessor/go.sum b/processor/routingprocessor/go.sum index db25ddba6ec4..890df1d361ff 100644 --- a/processor/routingprocessor/go.sum +++ b/processor/routingprocessor/go.sum @@ -88,6 +88,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/processor/spanprocessor/go.mod b/processor/spanprocessor/go.mod index e117ed5d2856..5adf270c47eb 100644 --- a/processor/spanprocessor/go.mod +++ b/processor/spanprocessor/go.mod @@ -53,6 +53,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.116.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/component/componentstatus v0.116.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/processor/spanprocessor/go.sum b/processor/spanprocessor/go.sum index 3c632bd92bd7..6b9e2158b928 100644 --- a/processor/spanprocessor/go.sum +++ b/processor/spanprocessor/go.sum @@ -82,6 +82,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/processor/tailsamplingprocessor/go.mod b/processor/tailsamplingprocessor/go.mod index f5c4bc5fa058..982b9748ddd0 100644 --- a/processor/tailsamplingprocessor/go.mod +++ b/processor/tailsamplingprocessor/go.mod @@ -58,6 +58,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/component/componentstatus v0.116.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/consumer/xconsumer v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/processor/tailsamplingprocessor/go.sum b/processor/tailsamplingprocessor/go.sum index c27f55a33d10..eacdcb698ad4 100644 --- a/processor/tailsamplingprocessor/go.sum +++ b/processor/tailsamplingprocessor/go.sum @@ -80,6 +80,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/processor/transformprocessor/go.mod b/processor/transformprocessor/go.mod index ada8210edc27..5b5c5892957d 100644 --- a/processor/transformprocessor/go.mod +++ b/processor/transformprocessor/go.mod @@ -63,6 +63,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.116.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect go.opentelemetry.io/collector/component/componentstatus v0.116.1-0.20241220212031-7c2639723f67 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.116.1-0.20241220212031-7c2639723f67 // indirect diff --git a/processor/transformprocessor/go.sum b/processor/transformprocessor/go.sum index 3c632bd92bd7..6b9e2158b928 100644 --- a/processor/transformprocessor/go.sum +++ b/processor/transformprocessor/go.sum @@ -82,6 +82,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/testbed/go.mod b/testbed/go.mod index 237d2bfb184f..6eee82c406d9 100644 --- a/testbed/go.mod +++ b/testbed/go.mod @@ -257,6 +257,7 @@ require ( github.com/tinylib/msgp v1.2.5 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect + github.com/twmb/murmur3 v1.1.8 // indirect github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 // indirect github.com/valyala/fastjson v1.6.4 // indirect github.com/vultr/govultr/v2 v2.17.2 // indirect diff --git a/testbed/go.sum b/testbed/go.sum index 5ff0cf78736d..2e0b86149c2d 100644 --- a/testbed/go.sum +++ b/testbed/go.sum @@ -716,6 +716,8 @@ github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0h github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg= +github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6 h1:SIKIoA4e/5Y9ZOl0DCe3eVMLPOQzJxgZpfdHHeauNTM= github.com/ua-parser/uap-go v0.0.0-20240611065828-3a4781585db6/go.mod h1:BUbeWZiieNxAuuADTBNb3/aeje6on3DhU3rpWsQSB1E= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= From 282d133f1121c724973b2f20e7831f868328ad03 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Mon, 6 Jan 2025 15:54:16 +0000 Subject: [PATCH 2/8] change log --- .chloggen/ottl_murmur3_func.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .chloggen/ottl_murmur3_func.yaml diff --git a/.chloggen/ottl_murmur3_func.yaml b/.chloggen/ottl_murmur3_func.yaml new file mode 100644 index 000000000000..8585d70568f3 --- /dev/null +++ b/.chloggen/ottl_murmur3_func.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add `Murmur3Hash`, `Murmur3Hash128`, `Murmur3Hex` and `Murmur3Hex128` functions to convert the `target` to Murmur3 hash in signed integer or hexadecimal string format + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [34077] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] From 90e26af7a942e56f1611fa8101646a4983217c24 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Mon, 6 Jan 2025 16:00:43 +0000 Subject: [PATCH 3/8] readme --- pkg/ottl/ottlfuncs/README.md | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 8e48eb3083b2..1845442a1b09 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -444,6 +444,10 @@ Available Converters: - [Minute](#minute) - [Minutes](#minutes) - [Month](#month) +- [Murmur3Hash](#murmur3hash) +- [Murmur3Hash128](#murmur3hash128) +- [Murmur3Hex](#murmur3hex) +- [Murmur3Hex128](#murmur3hex128) - [Nanoseconds](#nanoseconds) - [Now](#now) - [ParseCSV](#parsecsv) @@ -1237,6 +1241,62 @@ Examples: - `Month(Now())` +### Murmur3Hash + +`Murmur3Hash(target)` + +The `Murmur3Hash` Converter converts the `target` to a Murmur3 32-bit hash represented as a signed integer. + +`target` is a Getter that returns a string. + +The returned type is `int64`. + +Examples: + +- `Murmur3Hash(attributes["order.productId"])` + +### Murmur3Hash128 + +`Murmur3Hash128(target)` + +The `Murmur3Hash128` Converter converts the `target` to a Murmur3 128-bit hash represented as two signed integers. + +`target` is a Getter that returns a string. + +The returned type is a slice of two `int64`. + +Examples: + +- `Murmur3Hash128(attributes["order.productId"])` + +### Murmur3Hex + +`Murmur3Hex(target)` + +The `Murmur3Hex` Converter converts the `target` to a hexadecimal string in little-endian of the 32-bit Murmur3 hash. + +`target` is a Getter that returns a string. + +The returned type is `string`. + +Examples: + +- `Murmur3Hex(attributes["order.productId"])` + +### Murmur3Hex128 + +`Murmur3Hex128(target)` + +The `Murmur3Hex128` Converter converts the `target` to a hexadecimal string in little-endian of the 128-bit Murmur3 hash. + +`target` is a Getter that returns a string. + +The returned type is `string`. + +Examples: + +- `Murmur3Hex128(attributes["order.productId"])` + ### Nanoseconds `Nanoseconds(value)` From 30ffd7a433af81ca537f543abeab7c7c5dbabd9e Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Mon, 6 Jan 2025 19:20:54 +0000 Subject: [PATCH 4/8] lint --- pkg/ottl/ottlfuncs/func_murmur3.go | 14 +++++++------- pkg/ottl/ottlfuncs/func_murmur3_test.go | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/ottl/ottlfuncs/func_murmur3.go b/pkg/ottl/ottlfuncs/func_murmur3.go index 52ad7cd38405..458466931599 100644 --- a/pkg/ottl/ottlfuncs/func_murmur3.go +++ b/pkg/ottl/ottlfuncs/func_murmur3.go @@ -45,16 +45,16 @@ func NewMurmur3Hex128Factory[K any]() ottl.Factory[K] { func NewMurmur3Factory[K any](name string, variant murmur3Variant) ottl.Factory[K] { return ottl.NewFactory(name, &Murmur3Arguments[K]{}, func(_ ottl.FunctionContext, oArgs ottl.Arguments) (ottl.ExprFunc[K], error) { - args, ok := oArgs.(*Murmur3Arguments[K]) - if !ok { - return nil, fmt.Errorf("%s args must be of type *Murmur3Arguments[K]", name) - } - - return createMurmur3Function[K](args, variant) + return createMurmur3Function[K](name, oArgs, variant) }) } -func createMurmur3Function[K any](args *Murmur3Arguments[K], variant murmur3Variant) (ottl.ExprFunc[K], error) { +func createMurmur3Function[K any](name string, oArgs ottl.Arguments, variant murmur3Variant) (ottl.ExprFunc[K], error) { + args, ok := oArgs.(*Murmur3Arguments[K]) + if !ok { + return nil, fmt.Errorf("%s args must be of type *Murmur3Arguments[K]", name) + } + return func(ctx context.Context, tCtx K) (any, error) { val, err := args.Target.Get(ctx, tCtx) if err != nil { diff --git a/pkg/ottl/ottlfuncs/func_murmur3_test.go b/pkg/ottl/ottlfuncs/func_murmur3_test.go index e4cc364b8691..ca0e4d01516e 100644 --- a/pkg/ottl/ottlfuncs/func_murmur3_test.go +++ b/pkg/ottl/ottlfuncs/func_murmur3_test.go @@ -120,7 +120,7 @@ func Test_Murmur3(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - exprFunc, err := createMurmur3Function[any](tt.oArgs, tt.variant) + exprFunc, err := createMurmur3Function[any]("Murmur3Factory", tt.oArgs, tt.variant) assert.NoError(t, err) result, err := exprFunc(context.Background(), nil) if tt.funcError != "" { From c3a81ba48ac59ad71e9e393115c38624cfc5125d Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Mon, 6 Jan 2025 19:25:46 +0000 Subject: [PATCH 5/8] rename enum --- pkg/ottl/ottlfuncs/func_murmur3.go | 45 ++++++++++++++++--------- pkg/ottl/ottlfuncs/func_murmur3_test.go | 12 +++---- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/pkg/ottl/ottlfuncs/func_murmur3.go b/pkg/ottl/ottlfuncs/func_murmur3.go index 458466931599..3e68ac0fada8 100644 --- a/pkg/ottl/ottlfuncs/func_murmur3.go +++ b/pkg/ottl/ottlfuncs/func_murmur3.go @@ -17,42 +17,57 @@ import ( type murmur3Variant int const ( - Murmur3Sum32 murmur3Variant = iota - Murmur3Sum128 - Murmur3Hex32 + Murmur3Hash murmur3Variant = iota + Murmur3Hash128 + Murmur3Hex Murmur3Hex128 ) +func (v murmur3Variant) String() string { + switch v { + case Murmur3Hash: + return "Murmur3Hash" + case Murmur3Hash128: + return "Murmur3Hash128" + case Murmur3Hex: + return "Murmur3Hex" + case Murmur3Hex128: + return "Murmur3Hex128" + default: + return fmt.Sprintf("Unknown(%d) Murmur3 variant", v) + } +} + type Murmur3Arguments[K any] struct { Target ottl.StringGetter[K] } func NewMurmur3HashFactory[K any]() ottl.Factory[K] { - return NewMurmur3Factory[K]("Murmur3Hash", Murmur3Sum32) + return NewMurmur3Factory[K](Murmur3Hash) } func NewMurmur3Hash128Factory[K any]() ottl.Factory[K] { - return NewMurmur3Factory[K]("Murmur3Hash128", Murmur3Sum128) + return NewMurmur3Factory[K](Murmur3Hash128) } func NewMurmur3HexFactory[K any]() ottl.Factory[K] { - return NewMurmur3Factory[K]("Murmur3Hex", Murmur3Hex32) + return NewMurmur3Factory[K](Murmur3Hex) } func NewMurmur3Hex128Factory[K any]() ottl.Factory[K] { - return NewMurmur3Factory[K]("Murmur3Hex128", Murmur3Hex128) + return NewMurmur3Factory[K](Murmur3Hex128) } -func NewMurmur3Factory[K any](name string, variant murmur3Variant) ottl.Factory[K] { - return ottl.NewFactory(name, &Murmur3Arguments[K]{}, func(_ ottl.FunctionContext, oArgs ottl.Arguments) (ottl.ExprFunc[K], error) { - return createMurmur3Function[K](name, oArgs, variant) +func NewMurmur3Factory[K any](variant murmur3Variant) ottl.Factory[K] { + return ottl.NewFactory(variant.String(), &Murmur3Arguments[K]{}, func(_ ottl.FunctionContext, oArgs ottl.Arguments) (ottl.ExprFunc[K], error) { + return createMurmur3Function[K](oArgs, variant) }) } -func createMurmur3Function[K any](name string, oArgs ottl.Arguments, variant murmur3Variant) (ottl.ExprFunc[K], error) { +func createMurmur3Function[K any](oArgs ottl.Arguments, variant murmur3Variant) (ottl.ExprFunc[K], error) { args, ok := oArgs.(*Murmur3Arguments[K]) if !ok { - return nil, fmt.Errorf("%s args must be of type *Murmur3Arguments[K]", name) + return nil, fmt.Errorf("%s args must be of type *Murmur3Arguments[K]", variant.String()) } return func(ctx context.Context, tCtx K) (any, error) { @@ -62,15 +77,15 @@ func createMurmur3Function[K any](name string, oArgs ottl.Arguments, variant mur } switch variant { - case Murmur3Sum32: + case Murmur3Hash: h := murmur3.Sum32([]byte(val)) return int64(h), nil - case Murmur3Sum128: + case Murmur3Hash128: h1, h2 := murmur3.Sum128([]byte(val)) return []int64{int64(h1), int64(h2)}, nil // MurmurHash3 is sensitive to endianness // Hex returns the hexadecimal representation of the hash in little-endian - case Murmur3Hex32: + case Murmur3Hex: h := murmur3.Sum32([]byte(val)) b := make([]byte, 4) binary.LittleEndian.PutUint32(b, h) diff --git a/pkg/ottl/ottlfuncs/func_murmur3_test.go b/pkg/ottl/ottlfuncs/func_murmur3_test.go index ca0e4d01516e..c9543c78c835 100644 --- a/pkg/ottl/ottlfuncs/func_murmur3_test.go +++ b/pkg/ottl/ottlfuncs/func_murmur3_test.go @@ -30,7 +30,7 @@ func Test_Murmur3(t *testing.T) { }, }, }, - variant: Murmur3Sum32, + variant: Murmur3Hash, expected: int64(427197390), }, { @@ -42,7 +42,7 @@ func Test_Murmur3(t *testing.T) { }, }, }, - variant: Murmur3Sum128, + variant: Murmur3Hash128, expected: []int64{int64(1901405986810282715), int64(-8942425033498643417)}, }, { @@ -78,7 +78,7 @@ func Test_Murmur3(t *testing.T) { }, }, }, - variant: Murmur3Hex32, + variant: Murmur3Hex, expected: "ce837619", }, { @@ -102,7 +102,7 @@ func Test_Murmur3(t *testing.T) { }, }, }, - variant: Murmur3Sum32, + variant: Murmur3Hash, funcError: "expected string but got int", }, { @@ -114,13 +114,13 @@ func Test_Murmur3(t *testing.T) { }, }, }, - variant: Murmur3Sum32, + variant: Murmur3Hash, funcError: "expected string but got nil", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - exprFunc, err := createMurmur3Function[any]("Murmur3Factory", tt.oArgs, tt.variant) + exprFunc, err := createMurmur3Function[any](tt.oArgs, tt.variant) assert.NoError(t, err) result, err := exprFunc(context.Background(), nil) if tt.funcError != "" { From 8660fc960a002d94293a81fc429b47ff7eecc0f4 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Wed, 8 Jan 2025 22:39:25 +0000 Subject: [PATCH 6/8] change Factory to private --- pkg/ottl/ottlfuncs/func_murmur3.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/ottl/ottlfuncs/func_murmur3.go b/pkg/ottl/ottlfuncs/func_murmur3.go index 3e68ac0fada8..bf66228f5b26 100644 --- a/pkg/ottl/ottlfuncs/func_murmur3.go +++ b/pkg/ottl/ottlfuncs/func_murmur3.go @@ -43,22 +43,22 @@ type Murmur3Arguments[K any] struct { } func NewMurmur3HashFactory[K any]() ottl.Factory[K] { - return NewMurmur3Factory[K](Murmur3Hash) + return newMurmur3Factory[K](Murmur3Hash) } func NewMurmur3Hash128Factory[K any]() ottl.Factory[K] { - return NewMurmur3Factory[K](Murmur3Hash128) + return newMurmur3Factory[K](Murmur3Hash128) } func NewMurmur3HexFactory[K any]() ottl.Factory[K] { - return NewMurmur3Factory[K](Murmur3Hex) + return newMurmur3Factory[K](Murmur3Hex) } func NewMurmur3Hex128Factory[K any]() ottl.Factory[K] { - return NewMurmur3Factory[K](Murmur3Hex128) + return newMurmur3Factory[K](Murmur3Hex128) } -func NewMurmur3Factory[K any](variant murmur3Variant) ottl.Factory[K] { +func newMurmur3Factory[K any](variant murmur3Variant) ottl.Factory[K] { return ottl.NewFactory(variant.String(), &Murmur3Arguments[K]{}, func(_ ottl.FunctionContext, oArgs ottl.Arguments) (ottl.ExprFunc[K], error) { return createMurmur3Function[K](oArgs, variant) }) @@ -67,7 +67,7 @@ func NewMurmur3Factory[K any](variant murmur3Variant) ottl.Factory[K] { func createMurmur3Function[K any](oArgs ottl.Arguments, variant murmur3Variant) (ottl.ExprFunc[K], error) { args, ok := oArgs.(*Murmur3Arguments[K]) if !ok { - return nil, fmt.Errorf("%s args must be of type *Murmur3Arguments[K]", variant.String()) + return nil, fmt.Errorf("%sFactory args must be of type *Murmur3Arguments[K]", variant.String()) } return func(ctx context.Context, tCtx K) (any, error) { From 5269a9ff4f5784f1a2f4aaf8c687571c3025cc11 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Wed, 8 Jan 2025 22:39:49 +0000 Subject: [PATCH 7/8] test coverage --- pkg/ottl/ottlfuncs/func_murmur3_test.go | 55 ++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 5 deletions(-) diff --git a/pkg/ottl/ottlfuncs/func_murmur3_test.go b/pkg/ottl/ottlfuncs/func_murmur3_test.go index c9543c78c835..c922d263f214 100644 --- a/pkg/ottl/ottlfuncs/func_murmur3_test.go +++ b/pkg/ottl/ottlfuncs/func_murmur3_test.go @@ -5,6 +5,8 @@ package ottlfuncs import ( "context" + "fmt" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -15,11 +17,12 @@ import ( func Test_Murmur3(t *testing.T) { tests := []struct { - name string - oArgs *Murmur3Arguments[any] - variant murmur3Variant - expected any - funcError string + name string + oArgs ottl.Arguments + variant murmur3Variant + expected any + funcError string + createError string }{ { name: "string in Murmur3Hash", @@ -117,11 +120,22 @@ func Test_Murmur3(t *testing.T) { variant: Murmur3Hash, funcError: "expected string but got nil", }, + { + name: "Invalid args", + oArgs: nil, + variant: Murmur3Hash, + createError: "Murmur3HashFactory args must be of type *Murmur3Arguments[K]", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { exprFunc, err := createMurmur3Function[any](tt.oArgs, tt.variant) + if tt.createError != "" { + require.ErrorContains(t, err, tt.createError) + return + } assert.NoError(t, err) + result, err := exprFunc(context.Background(), nil) if tt.funcError != "" { require.ErrorContains(t, err, tt.funcError) @@ -131,3 +145,34 @@ func Test_Murmur3(t *testing.T) { }) } } + +func TestCreateMurmur3Func(t *testing.T) { + factory := NewMurmur3Hash128Factory[any]() + fCtx := ottl.FunctionContext{} + + // invalid args + exprFunc, err := factory.CreateFunction(fCtx, nil) + assert.Error(t, err) + assert.Nil(t, exprFunc) + + // valid args + exprFunc, err = factory.CreateFunction( + fCtx, &Murmur3Arguments[any]{ + Target: ottl.StandardStringGetter[any]{ + Getter: func(_ context.Context, _ any) (any, error) { + return "Hello World", nil + }, + }, + }) + assert.NoError(t, err) + assert.NotNil(t, exprFunc) +} + +func TestMurmur3VariantUnknownCase(t *testing.T) { + var invalidVariant murmur3Variant = 9999 + result := invalidVariant.String() + expectedString := fmt.Sprintf("Unknown(%d)", invalidVariant) + if !strings.Contains(result, expectedString) { + t.Errorf("%q does not contain %q", result, expectedString) + } +} From ebd6722085928d373ddcb5061f403c41c0ad64bc Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Thu, 9 Jan 2025 11:16:10 +0000 Subject: [PATCH 8/8] readme --- pkg/ottl/ottlfuncs/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 1845442a1b09..4deb965b104a 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -1245,7 +1245,7 @@ Examples: `Murmur3Hash(target)` -The `Murmur3Hash` Converter converts the `target` to a Murmur3 32-bit hash represented as a signed integer. +The `Murmur3Hash` Converter converts the `target` to a signed integer of the 32-bit Murmur3 hash. `target` is a Getter that returns a string. @@ -1259,7 +1259,7 @@ Examples: `Murmur3Hash128(target)` -The `Murmur3Hash128` Converter converts the `target` to a Murmur3 128-bit hash represented as two signed integers. +The `Murmur3Hash128` Converter converts the `target` to two signed integers of the 128-bit Murmur3 hash. `target` is a Getter that returns a string.