diff --git a/go.mod b/go.mod index 29c3d1fcb3..f6d7dfae83 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/Jeffail/gabs/v2 v2.7.0 github.com/Jeffail/grok v1.1.0 github.com/Jeffail/shutdown v1.0.0 + github.com/Masterminds/sprig/v3 v3.3.0 github.com/Masterminds/squirrel v1.5.4 github.com/OneOfOne/xxhash v1.2.8 github.com/PaesslerAG/gval v1.2.3 @@ -175,7 +176,9 @@ require ( cloud.google.com/go/pubsub/v2 v2.0.0 // indirect github.com/BurntSushi/toml v1.5.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.54.0 // indirect - github.com/apache/arrow-go/v18 v18.5.2 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/apache/arrow-go/v18 v18.5.1 // indirect github.com/apache/thrift v0.22.0 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect @@ -208,16 +211,20 @@ require ( github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect - github.com/hamba/avro/v2 v2.31.0 // indirect + github.com/hamba/avro/v2 v2.30.0 // indirect + github.com/huandu/xstrings v1.5.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/spf13/cast v1.7.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/theparanoids/crypki v1.20.9 // indirect @@ -258,7 +265,7 @@ require ( cloud.google.com/go/monitoring v1.24.3 // indirect cloud.google.com/go/spanner v1.86.1 cloud.google.com/go/trace v1.11.7 // indirect - dario.cat/mergo v1.0.0 // indirect + dario.cat/mergo v1.0.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect github.com/AthenZ/athenz v1.12.30 // indirect @@ -407,7 +414,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/schollz/progressbar/v2 v2.15.0 // indirect github.com/segmentio/asm v1.2.0 // indirect - github.com/shopspring/decimal v1.3.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect github.com/slack-go/slack v0.17.3 github.com/snowplow/snowplow-golang-analytics-sdk v0.4.0 github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 4a294be90e..1c64dcd695 100644 --- a/go.sum +++ b/go.sum @@ -638,8 +638,8 @@ cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13 h1:zkiIe8AxZ/k cuelabs.dev/go/oci/ociregistry v0.0.0-20231103182354-93e78c079a13/go.mod h1:XGKYSMtsJWfqQYPwq51ZygxAPqpEUj/9bdg16iDPTAA= cuelang.org/go v0.7.1 h1:wSuUSIKR9M1yrph57l8EJATWVRWHaq/Zd0dFUL10PC8= cuelang.org/go v0.7.1/go.mod h1:ix+3dM/bSpdG9xg6qpCgnJnpeLtciZu+O/rDbywoMII= -dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= -dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8= git.sr.ht/~sbinet/gg v0.3.1/go.mod h1:KGYtlADtqsqANL9ueOFkWymvzUvLMQllU5Ixo+8v3pc= @@ -719,9 +719,13 @@ github.com/Jeffail/grok v1.1.0/go.mod h1:dm0hLksrDwOMa6To7ORXCuLbuNtASIZTfYheavL github.com/Jeffail/shutdown v1.0.0 h1:afYjnY4pksqP/012m3NGJVccDI+WATdSzIMVHZKU8/Y= github.com/Jeffail/shutdown v1.0.0/go.mod h1:5dT4Y1oe60SJELCkmAB1pr9uQyHBhh6cwDLQTfmuO5U= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= +github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= @@ -751,8 +755,8 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/apache/arrow-go/v18 v18.5.2 h1:3uoHjoaEie5eVsxx/Bt64hKwZx4STb+beAkqKOlq/lY= -github.com/apache/arrow-go/v18 v18.5.2/go.mod h1:yNoizNTT4peTciJ7V01d2EgOkE1d0fQ1vZcFOsVtFsw= +github.com/apache/arrow-go/v18 v18.5.1 h1:yaQ6zxMGgf9YCYw4/oaeOU3AULySDlAYDOcnr4LdHdI= +github.com/apache/arrow-go/v18 v18.5.1/go.mod h1:OCCJsmdq8AsRm8FkBSSmYTwL/s4zHW9CqxeBxEytkNE= github.com/apache/arrow/go/v10 v10.0.1/go.mod h1:YvhnlEePVnBS4+0z3fhPfUy7W1Ikj0Ih0vcRo/gZ1M0= github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= @@ -1084,6 +1088,8 @@ github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/ github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= @@ -1349,8 +1355,8 @@ github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8 github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hamba/avro/v2 v2.31.0 h1:wv3nmua7lCEIwWsb6vqsTS3pXktTxcKg5eoyNu0VhrU= -github.com/hamba/avro/v2 v2.31.0/go.mod h1:t6lJYAGE5Mswfn17zjtyQsssRQgnqO6TXLBCHHWRqrw= +github.com/hamba/avro/v2 v2.30.0 h1:OaIdh0+dZIJ331FO/+YYBwZZRdGVyyHuRSyHsjZLJoA= +github.com/hamba/avro/v2 v2.30.0/go.mod h1:X6gDhYv6DQVAT56VqOKuW+PLnQrEQqGB9l1nhlMdAdQ= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v1.1.0 h1:QsGcniKx5/LuX2eYoeL+Np3UKYPNaN7YKpTh29h8rbw= @@ -1378,6 +1384,8 @@ github.com/hashicorp/raft v1.3.9 h1:9yuo1aR0bFTr1cw7pj3S2Bk6MhJCsnr2NAxvIBrP2x4= github.com/hashicorp/raft v1.3.9/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= +github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -1569,10 +1577,14 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= @@ -1787,8 +1799,9 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sijms/go-ora/v2 v2.8.22 h1:3ABgRzVKxS439cEgSLjFKutIwOyhnyi4oOSBywEdOlU= github.com/sijms/go-ora/v2 v2.8.22/go.mod h1:QgFInVi3ZWyqAiJwzBQA+nbKYKH77tdp1PYoCqhR2dU= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -1813,6 +1826,8 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= +github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= @@ -2454,8 +2469,8 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= gonum.org/v1/plot v0.9.0/go.mod h1:3Pcqqmp6RHvJI72kgb8fThyUnav364FOsdDo2aGW5lY= diff --git a/internal/impl/pure/processor_template.go b/internal/impl/pure/processor_template.go new file mode 100644 index 0000000000..e12075a80f --- /dev/null +++ b/internal/impl/pure/processor_template.go @@ -0,0 +1,125 @@ +package pure + +import ( + "bytes" + "context" + "errors" + "fmt" + "text/template" + + "github.com/Masterminds/sprig/v3" + "github.com/warpstreamlabs/bento/public/bloblang" + "github.com/warpstreamlabs/bento/public/service" +) + +const ( + tpFieldText = "text" + tpFieldFunctions = "functions" +) + +func TemplateProcessorSpec() *service.ConfigSpec { + return service.NewConfigSpec(). + Categories("Mapping"). + Beta(). + Summary("Transforms messages using Go template syntax."). + Description(`Transforms messages using Go template syntax. +Supports built-in Go template functions, +Sprig template functions (including string manipulation, math, date, and encoding utilities) +and a custom `+"`"+`meta`+"`"+` function to access message metadata (e.g., `+"`"+`{{ meta \"key\" }}`+"`"+`). +Additionally, users can define custom Bloblang-based functions via the `+"`"+`functions`+"`"+` field, which are available during template execution.`). + Fields( + service.NewStringField(tpFieldText). + Description("The Go template to apply to messages."). + Example("{{ .name }} - {{ meta \"source\" }}"). + Example("{{ range .items }}{{ .name }}: {{ .value }}{{ end }}"). + Default("{{ . }}"), + service.NewStringMapField(tpFieldFunctions). + Description("A map of Bloblang functions to make available to the template."). + Optional(), + ) +} + +func init() { + err := service.RegisterProcessor("template", TemplateProcessorSpec(), NewTemplateProcessorFromConfig) + if err != nil { + panic(err) + } +} + +type templateProc struct { + tmpl *template.Template +} + +func NewTemplateProcessorFromConfig(conf *service.ParsedConfig, mgr *service.Resources) (service.Processor, error) { + templateStr, err := conf.FieldString(tpFieldText) + if err != nil { + return nil, err + } + + var functions template.FuncMap + + if conf.Contains(tpFieldFunctions) { + functionsMap, err := conf.FieldStringMap(tpFieldFunctions) + if err != nil { + return nil, err + } + + functions = make(template.FuncMap, len(functionsMap)+1) + for name, fn := range functionsMap { + blob, err := bloblang.Parse(fn) + if err != nil { + return nil, err + } + if name == "meta" { + return nil, errors.New("can not define a function named meta") + } + functions[name] = func(msg any) (any, error) { + return blob.Query(msg) + } + } + functions["meta"] = func(name string) any { + return nil + } + } else { + functions = template.FuncMap{ + "meta": func(name string) any { + return nil + }, + } + } + + tmpl, err := template.New("template").Funcs(functions).Funcs(sprig.TxtFuncMap()).Parse(templateStr) + if err != nil { + return nil, fmt.Errorf("Failed to parse template: %w", err) + } + + return &templateProc{ + tmpl: tmpl, + }, nil +} + +func (t *templateProc) Process(ctx context.Context, msg *service.Message) (service.MessageBatch, error) { + obj, err := msg.AsStructured() + if err != nil { + return nil, fmt.Errorf("Failed to parse part into json: %v", err) + } + + var buf bytes.Buffer + if err := t.tmpl.Funcs(template.FuncMap{ + "meta": func(name string) any { + if val, exists := msg.MetaGetMut(name); exists { + return val + } + return nil + }, + }).Execute(&buf, obj); err != nil { + return nil, fmt.Errorf("Failed to execute template: %v", err) + } + + msg.SetBytes(buf.Bytes()) + return service.MessageBatch{msg}, nil +} + +func (t *templateProc) Close(context.Context) error { + return nil +} diff --git a/internal/impl/pure/processor_template_test.go b/internal/impl/pure/processor_template_test.go new file mode 100644 index 0000000000..bc42a49ad2 --- /dev/null +++ b/internal/impl/pure/processor_template_test.go @@ -0,0 +1,130 @@ +package pure_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/warpstreamlabs/bento/public/service" + + ipure "github.com/warpstreamlabs/bento/internal/impl/pure" +) + +func testTemplateProc(confStr string) (service.Processor, *service.Resources, error) { + pConf, err := ipure.TemplateProcessorSpec().ParseYAML(confStr, nil) + if err != nil { + return nil, nil, err + } + mgr := service.MockResources() + proc, err := ipure.NewTemplateProcessorFromConfig(pConf, mgr) + return proc, mgr, err +} + +func TestTemplateProcessor_Templating(t *testing.T) { + tests := []struct { + name string + template string + input []byte + metaKV map[string]string + expected string + expectedError bool + errorValue string + }{ + { + name: "basic template", + template: `text: "{{ .foo }} - {{ meta \"meta_foo\" }}"`, + input: []byte(`{"foo":"bar"}`), + metaKV: map[string]string{"meta_foo": "meta_bar"}, + expected: "bar - meta_bar", + }, + { + name: "range template", + template: `text: "{{ range .items }}{{ .name }}: {{ .value }}{{ end }}"`, + input: []byte(`{"items":[{"name":"foo","value":1},{"name":"bar","value":2}]}`), + expected: "foo: 1bar: 2", + }, + { + name: "meta access with values", + template: `text: "{{ meta \"key1\" }} - {{ meta \"key2\" }}"`, + input: []byte(`{}`), + metaKV: map[string]string{"key1": "value1", "key2": "value2"}, + expected: "value1 - value2", + }, + { + name: "meta access with nonexistent key", + template: `text: "{{ meta \"nonexistent\" }}"`, + input: []byte(`{}`), + expected: "", + }, + { + name: "field access with nonexistent key", + template: `text: "{{ .nonexistent }}"`, + input: []byte(`{}`), + expected: "", + }, + { + name: "invalid template syntax", + template: `text: "{{ invalid syntax"`, + expectedError: true, + errorValue: "Failed to parse template", + }, + { + name: "template functions", + template: ` +text: "{{ .foo }} - {{ uppercase .bar }}" +functions: + uppercase: | + root = this.uppercase() +`, + input: []byte(`{"foo":"hello","bar":"world"}`), + expected: "hello - WORLD", + }, + { + name: "template nonexistent functions", + template: `text: "{{ nonexistent_function .foo }}"`, + input: []byte(`{"foo":"hello","bar":"world"}`), + expectedError: true, + errorValue: "function \"nonexistent_function\" not defined", + }, + { + name: "sprig functions", + template: `text: "{{ $inc := .inc }}{{ $inc = add1 $inc }}{{$inc}}{{ $inc = add1 $inc }}{{$inc}}{{ $inc = add1 $inc }}{{$inc}}"`, + input: []byte(`{"inc":0}`), + expected: "123", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + proc, _, err := testTemplateProc(test.template) + if test.expectedError { + require.ErrorContains(t, err, test.errorValue) + return + } + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, proc.Close(t.Context())) }) + + msg := service.NewMessage(test.input) + for k, v := range test.metaKV { + msg.MetaSetMut(k, v) + } + + batch, err := proc.Process(t.Context(), msg) + require.NoError(t, err) + require.Len(t, batch, 1) + + msg = batch[0] + + for k, v := range test.metaKV { + m, ok := msg.MetaGetMut(k) + require.True(t, ok) + assert.Equal(t, m, v) + } + + result, err := batch[0].AsBytes() + require.NoError(t, err) + assert.Equal(t, test.expected, string(result)) + }) + } +} diff --git a/website/docs/components/processors/template.md b/website/docs/components/processors/template.md new file mode 100644 index 0000000000..6eff0fc995 --- /dev/null +++ b/website/docs/components/processors/template.md @@ -0,0 +1,62 @@ +--- +title: template +slug: template +type: processor +status: beta +categories: ["Mapping"] +--- + + + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +:::caution BETA +This component is mostly stable but breaking changes could still be made outside of major version releases if a fundamental problem with the component is found. +::: +Transforms messages using Go template syntax. + +```yml +# Config fields, showing default values +label: "" +template: + text: '{{ . }}' + functions: {} # No default (optional) +``` + +Transforms messages using Go template syntax. +Supports built-in Go template functions, +Sprig template functions (including string manipulation, math, date, and encoding utilities) +and a custom `meta` function to access message metadata (e.g., `{{ meta \"key\" }}`). +Additionally, users can define custom Bloblang-based functions via the `functions` field, which are available during template execution. + +## Fields + +### `text` + +The Go template to apply to messages. + + +Type: `string` +Default: `"{{ . }}"` + +```yml +# Examples + +text: '{{ .name }} - {{ meta "source" }}' + +text: '{{ range .items }}{{ .name }}: {{ .value }}{{ end }}' +``` + +### `functions` + +A map of Bloblang functions to make available to the template. + + +Type: `object` + +