From 0340cb9f949472eb44a59abdbd26b32bb582f79f Mon Sep 17 00:00:00 2001 From: christopherbrumm Date: Mon, 13 Jan 2025 16:07:18 +0100 Subject: [PATCH 1/5] fix: use `jsoniter` instead of `encoding/json` --- go.mod | 3 +++ go.sum | 5 +++++ merkle.go | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 1d32ecd..b3696f5 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( github.com/KYVENetwork/ksync v1.6.2 + github.com/json-iterator/go v1.1.12 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.1 @@ -18,6 +19,8 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect diff --git a/go.sum b/go.sum index fc25e83..6de88d0 100644 --- a/go.sum +++ b/go.sum @@ -254,6 +254,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -307,9 +309,12 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= diff --git a/merkle.go b/merkle.go index 9476637..cd05100 100644 --- a/merkle.go +++ b/merkle.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/json" "github.com/KYVENetwork/ksync/types" + jsoniter "github.com/json-iterator/go" ) type Integration interface { @@ -89,8 +90,7 @@ func createHashesForTendermintValue(dataItem types.DataItem) [32]byte { } func calculateSHA256Hash(obj interface{}) [32]byte { - // Serialize the object to JSON with keys sorted ascending by default - serializedObj, err := json.Marshal(obj) + serializedObj, err := jsoniter.Marshal(obj) if err != nil { panic(err) } From c428c9593b3950fde7a1e5867b682083844c0a94 Mon Sep 17 00:00:00 2001 From: christopherbrumm Date: Mon, 13 Jan 2025 16:11:26 +0100 Subject: [PATCH 2/5] fix: config.yaml --- config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/config.yaml b/config.yaml index 826679c..4e914b6 100644 --- a/config.yaml +++ b/config.yaml @@ -11,16 +11,16 @@ chain_rest: https://api.kyve.network # --- pools: - pool_id: 0 - target_bundle_id: 259772 + target_bundle_id: 259773 - pool_id: 1 - target_bundle_id: 165754 + target_bundle_id: 165755 - pool_id: 2 - target_bundle_id: 206232 + target_bundle_id: 206233 - pool_id: 3 - target_bundle_id: 196925 + target_bundle_id: 196926 - pool_id: 5 - target_bundle_id: 147373 + target_bundle_id: 147374 - pool_id: 7 - target_bundle_id: 92881 + target_bundle_id: 92882 - pool_id: 9 - target_bundle_id: 68399 \ No newline at end of file + target_bundle_id: 68400 \ No newline at end of file From 076f781b5f7c9263dea5dec11c8270634361f1c1 Mon Sep 17 00:00:00 2001 From: christopherbrumm Date: Mon, 13 Jan 2025 17:04:23 +0100 Subject: [PATCH 3/5] chore: remove unused dependencies --- go.mod | 3 --- merkle.go | 20 ++++---------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index b3696f5..1d32ecd 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.21 require ( github.com/KYVENetwork/ksync v1.6.2 - github.com/json-iterator/go v1.1.12 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.1 @@ -19,8 +18,6 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect diff --git a/merkle.go b/merkle.go index cd05100..fd8b46f 100644 --- a/merkle.go +++ b/merkle.go @@ -3,8 +3,8 @@ package merkle_script import ( "crypto/sha256" "encoding/json" + "fmt" "github.com/KYVENetwork/ksync/types" - jsoniter "github.com/json-iterator/go" ) type Integration interface { @@ -48,7 +48,7 @@ func BundleToHashes(bundle types.Bundle, runtime string) [][32]byte { switch runtime { case "@kyvejs/tendermint-bsync": for _, dataItem := range bundle { - leafHashes = append(leafHashes, calculateSHA256Hash(dataItem)) + leafHashes = append(leafHashes, sha256.Sum256([]byte(fmt.Sprintf("{\"key\":\"%s\",\"value\":%s}", dataItem.Key, string(dataItem.Value))))) } case "@kyvejs/tendermint": for _, dataItem := range bundle { @@ -83,20 +83,8 @@ func createHashesForTendermintValue(dataItem types.DataItem) [32]byte { var hashes [][32]byte - hashes = append(hashes, calculateSHA256Hash(tendermintValue.Block)) - hashes = append(hashes, calculateSHA256Hash(tendermintValue.BlockResults)) + hashes = append(hashes, sha256.Sum256(tendermintValue.Block)) + hashes = append(hashes, sha256.Sum256(tendermintValue.BlockResults)) return GenerateMerkleRoot(&hashes) } - -func calculateSHA256Hash(obj interface{}) [32]byte { - serializedObj, err := jsoniter.Marshal(obj) - if err != nil { - panic(err) - } - - // Calculate the SHA-256 hash - sha256Hash := sha256.Sum256(serializedObj) - - return sha256Hash -} From 3a5c4b3a4f1a20b9c459cec7b443f53e69e09ac0 Mon Sep 17 00:00:00 2001 From: christopherbrumm Date: Mon, 20 Jan 2025 16:28:44 +0100 Subject: [PATCH 4/5] chore: add `test` command --- cmd/merkle-script/commands/root.go | 2 + cmd/merkle-script/commands/test.go | 76 ++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 cmd/merkle-script/commands/test.go diff --git a/cmd/merkle-script/commands/root.go b/cmd/merkle-script/commands/root.go index 2f60307..c3344d8 100644 --- a/cmd/merkle-script/commands/root.go +++ b/cmd/merkle-script/commands/root.go @@ -7,6 +7,8 @@ import ( var ( config string + bundleId int64 + poolId int64 storageRest string ) diff --git a/cmd/merkle-script/commands/test.go b/cmd/merkle-script/commands/test.go new file mode 100644 index 0000000..5ad11b6 --- /dev/null +++ b/cmd/merkle-script/commands/test.go @@ -0,0 +1,76 @@ +package commands + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "github.com/KYVENetwork/ksync/collectors/bundles" + "github.com/KYVENetwork/ksync/types" + "github.com/KYVENetwork/ksync/utils" + m "github.com/KYVENetwork/merkle-script" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "strconv" + "strings" +) + +func init() { + testCmd.Flags().Int64Var(&bundleId, "bundle-id", 0, "bundle id") + + testCmd.Flags().Int64Var(&poolId, "pool-id", 0, "pool id") + + rootCmd.AddCommand(testCmd) +} + +var testCmd = &cobra.Command{ + Use: "test", + Short: "Start merkle-script to create historical Merkle roots", + Run: func(cmd *cobra.Command, args []string) { + viper.SetConfigName("config") + viper.SetConfigType("yaml") + viper.AddConfigPath(config) + err := viper.ReadInConfig() + if err != nil { + panic(fmt.Errorf("config file error: %w", err)) + } + + chainRest := utils.GetChainRest(viper.GetString("chain_id"), viper.GetString("chain_rest")) + storageRest = strings.TrimSuffix(storageRest, "/") + + bundlesPage, _, err := bundles.GetFinalizedBundlesPageWithOffset(chainRest, poolId, 1, bundleId, "", false) + if err != nil { + panic(err) + return + } + + deflated, err := bundles.GetDataFromFinalizedBundle(bundlesPage[0], storageRest) + if err != nil { + panic(err) + return + } + + var bundle types.Bundle + + if err := json.Unmarshal(deflated, &bundle); err != nil { + panic(err) + return + } + + bundleId, err := strconv.Atoi(bundlesPage[0].Id) + if err != nil { + panic(err) + return + } + + var leafHashes [][32]byte + + leafHashes = m.BundleToHashes(bundle, "@kyvejs/tendermint") + + merkleRoot := m.GenerateMerkleRoot(&leafHashes) + + logger.Info(). + Int("bundle-id", bundleId). + Str("root", hex.EncodeToString(merkleRoot[:])). + Msg("computed Merkle root") + }, +} From e748d2c2c3c4de5e7f1980edd458bb6c9e061d56 Mon Sep 17 00:00:00 2001 From: christopherbrumm Date: Mon, 20 Jan 2025 16:30:13 +0100 Subject: [PATCH 5/5] fix: Python implementation --- python/merkle_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/merkle_generator.py b/python/merkle_generator.py index c487126..67da80a 100644 --- a/python/merkle_generator.py +++ b/python/merkle_generator.py @@ -44,9 +44,9 @@ def tendermint_merkle_root(bundle_content): sha256(item["key"].encode('utf-8')).hexdigest(), merkle_root( [ - sha256(json.dumps(item["value"]["block"], separators=(',', ':')).encode('utf-8')).hexdigest(), + sha256(json.dumps(item["value"]["block"], separators=(',', ':'), ensure_ascii=False).encode('utf-8')).hexdigest(), sha256( - json.dumps(item["value"]["block_results"], separators=(',', ':')).encode('utf-8')).hexdigest() + json.dumps(item["value"]["block_results"], separators=(',', ':'), ensure_ascii=False).encode('utf-8')).hexdigest() ] ) ])