diff --git a/cmd/merkle-script/commands/root.go b/cmd/merkle-script/commands/root.go index 2f60307..c3344d8 100644 --- a/cmd/merkle-script/commands/root.go +++ b/cmd/merkle-script/commands/root.go @@ -7,6 +7,8 @@ import ( var ( config string + bundleId int64 + poolId int64 storageRest string ) diff --git a/cmd/merkle-script/commands/test.go b/cmd/merkle-script/commands/test.go new file mode 100644 index 0000000..5ad11b6 --- /dev/null +++ b/cmd/merkle-script/commands/test.go @@ -0,0 +1,76 @@ +package commands + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "github.com/KYVENetwork/ksync/collectors/bundles" + "github.com/KYVENetwork/ksync/types" + "github.com/KYVENetwork/ksync/utils" + m "github.com/KYVENetwork/merkle-script" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "strconv" + "strings" +) + +func init() { + testCmd.Flags().Int64Var(&bundleId, "bundle-id", 0, "bundle id") + + testCmd.Flags().Int64Var(&poolId, "pool-id", 0, "pool id") + + rootCmd.AddCommand(testCmd) +} + +var testCmd = &cobra.Command{ + Use: "test", + Short: "Start merkle-script to create historical Merkle roots", + Run: func(cmd *cobra.Command, args []string) { + viper.SetConfigName("config") + viper.SetConfigType("yaml") + viper.AddConfigPath(config) + err := viper.ReadInConfig() + if err != nil { + panic(fmt.Errorf("config file error: %w", err)) + } + + chainRest := utils.GetChainRest(viper.GetString("chain_id"), viper.GetString("chain_rest")) + storageRest = strings.TrimSuffix(storageRest, "/") + + bundlesPage, _, err := bundles.GetFinalizedBundlesPageWithOffset(chainRest, poolId, 1, bundleId, "", false) + if err != nil { + panic(err) + return + } + + deflated, err := bundles.GetDataFromFinalizedBundle(bundlesPage[0], storageRest) + if err != nil { + panic(err) + return + } + + var bundle types.Bundle + + if err := json.Unmarshal(deflated, &bundle); err != nil { + panic(err) + return + } + + bundleId, err := strconv.Atoi(bundlesPage[0].Id) + if err != nil { + panic(err) + return + } + + var leafHashes [][32]byte + + leafHashes = m.BundleToHashes(bundle, "@kyvejs/tendermint") + + merkleRoot := m.GenerateMerkleRoot(&leafHashes) + + logger.Info(). + Int("bundle-id", bundleId). + Str("root", hex.EncodeToString(merkleRoot[:])). + Msg("computed Merkle root") + }, +} diff --git a/config.yaml b/config.yaml index 84831ac..bb30217 100644 --- a/config.yaml +++ b/config.yaml @@ -15,16 +15,16 @@ worker: 32 # --- pools: - pool_id: 0 - target_bundle_id: 259772 + target_bundle_id: 259773 - pool_id: 1 - target_bundle_id: 165754 + target_bundle_id: 165755 - pool_id: 2 - target_bundle_id: 206232 + target_bundle_id: 206233 - pool_id: 3 - target_bundle_id: 196925 + target_bundle_id: 196926 - pool_id: 5 - target_bundle_id: 147373 + target_bundle_id: 147374 - pool_id: 7 - target_bundle_id: 92881 + target_bundle_id: 92882 - pool_id: 9 - target_bundle_id: 68399 \ No newline at end of file + target_bundle_id: 68400 \ No newline at end of file diff --git a/go.sum b/go.sum index fc25e83..6de88d0 100644 --- a/go.sum +++ b/go.sum @@ -254,6 +254,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -307,9 +309,12 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= diff --git a/merkle.go b/merkle.go index 3b2d045..7e84ad1 100644 --- a/merkle.go +++ b/merkle.go @@ -3,6 +3,7 @@ package merkle_script import ( "crypto/sha256" "encoding/json" + "fmt" "github.com/KYVENetwork/ksync/types" ) @@ -48,11 +49,11 @@ func BundleToHashes(bundle types.Bundle, runtime string) [][32]byte { switch runtime { case "@kyvejs/tendermint-bsync": for _, dataItem := range bundle { - leafHashes = append(leafHashes, calculateSHA256Hash(dataItem)) + leafHashes = append(leafHashes, sha256.Sum256([]byte(fmt.Sprintf("{\"key\":\"%s\",\"value\":%s}", dataItem.Key, string(dataItem.Value))))) } case "@kyvejs/tendermint": for _, dataItem := range bundle { - leafHashes = append(leafHashes, tendermintItemToSha256(dataItem)) + leafHashes = append(leafHashes, dataItemToSha256(dataItem)) } default: logger.Error(). @@ -64,7 +65,7 @@ func BundleToHashes(bundle types.Bundle, runtime string) [][32]byte { return leafHashes } -func tendermintItemToSha256(dataItem types.DataItem) [32]byte { +func dataItemToSha256(dataItem types.DataItem) [32]byte { merkleRoot := createHashesForTendermintValue(dataItem) keyBytes := sha256.Sum256([]byte(dataItem.Key)) @@ -83,21 +84,8 @@ func createHashesForTendermintValue(dataItem types.DataItem) [32]byte { var hashes [][32]byte - hashes = append(hashes, calculateSHA256Hash(tendermintValue.Block)) - hashes = append(hashes, calculateSHA256Hash(tendermintValue.BlockResults)) + hashes = append(hashes, sha256.Sum256(tendermintValue.Block)) + hashes = append(hashes, sha256.Sum256(tendermintValue.BlockResults)) return GenerateMerkleRoot(&hashes) } - -func calculateSHA256Hash(obj interface{}) [32]byte { - // Serialize the object to JSON with keys sorted ascending by default - serializedObj, err := json.Marshal(obj) - if err != nil { - panic(err) - } - - // Calculate the SHA-256 hash - sha256Hash := sha256.Sum256(serializedObj) - - return sha256Hash -} diff --git a/python/merkle_generator.py b/python/merkle_generator.py index c487126..67da80a 100644 --- a/python/merkle_generator.py +++ b/python/merkle_generator.py @@ -44,9 +44,9 @@ def tendermint_merkle_root(bundle_content): sha256(item["key"].encode('utf-8')).hexdigest(), merkle_root( [ - sha256(json.dumps(item["value"]["block"], separators=(',', ':')).encode('utf-8')).hexdigest(), + sha256(json.dumps(item["value"]["block"], separators=(',', ':'), ensure_ascii=False).encode('utf-8')).hexdigest(), sha256( - json.dumps(item["value"]["block_results"], separators=(',', ':')).encode('utf-8')).hexdigest() + json.dumps(item["value"]["block_results"], separators=(',', ':'), ensure_ascii=False).encode('utf-8')).hexdigest() ] ) ])