fix: different hashes

cmd/merkle-script/commands/root.go

@@ -7,6 +7,8 @@ import (
 
 var (
 	config      string
+	bundleId    int64
+	poolId      int64
 	storageRest string
 )
 

cmd/merkle-script/commands/test.go

@@ -0,0 +1,76 @@
+package commands
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"github.com/KYVENetwork/ksync/collectors/bundles"
+	"github.com/KYVENetwork/ksync/types"
+	"github.com/KYVENetwork/ksync/utils"
+	m "github.com/KYVENetwork/merkle-script"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"strconv"
+	"strings"
+)
+
+func init() {
+	testCmd.Flags().Int64Var(&bundleId, "bundle-id", 0, "bundle id")
+
+	testCmd.Flags().Int64Var(&poolId, "pool-id", 0, "pool id")
+
+	rootCmd.AddCommand(testCmd)
+}
+
+var testCmd = &cobra.Command{
+	Use:   "test",
+	Short: "Start merkle-script to create historical Merkle roots",
+	Run: func(cmd *cobra.Command, args []string) {
+		viper.SetConfigName("config")
+		viper.SetConfigType("yaml")
+		viper.AddConfigPath(config)
+		err := viper.ReadInConfig()
+		if err != nil {
+			panic(fmt.Errorf("config file error: %w", err))
+		}
+
+		chainRest := utils.GetChainRest(viper.GetString("chain_id"), viper.GetString("chain_rest"))
+		storageRest = strings.TrimSuffix(storageRest, "/")
+
+		bundlesPage, _, err := bundles.GetFinalizedBundlesPageWithOffset(chainRest, poolId, 1, bundleId, "", false)
+		if err != nil {
+			panic(err)
+			return
+		}
+
+		deflated, err := bundles.GetDataFromFinalizedBundle(bundlesPage[0], storageRest)
+		if err != nil {
+			panic(err)
+			return
+		}
+
+		var bundle types.Bundle
+
+		if err := json.Unmarshal(deflated, &bundle); err != nil {
+			panic(err)
+			return
+		}
+
+		bundleId, err := strconv.Atoi(bundlesPage[0].Id)
+		if err != nil {
+			panic(err)
+			return
+		}
+
+		var leafHashes [][32]byte
+
+		leafHashes = m.BundleToHashes(bundle, "@kyvejs/tendermint")
+
+		merkleRoot := m.GenerateMerkleRoot(&leafHashes)
+
+		logger.Info().
+			Int("bundle-id", bundleId).
+			Str("root", hex.EncodeToString(merkleRoot[:])).
+			Msg("computed Merkle root")
+	},
+}

config.yaml

@@ -15,16 +15,16 @@ worker: 32
 # ---
 pools:
   - pool_id: 0
-    target_bundle_id: 259772
+    target_bundle_id: 259773
   - pool_id: 1
-    target_bundle_id: 165754
+    target_bundle_id: 165755
   - pool_id: 2
-    target_bundle_id: 206232
+    target_bundle_id: 206233
   - pool_id: 3
-    target_bundle_id: 196925
+    target_bundle_id: 196926
   - pool_id: 5
-    target_bundle_id: 147373
+    target_bundle_id: 147374
   - pool_id: 7
-    target_bundle_id: 92881
+    target_bundle_id: 92882
   - pool_id: 9
-    target_bundle_id: 68399
+    target_bundle_id: 68400

go.sum

@@ -254,6 +254,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -307,9 +309,12 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

merkle.go

@@ -3,6 +3,7 @@ package merkle_script
 import (
 	"crypto/sha256"
 	"encoding/json"
+	"fmt"
 
 	"github.com/KYVENetwork/ksync/types"
 )
@@ -48,11 +49,11 @@ func BundleToHashes(bundle types.Bundle, runtime string) [][32]byte {
 	switch runtime {
 	case "@kyvejs/tendermint-bsync":
 		for _, dataItem := range bundle {
-			leafHashes = append(leafHashes, calculateSHA256Hash(dataItem))
+			leafHashes = append(leafHashes, sha256.Sum256([]byte(fmt.Sprintf("{\"key\":\"%s\",\"value\":%s}", dataItem.Key, string(dataItem.Value)))))
 		}
 	case "@kyvejs/tendermint":
 		for _, dataItem := range bundle {
-			leafHashes = append(leafHashes, tendermintItemToSha256(dataItem))
+			leafHashes = append(leafHashes, dataItemToSha256(dataItem))
 		}
 	default:
 		logger.Error().
@@ -64,7 +65,7 @@ func BundleToHashes(bundle types.Bundle, runtime string) [][32]byte {
 	return leafHashes
 }
 
-func tendermintItemToSha256(dataItem types.DataItem) [32]byte {
+func dataItemToSha256(dataItem types.DataItem) [32]byte {
 	merkleRoot := createHashesForTendermintValue(dataItem)
 
 	keyBytes := sha256.Sum256([]byte(dataItem.Key))
@@ -83,21 +84,8 @@ func createHashesForTendermintValue(dataItem types.DataItem) [32]byte {
 
 	var hashes [][32]byte
 
-	hashes = append(hashes, calculateSHA256Hash(tendermintValue.Block))
-	hashes = append(hashes, calculateSHA256Hash(tendermintValue.BlockResults))
+	hashes = append(hashes, sha256.Sum256(tendermintValue.Block))
+	hashes = append(hashes, sha256.Sum256(tendermintValue.BlockResults))
 
 	return GenerateMerkleRoot(&hashes)
 }
-
-func calculateSHA256Hash(obj interface{}) [32]byte {
-	// Serialize the object to JSON with keys sorted ascending by default
-	serializedObj, err := json.Marshal(obj)
-	if err != nil {
-		panic(err)
-	}
-
-	// Calculate the SHA-256 hash
-	sha256Hash := sha256.Sum256(serializedObj)
-
-	return sha256Hash
-}

python/merkle_generator.py

@@ -44,9 +44,9 @@ def tendermint_merkle_root(bundle_content):
             sha256(item["key"].encode('utf-8')).hexdigest(),
             merkle_root(
                 [
-                    sha256(json.dumps(item["value"]["block"], separators=(',', ':')).encode('utf-8')).hexdigest(),
+                    sha256(json.dumps(item["value"]["block"], separators=(',', ':'), ensure_ascii=False).encode('utf-8')).hexdigest(),
                     sha256(
-                        json.dumps(item["value"]["block_results"], separators=(',', ':')).encode('utf-8')).hexdigest()
+                        json.dumps(item["value"]["block_results"], separators=(',', ':'), ensure_ascii=False).encode('utf-8')).hexdigest()
                 ]
             )
         ])