Decompressor Library (#229)

* chore bring over changes from other branch * chore: blocks as byte arrays not lists; tx.S <- 1 * build: update compressor lib release script by jpnovais * docs and go generate --------- Co-authored-by: Arya Tabaie <15056835+Tabaie@users.noreply.github.com>
Consensys · Oct 22, 2024 · ce964fc · ce964fc
1 parent d2a120f
commit ce964fc
Show file tree

Hide file tree

Showing 33 changed files with 1,402 additions and 482 deletions.
diff --git a/.github/workflows/prover-native-lib-blob-compressor-release.yml b/.github/workflows/prover-native-lib-blob-compressor-release.yml
@@ -7,6 +7,7 @@ on:
         description: 'Version (e.g. v1.2.3)'
         required: true
         default: 'v0.0.0'
+        type: string
       draft-release:
         description: 'Draft Release'
         required: false
@@ -37,13 +38,16 @@ jobs:
           VERSION: ${{ github.event.inputs.version }}
           SRC_SHNARF: "./lib/shnarf_calculator/shnarf_calculator.go"
           TARGET_SHNARF: "shnarf_calculator"
-          SRC_COMPRESSOR: "./lib/compressor/libcompressor.go"
+          SRC_COMPRESSOR: "./lib/compressor/libcompressor/libcompressor.go"
           TARGET_COMPRESSOR: "blob_compressor"
+          SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
+          TARGET_DECOMPRESSOR: "blob_decompressor"
         run: |
           cd prover
           mkdir target
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_linux_x86_64.so ${SRC_SHNARF}
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_linux_x86_64.so ${SRC_COMPRESSOR}
+          GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_linux_x86_64.so ${SRC_DECOMPRESSOR}
 
       - name: Cache built binaries
         uses: actions/upload-artifact@master
@@ -69,13 +73,16 @@ jobs:
           VERSION: ${{ github.event.inputs.version }}
           SRC_SHNARF: "./lib/shnarf_calculator/shnarf_calculator.go"
           TARGET_SHNARF: "shnarf_calculator"
-          SRC_COMPRESSOR: "./lib/compressor/libcompressor.go"
+          SRC_COMPRESSOR: "./lib/compressor/libcompressor/libcompressor.go"
           TARGET_COMPRESSOR: "blob_compressor"
+          SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
+          TARGET_DECOMPRESSOR: "blob_decompressor"
         run: |
           cd prover
           mkdir target
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_linux_arm64.so ${SRC_SHNARF}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_linux_arm64.so ${SRC_COMPRESSOR}
+          GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_linux_arm64.so ${SRC_DECOMPRESSOR}
       - name: Cache built binaries
         uses: actions/upload-artifact@master
         with:
@@ -98,15 +105,19 @@ jobs:
           VERSION: ${{ github.event.inputs.version }}
           SRC_SHNARF: "./lib/shnarf_calculator/shnarf_calculator.go"
           TARGET_SHNARF: "shnarf_calculator"
-          SRC_COMPRESSOR: "./lib/compressor/libcompressor.go"
+          SRC_COMPRESSOR: "./lib/compressor/libcompressor/libcompressor.go"
           TARGET_COMPRESSOR: "blob_compressor"
+          SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
+          TARGET_DECOMPRESSOR: "blob_decompressor"
         run: |
           cd prover
           mkdir target
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_darwin_x86_64.dylib ${SRC_SHNARF}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_darwin_arm64.dylib ${SRC_SHNARF}
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_darwin_x86_64.dylib ${SRC_COMPRESSOR}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_darwin_arm64.dylib ${SRC_COMPRESSOR}
+          GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_darwin_x86_64.dylib ${SRC_DECOMPRESSOR}
+          GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_darwin_arm64.dylib ${SRC_DECOMPRESSOR}
 
       - name: Cache built binaries
         uses: actions/upload-artifact@v4

diff --git a/prover/backend/blobsubmission/blobcompression_test.go b/prover/backend/blobsubmission/blobcompression_test.go
@@ -9,7 +9,7 @@ import (
 	"strings"
 	"testing"
 
-	blob "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v1"
+	"github.com/consensys/linea-monorepo/prover/lib/compressor/blob/encode"
 
 	fr381 "github.com/consensys/gnark-crypto/ecc/bls12-381/fr"
 	"github.com/consensys/linea-monorepo/prover/utils"
@@ -270,7 +270,7 @@ func TestKZGWithPoint(t *testing.T) {
 	}
 
 	// Compute all the prover fields
-	snarkHash, err := blob.MiMCChecksumPackedData(blobBytes[:], fr381.Bits-1, blob.NoTerminalSymbol())
+	snarkHash, err := encode.MiMCChecksumPackedData(blobBytes[:], fr381.Bits-1, encode.NoTerminalSymbol())
 	assert.NoError(t, err)
 
 	xUnreduced := evaluationChallenge(snarkHash, blobHash[:])

diff --git a/prover/backend/blobsubmission/craft.go b/prover/backend/blobsubmission/craft.go
@@ -7,9 +7,9 @@ import (
 	"hash"
 
 	"github.com/consensys/linea-monorepo/prover/crypto/mimc"
+	"github.com/consensys/linea-monorepo/prover/lib/compressor/blob/encode"
 
 	fr381 "github.com/consensys/gnark-crypto/ecc/bls12-381/fr"
-	blob "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v1"
 	"github.com/consensys/linea-monorepo/prover/utils"
 	"golang.org/x/crypto/sha3"
 )
@@ -72,7 +72,7 @@ func CraftResponseCalldata(req *Request) (*Response, error) {
 	}
 
 	// Compute all the prover fields
-	snarkHash, err := blob.MiMCChecksumPackedData(compressedStream, fr381.Bits-1, blob.NoTerminalSymbol())
+	snarkHash, err := encode.MiMCChecksumPackedData(compressedStream, fr381.Bits-1, encode.NoTerminalSymbol())
 	if err != nil {
 		return nil, fmt.Errorf("crafting response: could not compute snark hash: %w", err)
 	}

diff --git a/prover/backend/blobsubmission/craft_eip4844.go b/prover/backend/blobsubmission/craft_eip4844.go
@@ -4,6 +4,7 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"github.com/consensys/linea-monorepo/prover/lib/compressor/blob/encode"
 
 	blob "github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v1"
 
@@ -91,7 +92,7 @@ func CraftResponse(req *Request) (*Response, error) {
 	}
 
 	// Compute all the prover fields
-	snarkHash, err := blob.MiMCChecksumPackedData(append(compressedStream, make([]byte, blob.MaxUsableBytes-len(compressedStream))...), fr381.Bits-1, blob.NoTerminalSymbol())
+	snarkHash, err := encode.MiMCChecksumPackedData(append(compressedStream, make([]byte, blob.MaxUsableBytes-len(compressedStream))...), fr381.Bits-1, encode.NoTerminalSymbol())
 	if err != nil {
 		return nil, fmt.Errorf("crafting response: could not compute snark hash: %w", err)
 	}

diff --git a/prover/backend/ethereum/signature_test.go b/prover/backend/ethereum/signature_test.go
@@ -117,10 +117,11 @@ func TestTransactionSigning(t *testing.T) {
 			assert.Equal(t, from.Hex(), recovered.Hex(), "Mismatch of the recovered address")
 
 			// Simulates the decoding of the transaction
-			var decodedTx types.Transaction
-			err = DecodeTxFromBytes(bytes.NewReader(rlp), &decodedTx)
-
+			decodedTxData, err := DecodeTxFromBytes(bytes.NewReader(rlp))
 			require.NoError(t, err)
+
+			decodedTx := types.NewTx(decodedTxData)
+
 			assert.Equal(t, tx.To(), decodedTx.To())
 			assert.Equal(t, tx.Nonce(), decodedTx.Nonce())
 			assert.Equal(t, tx.Data(), decodedTx.Data())

diff --git a/prover/backend/ethereum/tx_encoding.go b/prover/backend/ethereum/tx_encoding.go
@@ -97,25 +97,25 @@ const (
 // than the transaction then the remaining bytes are discarded and only the
 // first bytes are used to decode the transaction. The function returns the
 // transactions and the number of bytes read.
-func DecodeTxFromBytes(b *bytes.Reader, tx *types.Transaction) (err error) {
+func DecodeTxFromBytes(b *bytes.Reader) (tx types.TxData, err error) {
 
 	var (
 		firstByte byte
 	)
 
 	if b.Len() == 0 {
-		return fmt.Errorf("empty buffer")
+		return nil, fmt.Errorf("empty buffer")
 	}
 
 	if firstByte, err = b.ReadByte(); err != nil {
-		return fmt.Errorf("could not read the first byte: %w", err)
+		return nil, fmt.Errorf("could not read the first byte: %w", err)
 	}
 
 	switch {
 	case firstByte == types.DynamicFeeTxType:
-		return decodeDynamicFeeTx(b, tx)
+		return decodeDynamicFeeTx(b)
 	case firstByte == types.AccessListTxType:
-		return decodeAccessListTx(b, tx)
+		return decodeAccessListTx(b)
 	// According to the RLP rule, `0xc0 + x` or `0xf7` indicates that the current
 	// item is a list and this is what's used to identify that the transaction is
 	// a legacy transaction or a EIP-155 transaction.
@@ -125,69 +125,69 @@ func DecodeTxFromBytes(b *bytes.Reader, tx *types.Transaction) (err error) {
 		// Set the byte-reader backward so that we can apply the rlp-decoder
 		// over it.
 		b.UnreadByte()
-		return decodeLegacyTx(b, tx)
+		return decodeLegacyTx(b)
+	default:
+		return nil, fmt.Errorf("unexpected first byte: %x", firstByte)
 	}
-
-	return fmt.Errorf("unexpected first byte: %x", firstByte)
 }
 
 // decodeDynamicFeeTx encodes a [types.DynamicFeeTx] into a [bytes.Reader] and
 // returns an error if it did not pass.
-func decodeDynamicFeeTx(b *bytes.Reader, tx *types.Transaction) (err error) {
+func decodeDynamicFeeTx(b *bytes.Reader) (parsedTx *types.DynamicFeeTx, err error) {
 	decTx := []any{}
 
-	if err := rlp.Decode(b, &decTx); err != nil {
-		return fmt.Errorf("could not rlp decode transaction: %w", err)
+	if err = rlp.Decode(b, &decTx); err != nil {
+		return nil, fmt.Errorf("could not rlp decode transaction: %w", err)
 	}
 
 	if len(decTx) != dynFeeNumField {
-		return fmt.Errorf("invalid number of field for a dynamic transaction")
+		return nil, fmt.Errorf("invalid number of field for a dynamic transaction")
 	}
 
-	parsedTx := types.DynamicFeeTx{}
+	parsedTx = new(types.DynamicFeeTx)
+
 	err = errors.Join(
-		tryCast(&parsedTx.ChainID, decTx[0], "chainID"),
-		tryCast(&parsedTx.Nonce, decTx[1], "nonce"),
-		tryCast(&parsedTx.GasTipCap, decTx[2], "gas-tip-cap"),
-		tryCast(&parsedTx.GasFeeCap, decTx[3], "gas-fee-cap"),
-		tryCast(&parsedTx.Gas, decTx[4], "gas"),
-		tryCast(&parsedTx.To, decTx[5], "to"),
-		tryCast(&parsedTx.Value, decTx[6], "value"),
-		tryCast(&parsedTx.Data, decTx[7], "data"),
-		tryCast(&parsedTx.AccessList, decTx[8], "access-list"),
+		TryCast(&parsedTx.ChainID, decTx[0], "chainID"),
+		TryCast(&parsedTx.Nonce, decTx[1], "nonce"),
+		TryCast(&parsedTx.GasTipCap, decTx[2], "gas-tip-cap"),
+		TryCast(&parsedTx.GasFeeCap, decTx[3], "gas-fee-cap"),
+		TryCast(&parsedTx.Gas, decTx[4], "gas"),
+		TryCast(&parsedTx.To, decTx[5], "to"),
+		TryCast(&parsedTx.Value, decTx[6], "value"),
+		TryCast(&parsedTx.Data, decTx[7], "data"),
+		TryCast(&parsedTx.AccessList, decTx[8], "access-list"),
 	)
-	*tx = *types.NewTx(&parsedTx)
-	return err
+
+	return
 }
 
-// decodeAccessListTx decodes an [types.AccessListTx] from a [bytes.Reader]
+// decodeAccessListTx decodes a [types.AccessListTx] from a [bytes.Reader]
 // and returns an error if it did not pass.
-func decodeAccessListTx(b *bytes.Reader, tx *types.Transaction) (err error) {
+func decodeAccessListTx(b *bytes.Reader) (parsedTx *types.AccessListTx, err error) {
 
 	decTx := []any{}
 
 	if err := rlp.Decode(b, &decTx); err != nil {
-		return fmt.Errorf("could not rlp decode transaction: %w", err)
+		return nil, fmt.Errorf("could not rlp decode transaction: %w", err)
 	}
 
 	if len(decTx) != accessListTxNumField {
-		return fmt.Errorf("invalid number of field for a dynamic transaction")
+		return nil, fmt.Errorf("invalid number of field for a dynamic transaction")
 	}
 
-	parsedTx := types.AccessListTx{}
+	parsedTx = new(types.AccessListTx)
 	err = errors.Join(
-		tryCast(&parsedTx.ChainID, decTx[0], "chainID"),
-		tryCast(&parsedTx.Nonce, decTx[1], "nonce"),
-		tryCast(&parsedTx.GasPrice, decTx[2], "gas-price"),
-		tryCast(&parsedTx.Gas, decTx[3], "gas"),
-		tryCast(&parsedTx.To, decTx[4], "to"),
-		tryCast(&parsedTx.Value, decTx[5], "value"),
-		tryCast(&parsedTx.Data, decTx[6], "data"),
-		tryCast(&parsedTx.AccessList, decTx[7], "access-list"),
+		TryCast(&parsedTx.ChainID, decTx[0], "chainID"),
+		TryCast(&parsedTx.Nonce, decTx[1], "nonce"),
+		TryCast(&parsedTx.GasPrice, decTx[2], "gas-price"),
+		TryCast(&parsedTx.Gas, decTx[3], "gas"),
+		TryCast(&parsedTx.To, decTx[4], "to"),
+		TryCast(&parsedTx.Value, decTx[5], "value"),
+		TryCast(&parsedTx.Data, decTx[6], "data"),
+		TryCast(&parsedTx.AccessList, decTx[7], "access-list"),
 	)
 
-	*tx = *types.NewTx(&parsedTx)
-	return err
+	return
 }
 
 // decodeLegacyTx decodes a [types.LegacyTx] from a [bytes.Reader] and returns
@@ -197,44 +197,43 @@ func decodeAccessListTx(b *bytes.Reader, tx *types.Transaction) (err error) {
 // not decoded although it could. The reason is that it is complicated to set
 // it in the returned element as it "included" in the signature and we don't
 // encode the signature.
-func decodeLegacyTx(b *bytes.Reader, tx *types.Transaction) (err error) {
+func decodeLegacyTx(b *bytes.Reader) (parsedTx *types.LegacyTx, err error) {
 
 	decTx := []any{}
 
-	if err := rlp.Decode(b, &decTx); err != nil {
-		return fmt.Errorf("could not rlp decode transaction: %w", err)
+	if err = rlp.Decode(b, &decTx); err != nil {
+		return nil, fmt.Errorf("could not rlp decode transaction: %w", err)
 	}
 
 	if len(decTx) != legacyTxNumField && len(decTx) != unprotectedTxNumField {
-		return fmt.Errorf("unexpected number of field")
+		return nil, fmt.Errorf("unexpected number of field")
 	}
 
-	parsedTx := types.LegacyTx{}
+	parsedTx = new(types.LegacyTx)
 	err = errors.Join(
-		tryCast(&parsedTx.Nonce, decTx[0], "nonce"),
-		tryCast(&parsedTx.GasPrice, decTx[1], "gas-price"),
-		tryCast(&parsedTx.Gas, decTx[2], "gas"),
-		tryCast(&parsedTx.To, decTx[3], "to"),
-		tryCast(&parsedTx.Value, decTx[4], "value"),
-		tryCast(&parsedTx.Data, decTx[5], "data"),
+		TryCast(&parsedTx.Nonce, decTx[0], "nonce"),
+		TryCast(&parsedTx.GasPrice, decTx[1], "gas-price"),
+		TryCast(&parsedTx.Gas, decTx[2], "gas"),
+		TryCast(&parsedTx.To, decTx[3], "to"),
+		TryCast(&parsedTx.Value, decTx[4], "value"),
+		TryCast(&parsedTx.Data, decTx[5], "data"),
 	)
 
-	*tx = *types.NewTx(&parsedTx)
-	return err
+	return
 }
 
-// tryCast will attempt to set t with the underlying value of `from` will return
+// TryCast will attempt to set t with the underlying value of `from` will return
 // an error if the type does not match. The explainer string is used to generate
 // the error if any.
-func tryCast[T any](into *T, from any, explainer string) error {
+func TryCast[T any](into *T, from any, explainer string) error {
 
 	if into == nil || from == nil {
 		return fmt.Errorf("from or into is/are nil")
 	}
 
 	// The rlp encoding is not "type-aware", if the underlying field is an
 	// access-list, it will decode into []interface{} (and we recursively parse
-	// it) otherwise, it always decode to `[]byte`
+	// it) otherwise, it always decodes to `[]byte`
 	if list, ok := (from).([]interface{}); ok {
 
 		var (
@@ -249,7 +248,7 @@ func tryCast[T any](into *T, from any, explainer string) error {
 			for i := range accessList {
 				err = errors.Join(
 					err,
-					tryCast(&accessList[i], list[i], fmt.Sprintf("%v[%v]", explainer, i)),
+					TryCast(&accessList[i], list[i], fmt.Sprintf("%v[%v]", explainer, i)),
 				)
 			}
 			*into = (any(accessList)).(T)
@@ -258,16 +257,16 @@ func tryCast[T any](into *T, from any, explainer string) error {
 		case types.AccessTuple:
 			tuple := types.AccessTuple{}
 			err = errors.Join(
-				tryCast(&tuple.Address, list[0], fmt.Sprintf("%v.%v", explainer, "address")),
-				tryCast(&tuple.StorageKeys, list[1], fmt.Sprintf("%v.%v", explainer, "storage-key")),
+				TryCast(&tuple.Address, list[0], fmt.Sprintf("%v.%v", explainer, "address")),
+				TryCast(&tuple.StorageKeys, list[1], fmt.Sprintf("%v.%v", explainer, "storage-key")),
 			)
 			*into = (any(tuple)).(T)
 			return err
 
 		case []common.Hash:
 			hashes := make([]common.Hash, length)
 			for i := range hashes {
-				tryCast(&hashes[i], list[i], fmt.Sprintf("%v[%v]", explainer, i))
+				TryCast(&hashes[i], list[i], fmt.Sprintf("%v[%v]", explainer, i))
 			}
 			*into = (any(hashes)).(T)
 			return err
@@ -285,7 +284,7 @@ func tryCast[T any](into *T, from any, explainer string) error {
 	switch intoAny.(type) {
 	case *common.Address:
 		// Parse the bytes as an UTF8 string (= direct casting in go).
-		// Then, the string as an hexstring encoded address.
+		// Then, the string as a hex string encoded address.
 		address := common.BytesToAddress(fromBytes)
 		*into = any(&address).(T)
 	case common.Address:
@@ -295,7 +294,7 @@ func tryCast[T any](into *T, from any, explainer string) error {
 		*into = any(address).(T)
 	case common.Hash:
 		// Parse the bytes as an UTF8 string (= direct casting in go).
-		// Then, the string as an hexstring encoded address.
+		// Then, the string as a hexstring encoded address.
 		hash := common.BytesToHash(fromBytes)
 		*into = any(hash).(T)
 	case *big.Int:

diff --git a/prover/circuits/blobdecompression/v0/assign_test.go b/prover/circuits/blobdecompression/v0/assign_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	v0 "github.com/consensys/linea-monorepo/prover/circuits/blobdecompression/v0"
+	"github.com/consensys/linea-monorepo/prover/lib/compressor/blob/dictionary"
 	"github.com/consensys/linea-monorepo/prover/lib/compressor/blob/v1/test_utils"
 
 	"github.com/consensys/gnark-crypto/ecc"
@@ -74,7 +75,9 @@ func mustGetTestCompressedData(t *testing.T) (resp blobsubmission.Response, blob
 	blobBytes, err = base64.StdEncoding.DecodeString(resp.CompressedData)
 	assert.NoError(t, err)
 
-	_, _, _, err = blob.DecompressBlob(blobBytes, dict)
+	dictStore, err := dictionary.SingletonStore(dict, 0)
+	assert.NoError(t, err)
+	_, _, _, err = blob.DecompressBlob(blobBytes, dictStore)
 	assert.NoError(t, err)
 
 	return