Prover: assignment serialization for the limitless prover (#557)

* feat(encode): implements an codex for column assignment

* debug(col): add tests and debug the column assignment

* experiment(encoding): adds a target to encode and decode zkevm assignment and benchmark

* add lz4 compression library

* add assignment serialization with compression

* assign and encode in chunks instead of a file

* calculate correct ser size

* add num chunks variable

* Revert "calculate correct ser size"

This reverts commit 8ae74c6.

* calculate and print the exact size of ser

* add error handling for file write

* add deserialization

* refactor code

* add compressChunks

* refactor deserialize assignment

* add hashWAssignment to check the hash of input and output

* refactor SerializeAssignment

* use logrus

* remove cbor data size info

* clean up logs

* clean up logs for serialization

* measure time taken for serialization

* update logs

* fix logs

* set numChunks to the number of cores

* skip profiling

* add compression ratio

* add a unit test

---------

Co-authored-by: AlexandreBelling <alexandrebelling8@gmail.com>

Author: gusiri

prover/backend/execution/prove.go

@@ -1,11 +1,19 @@
 package execution
 
 import (
+	"fmt"
+	"math/rand/v2"
+	"os"
+	"runtime"
+	"strconv"
+	"time"
+
 	"github.com/consensys/gnark-crypto/ecc"
 	"github.com/consensys/linea-monorepo/prover/circuits"
 	"github.com/consensys/linea-monorepo/prover/circuits/dummy"
 	"github.com/consensys/linea-monorepo/prover/circuits/execution"
 	"github.com/consensys/linea-monorepo/prover/config"
+	"github.com/consensys/linea-monorepo/prover/protocol/serialization"
 	public_input "github.com/consensys/linea-monorepo/prover/public-input"
 	"github.com/consensys/linea-monorepo/prover/utils"
 	"github.com/consensys/linea-monorepo/prover/utils/profiling"
@@ -179,6 +187,31 @@ func mustProveAndPass(
 		logrus.Infof("Prover checks passed")
 		return "", ""
 
+	case config.ProverModeEncodeOnly:
+
+		profiling.ProfileTrace("encode-decode-no-circuit", true, false, func() {
+			filepath := "/tmp/wizard-assignment/blob-" + strconv.Itoa(rand.Int()) + ".bin"
+
+			encodeOnlyZkEvm := zkevm.EncodeOnlyZkEvm(traces)
+			numChunks := runtime.GOMAXPROCS(0)
+
+			// Serialize the assignment
+			encodingDuration := time.Now()
+			encodeOnlyZkEvm.AssignAndEncodeInChunks(filepath, w.ZkEVM, numChunks)
+
+			// Deserialize the assignment
+			decodingDuration := time.Now()
+			_, errDec := serialization.DeserializeAssignment(filepath, numChunks)
+			if errDec != nil {
+				panic(fmt.Sprintf("Error during deserialization: %v", errDec))
+			}
+			fmt.Printf("[Encoding Summary] took %v sec to encode an assignmente and write it into the files \n", time.Since(encodingDuration).Seconds())
+			fmt.Printf("[Decoding Summary] took %v sec to read the files and decode it into an assignment\n", time.Since(decodingDuration).Seconds())
+		})
+
+		os.Exit(0)
+		return "", ""
+
 	default:
 		panic("not implemented")
 	}

prover/config/config.go

@@ -203,7 +203,7 @@ type Execution struct {
 	WithRequestDir `mapstructure:",squash"`
 
 	// ProverMode stores the kind of prover to use.
-	ProverMode ProverMode `mapstructure:"prover_mode" validate:"required,oneof=dev partial full proofless bench check-only"`
+	ProverMode ProverMode `mapstructure:"prover_mode" validate:"required,oneof=dev partial full proofless bench check-only encode-only"`
 
 	// CanRunFullLarge indicates whether the prover is running on a large machine (and can run full large traces).
 	CanRunFullLarge bool `mapstructure:"can_run_full_large"`

prover/config/types.go

@@ -26,5 +26,6 @@ const (
 	// in a context where it is simpler to not have to deal with the setup.
 	ProverModeBench ProverMode = "bench"
 	// ProverModeCheckOnly is used to test the constraints of the whole system
-	ProverModeCheckOnly ProverMode = "check-only"
+	ProverModeCheckOnly  ProverMode = "check-only"
+	ProverModeEncodeOnly ProverMode = "encode-only"
 )

prover/go.mod

@@ -76,6 +76,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect

prover/go.sum

@@ -369,6 +369,8 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
+github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
+github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
 github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=

prover/maths/common/smartvectors/windowed.go

@@ -47,6 +47,21 @@ func (p *PaddedCircularWindow) Len() int {
 	return p.totLen
 }
 
+// Offset returns the offset of the PCW
+func (p *PaddedCircularWindow) Offset() int {
+	return p.offset
+}
+
+// Windows returns the length of the window of the PCQ
+func (p *PaddedCircularWindow) Window() []field.Element {
+	return p.window
+}
+
+// PaddingVal returns the value used for padding the window
+func (p *PaddedCircularWindow) PaddingVal() field.Element {
+	return p.paddingVal
+}
+
 // Returns a queries position
 func (p *PaddedCircularWindow) GetBase(n int) (field.Element, error) {
 	// Check if the queried index is in the window