adds blob decompressor

jvm-libs/linea/blob-decompressor/build.gradle

@@ -0,0 +1,59 @@
+plugins {
+  id 'net.consensys.zkevm.kotlin-library-conventions'
+  id 'net.consensys.zkevm.linea-native-libs-helper'
+  alias(libs.plugins.jreleaser)
+  id 'java-test-fixtures'
+}
+
+description = 'Java JNA wrapper for Linea Blob Decompressor Library implemented in GO Lang'
+apply from: rootProject.file("gradle/publishing.gradle")
+
+dependencies {
+  implementation "net.java.dev.jna:jna:${libs.versions.jna.get()}"
+  implementation project(":jvm-libs:generic:extensions:kotlin")
+
+  testImplementation project(":jvm-libs:linea:blob-compressor")
+  testImplementation(testFixtures(project(":jvm-libs:linea:blob-compressor")))
+  testImplementation(project(":jvm-libs:linea:testing:file-system"))
+  testImplementation("io.tmio:tuweni-bytes:${libs.versions.tuweni.get()}")
+  testImplementation("org.hyperledger.besu:besu-datatypes:${libs.versions.besu.get()}")
+  testImplementation "org.hyperledger.besu:evm:${libs.versions.besu.get()}"
+  testImplementation("org.hyperledger.besu.internal:core:${libs.versions.besu.get()}")
+  testImplementation("org.hyperledger.besu:plugin-api:${libs.versions.besu.get()}")
+  testImplementation("org.hyperledger.besu.internal:rlp:${libs.versions.besu.get()}")
+}
+
+jar {
+  dependsOn configurations.runtimeClasspath
+}
+
+test {
+  // we cannot have more 1 compressor per JVM, hence we disable parallel execution
+  // because multiple threads would cause issues with the native library
+  systemProperties["junit.jupiter.execution.parallel.enabled"] = false
+  maxParallelForks = 1
+}
+
+def libsZipDownloadOutputDir = project.parent.layout.buildDirectory.asFile.get().absolutePath
+
+task downloadNativeLibs {
+  doLast {
+    fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v1.1.0-test8/linea-blob-libs-v1.1.0-test8.zip", "blob_decompressor", libsZipDownloadOutputDir)
+  }
+}
+
+compileKotlin {
+  dependsOn tasks.downloadNativeLibs
+}
+
+task cleanResources(type: Delete) {
+  fileTree(project.layout.projectDirectory.dir('src/main/resources'))
+    .filter {
+      it.name.endsWith(".so") || it.name.endsWith(".dll") || it.name.endsWith(".dylib")
+    }.each {
+      println("Deleting: ${it}")
+      delete it
+    }
+}
+
+clean.dependsOn cleanResources

jvm-libs/linea/blob-decompressor/src/main/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressor.kt

@@ -0,0 +1,105 @@
+package net.consensys.linea.blob
+
+import build.linea.jvm.ResourcesUtil.copyResourceToTmpDir
+import com.sun.jna.Library
+import com.sun.jna.Native
+import java.nio.file.Path
+
+class DecompressionException(message: String) : RuntimeException(message)
+
+interface BlobDecompressor {
+  fun decompress(blob: ByteArray): ByteArray
+}
+
+internal class Adapter(
+  private val delegate: GoNativeBlobDecompressorJnaBinding,
+  private val maxExpectedCompressionRatio: Int = 10,
+  dictionaries: List<Path>
+) : BlobDecompressor {
+  init {
+    delegate.Init()
+
+    val paths = dictionaries.joinToString(separator = ":") { path -> path.toString() }
+
+    if (delegate.LoadDictionaries(paths) != dictionaries.size) {
+      throw DecompressionException("Failed to load dictionaries '$paths', error='${delegate.Error()}'")
+    }
+  }
+
+  override fun decompress(blob: ByteArray): ByteArray {
+    val decompressionBuffer = ByteArray(blob.size * maxExpectedCompressionRatio)
+    val decompressedSize = delegate.Decompress(blob, blob.size, decompressionBuffer, decompressionBuffer.size)
+    if (decompressedSize < 0) {
+      throw DecompressionException("Decompression failed, error='${delegate.Error()}'")
+    }
+    return decompressionBuffer.copyOf(decompressedSize)
+  }
+}
+
+internal interface GoNativeBlobDecompressorJnaBinding {
+
+  /**
+   * Init initializes the Decompressor. Must be run before anything else.
+   */
+  fun Init()
+
+  /**
+   * LoadDictionaries attempts to cache dictionaries from given paths, separated by colons,
+   * e.g. "../compressor_dict.bin:./other_dict"
+   * Returns the number of dictionaries successfully loaded, and -1 in case of failure, in which case Error() will
+   * return a description of the error.
+   *
+   * @param dictPaths a colon-separated list of paths to dictionaries, to be loaded into the decompressor
+   * @return the number of dictionaries loaded if successful, -1 if not.
+   */
+  fun LoadDictionaries(dictPaths: String): Int
+
+  /**
+
+   * Decompress processes a Linea blob and outputs an RLP encoded list of blocks.
+   * Due to information loss during pre-compression encoding, two pieces of information are represented "hackily":
+   * The block hash is in the ParentHash field.
+   * The transaction from address is in the signature.R field.
+   *
+   * Returns the number of bytes in out, or -1 in case of failure
+   * If -1 is returned, the Error() method will return a string describing the error.
+   *
+   * @param blob to be decompressed
+   * @param blob_len length of the blob
+   * @param out buffer to write the decompressed data
+   * @param out_max_len maximum length of the out buffer
+   * @return number of bytes in out, or -1 in case of failure
+   */
+  fun Decompress(blob: ByteArray, blob_len: Int, out: ByteArray, out_max_len: Int): Int
+
+  /**
+   * Error returns the last error message. Should be checked if Write returns false.
+   */
+  fun Error(): String?
+}
+
+internal interface GoNativeBlobDecompressorJnaLib : GoNativeBlobDecompressorJnaBinding, Library
+
+enum class BlobDecompressorVersion(val version: String) {
+  V1_1_0("v1.1.0")
+}
+
+class GoNativeBlobDecompressorFactory {
+  companion object {
+    private const val DICTIONARY_NAME = "compressor_dict.bin"
+    private val dictionaryPath = copyResourceToTmpDir(DICTIONARY_NAME)
+
+    private fun getLibFileName(version: String) = "blob_decompressor_jna_$version"
+
+    fun getInstance(
+      version: BlobDecompressorVersion
+    ): BlobDecompressor {
+      return Native.load(
+        Native.extractFromResourcePath(getLibFileName(version.version)).toString(),
+        GoNativeBlobDecompressorJnaLib::class.java
+      ).let {
+        Adapter(delegate = it, dictionaries = listOf(dictionaryPath))
+      }
+    }
+  }
+}

jvm-libs/linea/blob-decompressor/src/main/resources/.gitignore

@@ -0,0 +1,4 @@
+linux-aarch64/*
+linux-x86-64/*
+darwin-aarch64/*
+darwin-x86-64/*

jvm-libs/linea/blob-decompressor/src/test/kotlin/net/consensys/linea/blob/BlobDecompressorDataDecodingTest.kt

@@ -0,0 +1,67 @@
+package net.consensys.linea.blob
+
+import net.consensys.linea.testing.filesystem.findPathTo
+import org.apache.tuweni.bytes.Bytes
+import org.hyperledger.besu.ethereum.core.Block
+import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions
+import org.hyperledger.besu.ethereum.rlp.RLP
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Disabled
+import kotlin.io.path.readBytes
+
+class BlobDecompressorDataDecodingTest {
+  private lateinit var decompressor: BlobDecompressor
+
+  @BeforeEach
+  fun beforeEach() {
+    decompressor = GoNativeBlobDecompressorFactory.getInstance(BlobDecompressorVersion.V1_1_0)
+  }
+
+  @Disabled("Until Besu supports deserializing transactions without signatures validation")
+  fun `can deserialize native lib testdata blobs`() {
+    val blob = findPathTo("prover")!!
+      .resolve("lib/compressor/blob/testdata/v0/sample-blob-0151eda71505187b5.bin")
+      .readBytes()
+    val decompressedBlob = decompressor.decompress(blob)
+    val blocksRlpEncoded = rlpDecodeAsListOfBytes(decompressedBlob)
+    blocksRlpEncoded.forEachIndexed { index, blockRlp ->
+      val rlpInput = RLP.input(Bytes.wrap(blockRlp))
+      val decodedBlock = Block.readFrom(rlpInput, MainnetBlockHeaderFunctions())
+      println("$index: $decodedBlock")
+    }
+  }
+
+  @Disabled("for local dev validation")
+  fun `can decode  RLP`() {
+    val blockBytes = Bytes.wrap(
+      // INSERT HERE THE RLP ENCODED BLOCK
+      // 0x01ff.decodeHex()
+    )
+    RLP.validate(blockBytes)
+    val rlpInput = RLP.input(blockBytes)
+    val decodedBlock = Block.readFrom(rlpInput, MainnetBlockHeaderFunctions())
+    println(decodedBlock)
+  }
+
+  private fun rlpEncode(list: List<ByteArray>): ByteArray {
+    return RLP.encode { rlpWriter ->
+      rlpWriter.startList()
+      list.forEach { bytes ->
+        rlpWriter.writeBytes(Bytes.wrap(bytes))
+      }
+      rlpWriter.endList()
+    }.toArray()
+  }
+
+  private fun rlpDecodeAsListOfBytes(rlpEncoded: ByteArray): List<ByteArray> {
+    val decodedBytes = mutableListOf<ByteArray>()
+    RLP.input(Bytes.wrap(rlpEncoded), true).also { rlpInput ->
+      rlpInput.enterList()
+      while (!rlpInput.isEndOfCurrentList) {
+        decodedBytes.add(rlpInput.readBytes().toArray())
+      }
+      rlpInput.leaveList()
+    }
+    return decodedBytes
+  }
+}

jvm-libs/linea/blob-decompressor/src/test/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressorTest.kt

@@ -0,0 +1,41 @@
+package net.consensys.linea.blob
+
+import net.consensys.linea.nativecompressor.CompressorTestData
+import org.assertj.core.api.Assertions.assertThat
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+
+class GoNativeBlobDecompressorTest {
+  private val blobCompressedLimit = 10 * 1024
+  private lateinit var compressor: GoNativeBlobCompressor
+  private lateinit var decompressor: BlobDecompressor
+
+  @BeforeEach
+  fun beforeEach() {
+    compressor = GoNativeBlobCompressorFactory
+      .getInstance(BlobCompressorVersion.V1_0_1)
+      .apply {
+        Init(
+          dataLimit = blobCompressedLimit,
+          dictPath = GoNativeBlobCompressorFactory.dictionaryPath.toAbsolutePath().toString()
+        )
+        Reset()
+      }
+    decompressor = GoNativeBlobDecompressorFactory.getInstance(BlobDecompressorVersion.V1_1_0)
+  }
+
+  @Test
+  fun `when blocks are compressed with compressor shall decompress them back`() {
+    val blocks = CompressorTestData.blocksRlpEncoded.slice(0..1)
+    assertTrue(compressor.Write(blocks[0], blocks[0].size))
+    assertTrue(compressor.Write(blocks[1], blocks[1].size))
+
+    val compressedData = ByteArray(compressor.Len())
+    compressor.Bytes(compressedData)
+
+    val decompressedBlob = decompressor.decompress(compressedData)
+    assertThat(decompressedBlob.size).isGreaterThan(compressedData.size)
+    // TODO: assert decompressedDataBuffer content
+  }
+}