diff --git a/jvm-libs/linea/blob-compressor/src/test/resources/net/consensys/linea/nativecompressor/rlp_blocks.bin b/jvm-libs/linea/blob-compressor/src/test/resources/net/consensys/linea/nativecompressor/rlp_blocks.bin deleted file mode 100644 index 82c04c3ff..000000000 Binary files a/jvm-libs/linea/blob-compressor/src/test/resources/net/consensys/linea/nativecompressor/rlp_blocks.bin and /dev/null differ diff --git a/jvm-libs/linea/blob-decompressor/build.gradle b/jvm-libs/linea/blob-decompressor/build.gradle new file mode 100644 index 000000000..95718700c --- /dev/null +++ b/jvm-libs/linea/blob-decompressor/build.gradle @@ -0,0 +1,59 @@ +plugins { + id 'net.consensys.zkevm.kotlin-library-conventions' + id 'net.consensys.zkevm.linea-native-libs-helper' + alias(libs.plugins.jreleaser) + id 'java-test-fixtures' +} + +description = 'Java JNA wrapper for Linea Blob Decompressor Library implemented in GO Lang' +apply from: rootProject.file("gradle/publishing.gradle") + +dependencies { + implementation "net.java.dev.jna:jna:${libs.versions.jna.get()}" + implementation project(":jvm-libs:generic:extensions:kotlin") + + testImplementation project(":jvm-libs:linea:blob-compressor") + testImplementation(testFixtures(project(":jvm-libs:linea:blob-compressor"))) + testImplementation(project(":jvm-libs:linea:testing:file-system")) + testImplementation("io.tmio:tuweni-bytes:${libs.versions.tuweni.get()}") + testImplementation("org.hyperledger.besu:besu-datatypes:${libs.versions.besu.get()}") + testImplementation "org.hyperledger.besu:evm:${libs.versions.besu.get()}" + testImplementation("org.hyperledger.besu.internal:core:${libs.versions.besu.get()}") + testImplementation("org.hyperledger.besu:plugin-api:${libs.versions.besu.get()}") + testImplementation("org.hyperledger.besu.internal:rlp:${libs.versions.besu.get()}") +} + +jar { + dependsOn configurations.runtimeClasspath +} + +test { + // we cannot have more 1 compressor per JVM, hence we disable parallel execution + // because multiple threads would cause issues with the native library + systemProperties["junit.jupiter.execution.parallel.enabled"] = false + maxParallelForks = 1 +} + +def libsZipDownloadOutputDir = project.parent.layout.buildDirectory.asFile.get().absolutePath + +task downloadNativeLibs { + doLast { + fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v1.1.0-test8/linea-blob-libs-v1.1.0-test8.zip", "blob_decompressor", libsZipDownloadOutputDir) + } +} + +compileKotlin { + dependsOn tasks.downloadNativeLibs +} + +task cleanResources(type: Delete) { + fileTree(project.layout.projectDirectory.dir('src/main/resources')) + .filter { + it.name.endsWith(".so") || it.name.endsWith(".dll") || it.name.endsWith(".dylib") + }.each { + println("Deleting: ${it}") + delete it + } +} + +clean.dependsOn cleanResources diff --git a/jvm-libs/linea/blob-decompressor/src/main/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressor.kt b/jvm-libs/linea/blob-decompressor/src/main/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressor.kt new file mode 100644 index 000000000..fa78dca2c --- /dev/null +++ b/jvm-libs/linea/blob-decompressor/src/main/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressor.kt @@ -0,0 +1,105 @@ +package net.consensys.linea.blob + +import build.linea.jvm.ResourcesUtil.copyResourceToTmpDir +import com.sun.jna.Library +import com.sun.jna.Native +import java.nio.file.Path + +class DecompressionException(message: String) : RuntimeException(message) + +interface BlobDecompressor { + fun decompress(blob: ByteArray): ByteArray +} + +internal class Adapter( + private val delegate: GoNativeBlobDecompressorJnaBinding, + private val maxExpectedCompressionRatio: Int = 10, + dictionaries: List +) : BlobDecompressor { + init { + delegate.Init() + + val paths = dictionaries.joinToString(separator = ":") { path -> path.toString() } + + if (delegate.LoadDictionaries(paths) != dictionaries.size) { + throw DecompressionException("Failed to load dictionaries '$paths', error='${delegate.Error()}'") + } + } + + override fun decompress(blob: ByteArray): ByteArray { + val decompressionBuffer = ByteArray(blob.size * maxExpectedCompressionRatio) + val decompressedSize = delegate.Decompress(blob, blob.size, decompressionBuffer, decompressionBuffer.size) + if (decompressedSize < 0) { + throw DecompressionException("Decompression failed, error='${delegate.Error()}'") + } + return decompressionBuffer.copyOf(decompressedSize) + } +} + +internal interface GoNativeBlobDecompressorJnaBinding { + + /** + * Init initializes the Decompressor. Must be run before anything else. + */ + fun Init() + + /** + * LoadDictionaries attempts to cache dictionaries from given paths, separated by colons, + * e.g. "../compressor_dict.bin:./other_dict" + * Returns the number of dictionaries successfully loaded, and -1 in case of failure, in which case Error() will + * return a description of the error. + * + * @param dictPaths a colon-separated list of paths to dictionaries, to be loaded into the decompressor + * @return the number of dictionaries loaded if successful, -1 if not. + */ + fun LoadDictionaries(dictPaths: String): Int + + /** + + * Decompress processes a Linea blob and outputs an RLP encoded list of blocks. + * Due to information loss during pre-compression encoding, two pieces of information are represented "hackily": + * The block hash is in the ParentHash field. + * The transaction from address is in the signature.R field. + * + * Returns the number of bytes in out, or -1 in case of failure + * If -1 is returned, the Error() method will return a string describing the error. + * + * @param blob to be decompressed + * @param blob_len length of the blob + * @param out buffer to write the decompressed data + * @param out_max_len maximum length of the out buffer + * @return number of bytes in out, or -1 in case of failure + */ + fun Decompress(blob: ByteArray, blob_len: Int, out: ByteArray, out_max_len: Int): Int + + /** + * Error returns the last error message. Should be checked if Write returns false. + */ + fun Error(): String? +} + +internal interface GoNativeBlobDecompressorJnaLib : GoNativeBlobDecompressorJnaBinding, Library + +enum class BlobDecompressorVersion(val version: String) { + V1_1_0("v1.1.0") +} + +class GoNativeBlobDecompressorFactory { + companion object { + private const val DICTIONARY_NAME = "compressor_dict.bin" + private val dictionaryPath = copyResourceToTmpDir(DICTIONARY_NAME) + + private fun getLibFileName(version: String) = "blob_decompressor_jna_$version" + + fun getInstance( + version: BlobDecompressorVersion + ): BlobDecompressor { + return Native.load( + Native.extractFromResourcePath(getLibFileName(version.version)).toString(), + GoNativeBlobDecompressorJnaLib::class.java + ).let { + Adapter(delegate = it, dictionaries = listOf(dictionaryPath)) + } + } + } +} diff --git a/jvm-libs/linea/blob-decompressor/src/main/resources/.gitignore b/jvm-libs/linea/blob-decompressor/src/main/resources/.gitignore new file mode 100644 index 000000000..272f436b7 --- /dev/null +++ b/jvm-libs/linea/blob-decompressor/src/main/resources/.gitignore @@ -0,0 +1,4 @@ +linux-aarch64/* +linux-x86-64/* +darwin-aarch64/* +darwin-x86-64/* diff --git a/jvm-libs/linea/blob-decompressor/src/main/resources/compressor_dict.bin b/jvm-libs/linea/blob-decompressor/src/main/resources/compressor_dict.bin new file mode 100644 index 000000000..62296498c Binary files /dev/null and b/jvm-libs/linea/blob-decompressor/src/main/resources/compressor_dict.bin differ diff --git a/jvm-libs/linea/blob-decompressor/src/test/kotlin/net/consensys/linea/blob/BlobDecompressorDataDecodingTest.kt b/jvm-libs/linea/blob-decompressor/src/test/kotlin/net/consensys/linea/blob/BlobDecompressorDataDecodingTest.kt new file mode 100644 index 000000000..f93616475 --- /dev/null +++ b/jvm-libs/linea/blob-decompressor/src/test/kotlin/net/consensys/linea/blob/BlobDecompressorDataDecodingTest.kt @@ -0,0 +1,67 @@ +package net.consensys.linea.blob + +import net.consensys.linea.testing.filesystem.findPathTo +import org.apache.tuweni.bytes.Bytes +import org.hyperledger.besu.ethereum.core.Block +import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions +import org.hyperledger.besu.ethereum.rlp.RLP +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Disabled +import kotlin.io.path.readBytes + +class BlobDecompressorDataDecodingTest { + private lateinit var decompressor: BlobDecompressor + + @BeforeEach + fun beforeEach() { + decompressor = GoNativeBlobDecompressorFactory.getInstance(BlobDecompressorVersion.V1_1_0) + } + + @Disabled("Until Besu supports deserializing transactions without signatures validation") + fun `can deserialize native lib testdata blobs`() { + val blob = findPathTo("prover")!! + .resolve("lib/compressor/blob/testdata/v0/sample-blob-0151eda71505187b5.bin") + .readBytes() + val decompressedBlob = decompressor.decompress(blob) + val blocksRlpEncoded = rlpDecodeAsListOfBytes(decompressedBlob) + blocksRlpEncoded.forEachIndexed { index, blockRlp -> + val rlpInput = RLP.input(Bytes.wrap(blockRlp)) + val decodedBlock = Block.readFrom(rlpInput, MainnetBlockHeaderFunctions()) + println("$index: $decodedBlock") + } + } + + @Disabled("for local dev validation") + fun `can decode RLP`() { + val blockBytes = Bytes.wrap( + // INSERT HERE THE RLP ENCODED BLOCK + // 0x01ff.decodeHex() + ) + RLP.validate(blockBytes) + val rlpInput = RLP.input(blockBytes) + val decodedBlock = Block.readFrom(rlpInput, MainnetBlockHeaderFunctions()) + println(decodedBlock) + } + + private fun rlpEncode(list: List): ByteArray { + return RLP.encode { rlpWriter -> + rlpWriter.startList() + list.forEach { bytes -> + rlpWriter.writeBytes(Bytes.wrap(bytes)) + } + rlpWriter.endList() + }.toArray() + } + + private fun rlpDecodeAsListOfBytes(rlpEncoded: ByteArray): List { + val decodedBytes = mutableListOf() + RLP.input(Bytes.wrap(rlpEncoded), true).also { rlpInput -> + rlpInput.enterList() + while (!rlpInput.isEndOfCurrentList) { + decodedBytes.add(rlpInput.readBytes().toArray()) + } + rlpInput.leaveList() + } + return decodedBytes + } +} diff --git a/jvm-libs/linea/blob-decompressor/src/test/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressorTest.kt b/jvm-libs/linea/blob-decompressor/src/test/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressorTest.kt new file mode 100644 index 000000000..c4fc22ec2 --- /dev/null +++ b/jvm-libs/linea/blob-decompressor/src/test/kotlin/net/consensys/linea/blob/GoNativeBlobDecompressorTest.kt @@ -0,0 +1,41 @@ +package net.consensys.linea.blob + +import net.consensys.linea.nativecompressor.CompressorTestData +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class GoNativeBlobDecompressorTest { + private val blobCompressedLimit = 10 * 1024 + private lateinit var compressor: GoNativeBlobCompressor + private lateinit var decompressor: BlobDecompressor + + @BeforeEach + fun beforeEach() { + compressor = GoNativeBlobCompressorFactory + .getInstance(BlobCompressorVersion.V1_0_1) + .apply { + Init( + dataLimit = blobCompressedLimit, + dictPath = GoNativeBlobCompressorFactory.dictionaryPath.toAbsolutePath().toString() + ) + Reset() + } + decompressor = GoNativeBlobDecompressorFactory.getInstance(BlobDecompressorVersion.V1_1_0) + } + + @Test + fun `when blocks are compressed with compressor shall decompress them back`() { + val blocks = CompressorTestData.blocksRlpEncoded.slice(0..1) + assertTrue(compressor.Write(blocks[0], blocks[0].size)) + assertTrue(compressor.Write(blocks[1], blocks[1].size)) + + val compressedData = ByteArray(compressor.Len()) + compressor.Bytes(compressedData) + + val decompressedBlob = decompressor.decompress(compressedData) + assertThat(decompressedBlob.size).isGreaterThan(compressedData.size) + // TODO: assert decompressedDataBuffer content + } +}