Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: use snappy-wasm #6483

Open
wants to merge 14 commits into
base: unstable
Choose a base branch
from
Open

feat: use snappy-wasm #6483

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
0c088c8
feat: use snappy for larger gossip payloads
wemeetagain Jan 30, 2024
68891ee
chore: tweak snappyjs uncompress threshold
wemeetagain Jan 30, 2024
3e24055
Merge branch 'unstable' into cayman/snappy-tweak
wemeetagain Feb 26, 2024
41d4bae
chore: add run factor to snappy benchmark
wemeetagain Feb 27, 2024
864d361
chore: adjust snappyjs thresholds
wemeetagain Feb 27, 2024
53622cd
chore: tweak perf output
wemeetagain Feb 29, 2024
b972a2c
Merge branch 'unstable' into cayman/snappy-tweak
wemeetagain Jul 24, 2024
96333a9
feat: use snappy-wasm
wemeetagain Jul 24, 2024
a667200
Merge branch 'unstable' into cayman/snappy-tweak
wemeetagain Aug 1, 2024
37d359d
Merge branch 'unstable' into cayman/snappy-tweak
wemeetagain Aug 1, 2024
093a469
feat: use @chainsafe/snappy-wasm with prealloc
wemeetagain Aug 5, 2024
33ca8e7
chore: remove snappy dep
wemeetagain Aug 6, 2024
d730129
Merge branch 'unstable' into cayman/snappy-tweak
wemeetagain Oct 9, 2024
b5a8fc1
Merge branch 'unstable' into cayman/snappy-tweak
wemeetagain Oct 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/beacon-node/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@
"multiformats": "^11.0.1",
"prom-client": "^15.1.0",
"qs": "^6.11.1",
"snappyjs": "^0.7.0",
"@chainsafe/snappy-wasm": "^0.5.0",
"strict-event-emitter-types": "^2.0.0",
"systeminformation": "^5.22.9",
"uint8arraylist": "^2.4.7",
Expand Down
25 changes: 18 additions & 7 deletions packages/beacon-node/src/network/gossip/encoding.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
import {compress, uncompress} from "snappyjs";
import xxhashFactory from "xxhash-wasm";
import {Message} from "@libp2p/interface";
import {digest} from "@chainsafe/as-sha256";
import {RPC} from "@chainsafe/libp2p-gossipsub/message";
import {DataTransform} from "@chainsafe/libp2p-gossipsub/types";
import snappyWasm from "@chainsafe/snappy-wasm";
import {intToBytes} from "@lodestar/utils";
import {ForkName} from "@lodestar/params";
import {MESSAGE_DOMAIN_VALID_SNAPPY} from "./constants.js";
Expand All @@ -15,6 +15,10 @@ const xxhash = await xxhashFactory();
// Use salt to prevent msgId from being mined for collisions
const h64Seed = BigInt(Math.floor(Math.random() * 1e9));

// create singleton snappy encoder + decoder
const encoder = new snappyWasm.Encoder();
const decoder = new snappyWasm.Decoder();

// Shared buffer to convert msgId to string
const sharedMsgIdBuf = Buffer.alloc(20);

Expand Down Expand Up @@ -79,11 +83,12 @@ export class DataTransformSnappy implements DataTransform {
* - `outboundTransform()`: compress snappy payload
*/
inboundTransform(topicStr: string, data: Uint8Array): Uint8Array {
const uncompressedData = uncompress(data, this.maxSizePerMessage);
// check uncompressed data length before we actually decompress
const uncompressedDataLength = snappyWasm.decompress_len(data);
if (uncompressedDataLength > this.maxSizePerMessage) {
throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${this.maxSizePerMessage}`);
}

// check uncompressed data length before we extract beacon block root, slot or
// attestation data at later steps
const uncompressedDataLength = uncompressedData.length;
const topic = this.gossipTopicCache.getTopic(topicStr);
const sszType = getGossipSSZType(topic);

Expand All @@ -94,18 +99,24 @@ export class DataTransformSnappy implements DataTransform {
throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${sszType.maxSize}`);
}

// Only after saniy length checks, we can decompress the data
const uncompressedData = Buffer.allocUnsafe(uncompressedDataLength);
decoder.decompress_into(data, uncompressedData);
return uncompressedData;
}

/**
* Takes the data to be published (a topic and associated data) transforms the data. The
* transformed data will then be used to create a `RawGossipsubMessage` to be sent to peers.
*/
// No need to parse topic, everything is snappy compressed
outboundTransform(_topicStr: string, data: Uint8Array): Uint8Array {
if (data.length > this.maxSizePerMessage) {
throw Error(`ssz_snappy encoded data length ${data.length} > ${this.maxSizePerMessage}`);
}
// No need to parse topic, everything is snappy compressed
return compress(data);

const compressedData = Buffer.allocUnsafe(snappyWasm.max_compress_len(data.length));
const compressedLen = encoder.compress_into(data, compressedData);
return compressedData.subarray(0, compressedLen);
}
}
180 changes: 180 additions & 0 deletions packages/beacon-node/test/perf/network/gossip/snappy.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
import {randomBytes} from "node:crypto";
import * as snappyjs from "snappyjs";
import * as snappy from "snappy";
import {itBench} from "@dapplion/benchmark";
import snappyWasm from "@chainsafe/snappy-wasm";

/* 2024-08-05 - Linux 5.15 x86_64 - Node.js v22.4.1

network / gossip / snappy
compress
✔ 100 bytes - compress - snappyjs 335566.9 ops/s 2.980032 us/op - 685 runs 2.54 s
✔ 100 bytes - compress - snappy 388610.3 ops/s 2.573272 us/op - 870 runs 2.74 s
✔ 100 bytes - compress - snappy-wasm 583254.0 ops/s 1.714519 us/op - 476 runs 1.32 s
✔ 100 bytes - compress - snappy-wasm - prealloc 1586695 ops/s 630.2410 ns/op - 481 runs 0.804 s
✔ 200 bytes - compress - snappyjs 298272.8 ops/s 3.352636 us/op - 213 runs 1.22 s
✔ 200 bytes - compress - snappy 419528.0 ops/s 2.383631 us/op - 926 runs 2.71 s
✔ 200 bytes - compress - snappy-wasm 472468.5 ops/s 2.116543 us/op - 577 runs 1.72 s
✔ 200 bytes - compress - snappy-wasm - prealloc 1430445 ops/s 699.0830 ns/op - 868 runs 1.11 s
✔ 300 bytes - compress - snappyjs 265124.9 ops/s 3.771807 us/op - 137 runs 1.02 s
✔ 300 bytes - compress - snappy 361683.9 ops/s 2.764845 us/op - 1332 runs 4.18 s
✔ 300 bytes - compress - snappy-wasm 443688.4 ops/s 2.253834 us/op - 859 runs 2.44 s
✔ 300 bytes - compress - snappy-wasm - prealloc 1213825 ops/s 823.8420 ns/op - 370 runs 0.807 s
✔ 400 bytes - compress - snappyjs 262168.5 ops/s 3.814341 us/op - 358 runs 1.87 s
✔ 400 bytes - compress - snappy 382494.9 ops/s 2.614414 us/op - 1562 runs 4.58 s
✔ 400 bytes - compress - snappy-wasm 406373.2 ops/s 2.460792 us/op - 797 runs 2.46 s
✔ 400 bytes - compress - snappy-wasm - prealloc 1111715 ops/s 899.5110 ns/op - 450 runs 0.906 s
✔ 500 bytes - compress - snappyjs 229213.1 ops/s 4.362753 us/op - 359 runs 2.07 s
✔ 500 bytes - compress - snappy 373695.8 ops/s 2.675973 us/op - 2050 runs 5.99 s
✔ 500 bytes - compress - snappy-wasm 714917.4 ops/s 1.398763 us/op - 960 runs 1.84 s
✔ 500 bytes - compress - snappy-wasm - prealloc 1054619 ops/s 948.2100 ns/op - 427 runs 0.907 s
✔ 1000 bytes - compress - snappyjs 148702.3 ops/s 6.724847 us/op - 171 runs 1.65 s
✔ 1000 bytes - compress - snappy 423688.1 ops/s 2.360227 us/op - 525 runs 1.74 s
✔ 1000 bytes - compress - snappy-wasm 524350.6 ops/s 1.907121 us/op - 273 runs 1.03 s
✔ 1000 bytes - compress - snappy-wasm - prealloc 685191.5 ops/s 1.459446 us/op - 349 runs 1.01 s
✔ 10000 bytes - compress - snappyjs 21716.92 ops/s 46.04704 us/op - 16 runs 1.24 s
✔ 10000 bytes - compress - snappy 98051.32 ops/s 10.19874 us/op - 184 runs 2.39 s
✔ 10000 bytes - compress - snappy-wasm 114681.8 ops/s 8.719783 us/op - 49 runs 0.937 s
✔ 10000 bytes - compress - snappy-wasm - prealloc 111203.6 ops/s 8.992518 us/op - 49 runs 0.953 s
✔ 100000 bytes - compress - snappyjs 2947.313 ops/s 339.2921 us/op - 12 runs 4.74 s
✔ 100000 bytes - compress - snappy 14963.78 ops/s 66.82801 us/op - 70 runs 5.19 s
✔ 100000 bytes - compress - snappy-wasm 19868.33 ops/s 50.33136 us/op - 14 runs 1.21 s
✔ 100000 bytes - compress - snappy-wasm - prealloc 24579.34 ops/s 40.68457 us/op - 13 runs 1.06 s
uncompress
✔ 100 bytes - uncompress - snappyjs 589201.6 ops/s 1.697212 us/op - 242 runs 0.911 s
✔ 100 bytes - uncompress - snappy 537424.1 ops/s 1.860728 us/op - 220 runs 0.910 s
✔ 100 bytes - uncompress - snappy-wasm 634966.2 ops/s 1.574887 us/op - 194 runs 0.808 s
✔ 100 bytes - uncompress - snappy-wasm - prealloc 1846964 ops/s 541.4290 ns/op - 559 runs 0.804 s
✔ 200 bytes - uncompress - snappyjs 395141.8 ops/s 2.530737 us/op - 281 runs 1.22 s
✔ 200 bytes - uncompress - snappy 536862.6 ops/s 1.862674 us/op - 274 runs 1.01 s
✔ 200 bytes - uncompress - snappy-wasm 420251.6 ops/s 2.379527 us/op - 129 runs 0.810 s
✔ 200 bytes - uncompress - snappy-wasm - prealloc 1746167 ops/s 572.6830 ns/op - 529 runs 0.804 s
✔ 300 bytes - uncompress - snappyjs 441676.2 ops/s 2.264102 us/op - 898 runs 2.53 s
✔ 300 bytes - uncompress - snappy 551313.2 ops/s 1.813851 us/op - 336 runs 1.11 s
✔ 300 bytes - uncompress - snappy-wasm 494773.0 ops/s 2.021129 us/op - 203 runs 0.912 s
✔ 300 bytes - uncompress - snappy-wasm - prealloc 1528680 ops/s 654.1590 ns/op - 465 runs 0.805 s
✔ 400 bytes - uncompress - snappyjs 383746.1 ops/s 2.605890 us/op - 235 runs 1.11 s
✔ 400 bytes - uncompress - snappy 515986.6 ops/s 1.938035 us/op - 158 runs 0.809 s
✔ 400 bytes - uncompress - snappy-wasm 392947.8 ops/s 2.544867 us/op - 322 runs 1.32 s
✔ 400 bytes - uncompress - snappy-wasm - prealloc 1425978 ops/s 701.2730 ns/op - 721 runs 1.01 s
✔ 500 bytes - uncompress - snappyjs 330727.5 ops/s 3.023637 us/op - 173 runs 1.02 s
✔ 500 bytes - uncompress - snappy 513874.1 ops/s 1.946002 us/op - 157 runs 0.806 s
✔ 500 bytes - uncompress - snappy-wasm 389263.0 ops/s 2.568957 us/op - 161 runs 0.914 s
✔ 500 bytes - uncompress - snappy-wasm - prealloc 1330936 ops/s 751.3510 ns/op - 672 runs 1.01 s
✔ 1000 bytes - uncompress - snappyjs 241393.9 ops/s 4.142606 us/op - 126 runs 1.03 s
✔ 1000 bytes - uncompress - snappy 491119.6 ops/s 2.036164 us/op - 201 runs 0.911 s
✔ 1000 bytes - uncompress - snappy-wasm 361794.5 ops/s 2.764000 us/op - 148 runs 0.910 s
✔ 1000 bytes - uncompress - snappy-wasm - prealloc 959026.5 ops/s 1.042724 us/op - 390 runs 0.909 s
✔ 10000 bytes - uncompress - snappyjs 40519.03 ops/s 24.67976 us/op - 16 runs 0.913 s
✔ 10000 bytes - uncompress - snappy 202537.6 ops/s 4.937355 us/op - 796 runs 4.43 s
✔ 10000 bytes - uncompress - snappy-wasm 165017.6 ops/s 6.059960 us/op - 52 runs 0.822 s
✔ 10000 bytes - uncompress - snappy-wasm - prealloc 175061.5 ops/s 5.712277 us/op - 130 runs 1.25 s
✔ 100000 bytes - uncompress - snappyjs 4030.391 ops/s 248.1149 us/op - 12 runs 3.71 s
✔ 100000 bytes - uncompress - snappy 35459.43 ops/s 28.20124 us/op - 41 runs 1.67 s
✔ 100000 bytes - uncompress - snappy-wasm 22449.16 ops/s 44.54509 us/op - 13 runs 1.11 s
✔ 100000 bytes - uncompress - snappy-wasm - prealloc 27169.50 ops/s 36.80598 us/op - 13 runs 0.997 s

*/

describe("network / gossip / snappy", () => {
const msgLens = [100, 200, 300, 400, 500, 1000, 10000, 100000];
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I used to see blocks with 2MB, need to add 2MB, 5MB and 10MB (max size of gossip message) message length

describe("compress", () => {
const encoder = new snappyWasm.Encoder();

for (const msgLen of msgLens) {
const uncompressed = randomBytes(msgLen);
const RUNS_FACTOR = 1000;

itBench({
id: `${msgLen} bytes - compress - snappyjs`,
runsFactor: RUNS_FACTOR,
fn: () => {
for (let i = 0; i < RUNS_FACTOR; i++) {
snappyjs.compress(uncompressed);
}
},
});

itBench({
id: `${msgLen} bytes - compress - snappy`,
runsFactor: RUNS_FACTOR,
fn: () => {
for (let i = 0; i < RUNS_FACTOR; i++) {
snappy.compressSync(uncompressed);
}
},
});

itBench({
id: `${msgLen} bytes - compress - snappy-wasm`,
runsFactor: RUNS_FACTOR,
fn: () => {
for (let i = 0; i < RUNS_FACTOR; i++) {
encoder.compress(uncompressed);
}
},
});

itBench({
id: `${msgLen} bytes - compress - snappy-wasm - prealloc`,
runsFactor: RUNS_FACTOR,
fn: () => {
for (let i = 0; i < RUNS_FACTOR; i++) {
let out = Buffer.allocUnsafe(snappyWasm.max_compress_len(uncompressed.length));
const len = encoder.compress_into(uncompressed, out);
out = out.subarray(0, len);
}
},
});
}
});
describe("uncompress", () => {
const decoder = new snappyWasm.Decoder();

for (const msgLen of msgLens) {
const uncompressed = randomBytes(msgLen);
const compressed = snappyjs.compress(uncompressed);
const RUNS_FACTOR = 1000;

itBench({
id: `${msgLen} bytes - uncompress - snappyjs`,
runsFactor: RUNS_FACTOR,
fn: () => {
for (let i = 0; i < RUNS_FACTOR; i++) {
snappyjs.uncompress(compressed);
}
},
});

itBench({
id: `${msgLen} bytes - uncompress - snappy`,
runsFactor: RUNS_FACTOR,
fn: () => {
for (let i = 0; i < RUNS_FACTOR; i++) {
snappy.uncompressSync(compressed);
}
},
});

itBench({
id: `${msgLen} bytes - uncompress - snappy-wasm`,
runsFactor: RUNS_FACTOR,
fn: () => {
for (let i = 0; i < RUNS_FACTOR; i++) {
decoder.decompress(compressed);
}
},
});

itBench({
id: `${msgLen} bytes - uncompress - snappy-wasm - prealloc`,
runsFactor: RUNS_FACTOR,
fn: () => {
for (let i = 0; i < RUNS_FACTOR; i++) {
decoder.decompress_into(compressed, Buffer.allocUnsafe(snappyWasm.decompress_len(compressed)));
}
},
});
}
});
});
14 changes: 13 additions & 1 deletion yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -641,6 +641,11 @@
"@chainsafe/pubkey-index-map-linux-x64-gnu" "2.0.0"
"@chainsafe/pubkey-index-map-win32-x64-msvc" "2.0.0"

"@chainsafe/snappy-wasm@^0.5.0":
version "0.5.0"
resolved "https://registry.yarnpkg.com/@chainsafe/snappy-wasm/-/snappy-wasm-0.5.0.tgz#067e534341ef746706e2dbf255bd7604c849be73"
integrity sha512-ydXvhr9p+JjvzSSEyi6XExq8pHugFnrk70mk17T6mhDsklPvaXc+8K90G7TJF+u51lxI/fpv7MahrA5ayjFcSA==

"@chainsafe/ssz@^0.11.1":
version "0.11.1"
resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.11.1.tgz#d4aec883af2ec5196ae67b96242c467da20b2476"
Expand Down Expand Up @@ -11221,11 +11226,18 @@ semver@^6.1.0, semver@^6.2.0:
resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"
integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==

semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3:
semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.3:
version "7.6.3"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143"
integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==

semver@^7.6.0:
version "7.6.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d"
integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==
dependencies:
lru-cache "^6.0.0"

semver@~7.5.4:
version "7.5.4"
resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e"
Expand Down
Loading