upgrade to libp2p 0.52 (sigp#4431)

Upgrade libp2p to v0.52 - **Workflows**: remove installation of `protoc` - **Book**: remove installation of `protoc` - **`Dockerfile`s and `cross`**: remove custom base `Dockerfile` for cross since it's no longer needed. Remove `protoc` from remaining `Dockerfiles`s - **Upgrade `discv5` to `v0.3.1`:** we have some cool stuff in there: no longer needs `protoc` and faster ip updates on cold start - **Upgrade `prometheus` to `0.21.0`**, now it no longer needs encoding checks - **things that look like refactors:** bunch of api types were renamed and need to be accessed in a different (clearer) way - **Lighthouse network** - connection limits is now a behaviour - banned peers no longer exist on the swarm level, but at the behaviour level - `connection_event_buffer_size` now is handled per connection with a buffer size of 4 - `mplex` is deprecated and was removed - rpc handler now logs the peer to which it belongs Tried to keep as much behaviour unchanged as possible. However, there is a great deal of improvements we can do _after_ this upgrade: - Smart connection limits: Connection limits have been checked only based on numbers, we can now use information about the incoming peer to decide if we want it - More powerful peer management: Dial attempts from other behaviours can be rejected early - Incoming connections can be rejected early - Banning can be returned exclusively to the peer management: We should not get connections to banned peers anymore making use of this - TCP Nat updates: We might be able to take advantage of confirmed external addresses to check out tcp ports/ips Co-authored-by: Age Manning <Age@AgeManning.com> Co-authored-by: Akihito Nakano <sora.akatsuki@gmail.com>
Woodpile37 · Jan 6, 2024 · bf11f93 · bf11f93
1 parent 223e1ec
commit bf11f93
Show file tree

Hide file tree

Showing 27 changed files with 391 additions and 371 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -79,15 +79,6 @@ jobs:
               if: startsWith(matrix.arch, 'x86_64-windows')
               run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV
 
-            # ==============================
-            # Windows & Mac dependencies
-            # ==============================
-            - name: Install Protoc
-              if: contains(matrix.arch, 'darwin') || contains(matrix.arch, 'windows')
-              uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-              with:
-                repo-token: ${{ secrets.GITHUB_TOKEN }}
-
             # ==============================
             #       Builds
             # ==============================

diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml
@@ -60,10 +60,6 @@ jobs:
     - name: Get latest version of stable Rust
       if: env.SELF_HOSTED_RUNNERS == false
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install Foundry (anvil)
       uses: foundry-rs/foundry-toolchain@v1
     - name: Run tests in release
@@ -83,7 +79,7 @@ jobs:
         node-version: '14'
     - name: Install windows build tools
       run: |
-        choco install python protoc visualstudio2019-workload-vctools -y
+        choco install python visualstudio2019-workload-vctools -y
         npm config set msvs_version 2019
     - name: Install Foundry (anvil)
       uses: foundry-rs/foundry-toolchain@v1
@@ -108,10 +104,6 @@ jobs:
     - name: Get latest version of stable Rust
       if: env.SELF_HOSTED_RUNNERS == false
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run beacon_chain tests for all known forks
       run: make test-beacon-chain
   op-pool-tests:
@@ -122,10 +114,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run operation_pool tests for all known forks
       run: make test-op-pool
   slasher-tests:
@@ -148,10 +136,6 @@ jobs:
     - name: Get latest version of stable Rust
       if: env.SELF_HOSTED_RUNNERS == false
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install Foundry (anvil)
       uses: foundry-rs/foundry-toolchain@v1
     - name: Run tests in debug
@@ -164,10 +148,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run state_transition_vectors in release.
       run: make run-state-transition-tests
   ef-tests-ubuntu:
@@ -180,10 +160,6 @@ jobs:
     - name: Get latest version of stable Rust
       if: env.SELF_HOSTED_RUNNERS == false
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run consensus-spec-tests with blst, milagro and fake_crypto
       run: make test-ef
   dockerfile-ubuntu:
@@ -206,10 +182,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install Foundry (anvil)
       uses: foundry-rs/foundry-toolchain@v1
     - name: Run the beacon chain sim that starts from an eth1 contract
@@ -222,10 +194,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install Foundry (anvil)
       uses: foundry-rs/foundry-toolchain@v1
     - name: Run the beacon chain sim and go through the merge transition
@@ -238,10 +206,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run the beacon chain sim without an eth1 connection
       run: cargo run --release --bin simulator no-eth1-sim
   syncing-simulator-ubuntu:
@@ -252,10 +216,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install Foundry (anvil)
       uses: foundry-rs/foundry-toolchain@v1
     - name: Run the syncing simulator
@@ -268,10 +228,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install geth
       run: |
           sudo add-apt-repository -y ppa:ethereum/ethereum
@@ -303,10 +259,6 @@ jobs:
         dotnet-version: '6.0.201'
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run exec engine integration tests in release
       run: make test-exec-engine
   check-benchmarks:
@@ -317,10 +269,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Typecheck benchmark code without running it
       run: make check-benches
   clippy:
@@ -331,10 +279,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Get latest version of stable Rust
       run: rustup update stable
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Lint code for quality and style with Clippy
       run: make lint
     - name: Certify Cargo.lock freshness
@@ -347,10 +291,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Install Rust @ MSRV (${{ needs.extract-msrv.outputs.MSRV }})
       run: rustup override set ${{ needs.extract-msrv.outputs.MSRV }}
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Run cargo check
       run: cargo check --workspace
   arbitrary-check:
@@ -389,10 +329,6 @@ jobs:
     - uses: actions/checkout@v3
     - name: Install Rust (${{ env.PINNED_NIGHTLY }})
       run: rustup toolchain install $PINNED_NIGHTLY
-    - name: Install Protoc
-      uses: arduino/setup-protoc@e52d9eb8f7b63115df1ac544a1376fdbf5a39612
-      with:
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
     - name: Install cargo-udeps
       run: cargo install cargo-udeps --locked --force
     - name: Create Cargo config dir
@@ -410,7 +346,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Install dependencies
-      run: sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang protobuf-compiler
+      run: sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang
     - name: Use Rust beta
       run: rustup override set beta
     - name: Run make

diff --git a/Cross.toml b/Cross.toml
@@ -1,5 +1,5 @@
 [target.x86_64-unknown-linux-gnu]
-dockerfile = './scripts/cross/Dockerfile'
+pre-build = ["apt-get install -y cmake clang-3.9"]
 
 [target.aarch64-unknown-linux-gnu]
-dockerfile = './scripts/cross/Dockerfile'
+pre-build = ["apt-get install -y cmake clang-3.9"]
diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs
@@ -7,12 +7,9 @@ use directory::DEFAULT_ROOT_DIR;
 use eth2::{BeaconNodeHttpClient, Timeouts};
 use lighthouse_network::{
     discv5::enr::{CombinedKey, EnrBuilder},
-    libp2p::{
-        core::connection::ConnectionId,
-        swarm::{
-            behaviour::{ConnectionEstablished, FromSwarm},
-            NetworkBehaviour,
-        },
+    libp2p::swarm::{
+        behaviour::{ConnectionEstablished, FromSwarm},
+        ConnectionId, NetworkBehaviour,
     },
     rpc::methods::{MetaData, MetaDataV2},
     types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield, SyncState},
@@ -167,7 +164,7 @@ pub async fn create_api_server_on_port<T: BeaconChainTypes>(
         local_addr: EXTERNAL_ADDR.parse().unwrap(),
         send_back_addr: EXTERNAL_ADDR.parse().unwrap(),
     };
-    let connection_id = ConnectionId::new(1);
+    let connection_id = ConnectionId::new_unchecked(1);
     pm.on_swarm_event(FromSwarm::ConnectionEstablished(ConnectionEstablished {
         peer_id,
         connection_id,

diff --git a/beacon_node/http_metrics/src/metrics.rs b/beacon_node/http_metrics/src/metrics.rs
@@ -1,6 +1,6 @@
 use crate::Context;
 use beacon_chain::BeaconChainTypes;
-use lighthouse_metrics::{Encoder, TextEncoder};
+use lighthouse_metrics::TextEncoder;
 use lighthouse_network::prometheus_client::encoding::text::encode;
 use malloc_utils::scrape_allocator_metrics;
 
@@ -9,7 +9,7 @@ pub use lighthouse_metrics::*;
 pub fn gather_prometheus_metrics<T: BeaconChainTypes>(
     ctx: &Context<T>,
 ) -> std::result::Result<String, String> {
-    let mut buffer = vec![];
+    let mut buffer = String::new();
     let encoder = TextEncoder::new();
 
     // There are two categories of metrics:
@@ -50,7 +50,7 @@ pub fn gather_prometheus_metrics<T: BeaconChainTypes>(
     }
 
     encoder
-        .encode(&lighthouse_metrics::gather(), &mut buffer)
+        .encode_utf8(&lighthouse_metrics::gather(), &mut buffer)
         .unwrap();
     // encode gossipsub metrics also if they exist
     if let Some(registry) = ctx.gossipsub_registry.as_ref() {
@@ -59,5 +59,5 @@ pub fn gather_prometheus_metrics<T: BeaconChainTypes>(
         }
     }
 
-    String::from_utf8(buffer).map_err(|e| format!("Failed to encode prometheus info: {:?}", e))
+    Ok(buffer)
 }
diff --git a/beacon_node/lighthouse_network/Cargo.toml b/beacon_node/lighthouse_network/Cargo.toml
@@ -5,7 +5,7 @@ authors = ["Sigma Prime <contact@sigmaprime.io>"]
 edition = "2021"
 
 [dependencies]
-discv5 = { version = "0.3.0", features = ["libp2p"]}
+discv5 = { version = "0.3.1", features = ["libp2p"] }
 unsigned-varint = { version = "0.6.0", features = ["codec"] }
 types = { path =  "../../consensus/types" }
 ssz_types = "0.5.3"
@@ -40,15 +40,15 @@ directory = { path = "../../common/directory" }
 regex = "1.5.5"
 strum = { version = "0.24.0", features = ["derive"] }
 superstruct = "0.5.0"
-prometheus-client = "0.18.0"
+prometheus-client = "0.21.0"
 unused_port = { path = "../../common/unused_port" }
 delay_map = "0.3.0"
 void = "1"
 
 [dependencies.libp2p]
-version = "0.50.0"
+version = "0.52"
 default-features = false
-features = ["websocket", "identify", "mplex", "yamux", "noise", "gossipsub", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa"]
+features = ["websocket", "identify", "yamux", "noise", "gossipsub", "dns", "tcp", "tokio", "plaintext", "secp256k1", "macros", "ecdsa"]
 
 [dev-dependencies]
 slog-term = "2.6.0"

diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs
@@ -6,10 +6,7 @@ use directory::{
     DEFAULT_BEACON_NODE_DIR, DEFAULT_HARDCODED_NETWORK, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR,
 };
 use discv5::{Discv5Config, Discv5ConfigBuilder};
-use libp2p::gossipsub::{
-    FastMessageId, GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId,
-    RawGossipsubMessage, ValidationMode,
-};
+use libp2p::gossipsub;
 use libp2p::Multiaddr;
 use serde_derive::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
@@ -83,7 +80,7 @@ pub struct Config {
 
     /// Gossipsub configuration parameters.
     #[serde(skip)]
-    pub gs_config: GossipsubConfig,
+    pub gs_config: gossipsub::Config,
 
     /// Discv5 configuration parameters.
     #[serde(skip)]
@@ -265,7 +262,7 @@ impl Default for Config {
 
         // Note: Using the default config here. Use `gossipsub_config` function for getting
         // Lighthouse specific configuration for gossipsub.
-        let gs_config = GossipsubConfigBuilder::default()
+        let gs_config = gossipsub::ConfigBuilder::default()
             .build()
             .expect("valid gossipsub configuration");
 
@@ -416,16 +413,16 @@ impl From<u8> for NetworkLoad {
 }
 
 /// Return a Lighthouse specific `GossipsubConfig` where the `message_id_fn` depends on the current fork.
-pub fn gossipsub_config(network_load: u8, fork_context: Arc<ForkContext>) -> GossipsubConfig {
+pub fn gossipsub_config(network_load: u8, fork_context: Arc<ForkContext>) -> gossipsub::Config {
     // The function used to generate a gossipsub message id
     // We use the first 8 bytes of SHA256(topic, data) for content addressing
-    let fast_gossip_message_id = |message: &RawGossipsubMessage| {
+    let fast_gossip_message_id = |message: &gossipsub::RawMessage| {
         let data = [message.topic.as_str().as_bytes(), &message.data].concat();
-        FastMessageId::from(&Sha256::digest(data)[..8])
+        gossipsub::FastMessageId::from(&Sha256::digest(data)[..8])
     };
     fn prefix(
         prefix: [u8; 4],
-        message: &GossipsubMessage,
+        message: &gossipsub::Message,
         fork_context: Arc<ForkContext>,
     ) -> Vec<u8> {
         let topic_bytes = message.topic.as_str().as_bytes();
@@ -453,8 +450,8 @@ pub fn gossipsub_config(network_load: u8, fork_context: Arc<ForkContext>) -> Gos
     }
 
     let is_merge_enabled = fork_context.fork_exists(ForkName::Merge);
-    let gossip_message_id = move |message: &GossipsubMessage| {
-        MessageId::from(
+    let gossip_message_id = move |message: &gossipsub::Message| {
+        gossipsub::MessageId::from(
             &Sha256::digest(
                 prefix(MESSAGE_DOMAIN_VALID_SNAPPY, message, fork_context.clone()).as_slice(),
             )[..20],
@@ -463,7 +460,7 @@ pub fn gossipsub_config(network_load: u8, fork_context: Arc<ForkContext>) -> Gos
 
     let load = NetworkLoad::from(network_load);
 
-    GossipsubConfigBuilder::default()
+    gossipsub::ConfigBuilder::default()
         .max_transmit_size(gossip_max_size(is_merge_enabled))
         .heartbeat_interval(load.heartbeat_interval)
         .mesh_n(load.mesh_n)
@@ -476,7 +473,7 @@ pub fn gossipsub_config(network_load: u8, fork_context: Arc<ForkContext>) -> Gos
         .max_messages_per_rpc(Some(500)) // Responses to IWANT can be quite large
         .history_gossip(load.history_gossip)
         .validate_messages() // require validation before propagation
-        .validation_mode(ValidationMode::Anonymous)
+        .validation_mode(gossipsub::ValidationMode::Anonymous)
         .duplicate_cache_time(DUPLICATE_CACHE_TIME)
         .message_id_fn(gossip_message_id)
         .fast_message_id_fn(fast_gossip_message_id)