Merge pull request #4 from gnosischain/reflow

New Gnosis tests

.circleci/config.yml

@@ -1,79 +1,15 @@
 version: 2.1
-orbs:
-  go: circleci/go@1.5.0
-
-jobs:
-  # This job builds the hive executable and stores it in the workspace.
-  build:
-    docker:
-      - image: cimg/go:1.19
-    steps:
-      # Build it.
-      - checkout
-      - go/load-cache
-      - go/mod-download
-      - go/save-cache
-      - run: {command: 'go build -ldflags="-extldflags=-static" .'}
-      # Store the executable.
-      - persist_to_workspace:
-          root: .
-          paths: ["hive"]
 
-  # This job runs the smoke test simulations. This requires a virtual
-  # machine instead of the container-based build environment because
-  # hive needs to be able to talk to the docker containers it creates.
-  smoke-tests:
-    machine:
-      image: ubuntu-2004:202201-02
-    steps:
-      - checkout
-      - attach_workspace: {at: "/tmp/build"}
-      - run:
-          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum"
-      - run:
-          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum"
+setup: true
 
-  # This job also runs the smoke test simulations, but against a remote dockerd.
-  smoke-tests-remote-docker:
-    docker:
-      - image: cimg/base:2022.04
-    steps:
-      - checkout
-      - attach_workspace: {at: "/tmp/build"}
-      - setup_remote_docker: {version: 20.10.14}
-      - run:
-          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum --loglevel 5"
-      - run:
-          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum --loglevel 5"
-
-  # This job runs the go unit tests.
-  go-test:
-    docker:
-      - image: cimg/go:1.19
-    steps:
-      # Get the source.
-      - checkout
-      - go/load-cache
-      - go/mod-download
-      - go/save-cache
-      # Run the tests.
-      - run:
-          name: "hive module tests"
-          command: "go test -cover ./..."
-      - run:
-          name: "hiveproxy module tests"
-          command: "go test -cover ./..."
-          working_directory: "./hiveproxy"
-      - run:
-          name: "Compile Go simulators"
-          command: ".circleci/compile-simulators.sh"
+orbs:
+  path-filtering: circleci/path-filtering@0.0.1
 
 workflows:
-  main:
+  setup-workflow:
     jobs:
-      - go-test
-      - build
-      - smoke-tests:
-          requires: ["build"]
-      - smoke-tests-remote-docker:
-          requires: ["build"]
+      - path-filtering/filter:
+          mapping: |
+            hivesim-rs/.* hivesim-rs-ci true
+            simulators/portal/.* rust-ci true
+          base-revision: origin/master

.circleci/continue_config.yml

@@ -0,0 +1,136 @@
+version: 2.1
+orbs:
+  go: circleci/go@1.5.0
+
+parameters:
+  rust-ci:
+    type: boolean
+    default: false  
+  hivesim-rs-ci:
+    type: boolean
+    default: false
+
+jobs:
+  # This job builds the hive executable and stores it in the workspace.
+  build:
+    docker:
+      - image: cimg/go:1.21
+    steps:
+      # Build it.
+      - checkout
+      - go/load-cache
+      - go/mod-download
+      - go/save-cache
+      - run: {command: 'go build -ldflags="-s -extldflags=-static" -tags "osusergo netgo static_build" .'}
+      # Store the executable.
+      - persist_to_workspace:
+          root: .
+          paths: ["hive"]
+
+  # This job runs the smoke test simulations. This requires a virtual
+  # machine instead of the container-based build environment because
+  # hive needs to be able to talk to the docker containers it creates.
+  smoke-tests:
+    machine:
+      image: default
+    steps:
+      - checkout
+      - attach_workspace: {at: "/tmp/build"}
+      - run:
+          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum"
+      - run:
+          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum"
+
+  # This job also runs the smoke test simulations, but against a remote dockerd.
+  smoke-tests-remote-docker:
+    docker:
+      - image: cimg/base:2022.04
+    steps:
+      - checkout
+      - attach_workspace: {at: "/tmp/build"}
+      - setup_remote_docker
+      - run:
+          command: "/tmp/build/hive --sim=smoke/genesis --client=go-ethereum --loglevel 5"
+      - run:
+          command: "/tmp/build/hive --sim=smoke/network --client=go-ethereum --loglevel 5"
+
+  # This job runs the go unit tests.
+  go-test:
+    docker:
+      - image: cimg/go:1.21
+    steps:
+      # Get the source.
+      - checkout
+      - go/load-cache
+      - go/mod-download
+      - go/save-cache
+      # Run the tests.
+      - run:
+          name: "hive module tests"
+          command: "go test -cover ./..."
+      - run:
+          name: "hiveproxy module tests"
+          command: "go test -cover ./..."
+          working_directory: "./hiveproxy"
+      - run:
+          name: "Compile Go simulators"
+          command: ".circleci/compile-simulators.sh"
+  # this makes sure the rust code is good
+  hivesim-rs:
+    docker:
+      - image: cimg/rust:1.75.0
+    steps:
+      - checkout
+      - run:
+          name: Install rustfmt
+          command: rustup component add rustfmt
+      - run:
+          name: Install Clippy
+          command: rustup component add clippy
+      - run:
+          name: Install Clang
+          command: sudo apt update && sudo apt-get install clang -y
+      - run:
+          name: "Lint"
+          command: "cd hivesim-rs && cargo fmt --all -- --check"
+      - run:
+          name: "Build"
+          command: "cd hivesim-rs && cargo clippy --all --all-targets --all-features --no-deps -- --deny warnings"
+      - run:
+          name: "Test hivesim-rs"
+          command: "cd hivesim-rs && cargo test --workspace -- --nocapture"    
+  rust-simulators:
+    docker:
+      - image: cimg/rust:1.75.0
+    steps:
+      - checkout
+      - run:
+          name: Install rustfmt
+          command: rustup component add rustfmt
+      - run:
+          name: Install Clippy
+          command: rustup component add clippy
+      - run:
+          name: Install Clang
+          command: sudo apt update && sudo apt-get install clang -y
+      - run:
+          name: "Lint, build, test Rust simulators"
+          command: ".circleci/rust-simulators.sh"          
+
+workflows:
+  main:
+    jobs:
+      - go-test
+      - build
+      - smoke-tests:
+          requires: ["build"]
+      - smoke-tests-remote-docker:
+          requires: ["build"]
+  rust-simulator-jobs:
+    when: << pipeline.parameters.rust-ci >>
+    jobs:
+      - rust-simulators
+  hivesim-rs-jobs:
+    when: << pipeline.parameters.hivesim-rs-ci >>
+    jobs:
+      - hivesim-rs

.circleci/rust-simulators.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# This causes the bash script to exit immediately if any commands errors out
+set -e
+
+failed=""
+sims=$(find simulators -name Cargo.toml)
+for d in $sims; do
+    d="$(dirname "$d")"
+    echo "Lint, build, test $d"
+    ( cd "$d" || exit 1;
+    cargo fmt --all -- --check;
+    cargo clippy --all --all-targets --all-features --no-deps -- --deny warnings;
+    cargo test --workspace -- --nocapture;
+    )
+done

.circleci/upgrade-hivesim.sh

@@ -13,6 +13,6 @@ for d in $sims; do
         set -e
         cd $d
         go get -d "github.com/ethereum/hive@$version"
-        go mod tidy -compat=1.17
+        go mod tidy -compat=1.21
     )
 done

.devcontainer/Dockerfile

@@ -0,0 +1,3 @@
+FROM golang:1.22
+
+EXPOSE 8000

.devcontainer/devcontainer.json

@@ -0,0 +1,14 @@
+{
+  "name": "Simulator",
+  "dockerFile": "Dockerfile",
+  "appPort": [
+    "8000:8000"
+  ],
+  "postCreateCommand": "ls -la",
+  "containerEnv": {
+    "MY_VARIABLE": "1"
+  },
+  "shutdownAction": "none",
+  "privileged": true,
+  "capAdd": []
+}

.gitignore

@@ -3,3 +3,17 @@ workspace
 .idea/
 # build output
 /hive
+
+# build output for rust simulators and hivesim-rs files
+simulators/**/Cargo.lock
+simulators/**/target
+hivesim-rs/Cargo.lock
+hivesim-rs/target
+
+scripts/experiments/*
+scripts/proxy/*
+
+.DS_Store
+hiveview
+.lh/*
+simulators/eth2/go.work.sum

.vscode/launch.json

@@ -0,0 +1,42 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Connect to JQ hive",
+      "type": "go",
+      "request": "attach",
+      "mode": "remote",
+      "apiVersion": 2,
+      "remotePath": "/source",
+      "port": 40000,
+      "host": "127.0.0.1",
+      "cwd": "${workspaceFolder}/simulators/ethereum/gnosis-engine-jq",
+      "substitutePath": [
+        {
+          "from": "${workspaceFolder}/simulators/ethereum/gnosis-engine-jq",
+          "to": "/source"
+        }
+      ]
+    },
+    {
+      "name": "Connect to OLD hive",
+      "type": "go",
+      "request": "attach",
+      "mode": "remote",
+      "apiVersion": 2,
+      "remotePath": "/source",
+      "port": 40000,
+      "host": "127.0.0.1",
+      "cwd": "${workspaceFolder}/simulators/ethereum/engine-gnosis-nethermind",
+      "substitutePath": [
+        {
+          "from": "${workspaceFolder}/simulators/ethereum/engine-gnosis-nethermind",
+          "to": "/source"
+        }
+      ]
+    }
+  ]
+}

clients/besu/Dockerfile

@@ -17,7 +17,6 @@ COPY genesis.json /genesis.json
 COPY mapper.jq /mapper.jq
 COPY besu.sh /opt/besu/bin/besu-hive.sh
 COPY enode.sh /hive-bin/enode.sh
-COPY trusted_setup.txt /trusted_setup.txt
 
 # Set execute permissions for scripts
 RUN chmod +x /opt/besu/bin/besu-hive.sh /hive-bin/enode.sh

clients/besu/Dockerfile.git

@@ -1,24 +1,28 @@
 ### Build Besu From Git:
 
 ## Builder stage: Compiles besu from a git repository
-FROM openjdk:17-jdk-slim as builder
+FROM ubuntu:22.04 as builder
 
 ARG tag=main
 ARG github=hyperledger/besu
 
-RUN echo "Cloning: $github - $tag" && \
-    apt-get update && apt-get install -y git libsodium-dev libnss3-dev \
+RUN echo "installing java on ubuntu base image" \
+    && apt-get update && apt-get install -y git libsodium-dev libnss3-dev \
+    && apt-get install --no-install-recommends -q --assume-yes ca-certificates-java=20190909* \
+    && apt-get install --no-install-recommends -q --assume-yes openjdk-17-jre-headless=17* libjemalloc-dev=5.* \
+    && echo "Cloning: $github - $tag" \
     && git clone --depth 1 --branch $tag https://github.com/$github \
     && cd besu && ./gradlew installDist
 
 ## Final stage: Sets up the environment for running besu
-FROM openjdk:17-jdk-slim
+FROM ubuntu:22.04
 
 # Copy compiled binary from builder
 COPY --from=builder /besu/build/install/besu /opt/besu
 
 RUN apt-get update && apt-get install -y curl jq libsodium23 libnss3-dev \
-    && apt-get clean && rm -rf /var/lib/apt/lists/*
+    && apt-get install --no-install-recommends -q --assume-yes openjdk-17-jre-headless=17* libjemalloc-dev=5.*  \
+    && apt-get clean && rm -rf /var/lib/apt/lists/* 
 
 # Create version.txt
 RUN /opt/besu/bin/besu --version > /version.txt
@@ -28,7 +32,6 @@ COPY genesis.json /genesis.json
 COPY mapper.jq /mapper.jq
 COPY besu.sh /opt/besu/bin/besu-hive.sh
 COPY enode.sh /hive-bin/enode.sh
-COPY trusted_setup.txt /trusted_setup.txt
 
 # Set execute permissions for scripts
 RUN chmod +x /opt/besu/bin/besu-hive.sh /hive-bin/enode.sh