Skip to content

Commit

Permalink
feat(l1): enchance hive report. (#1489)
Browse files Browse the repository at this point in the history
**Motivation**
We want more descriptive names, and a better ordering
  • Loading branch information
mpaulucci authored Dec 12, 2024
1 parent cf69ecf commit 6c75012
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 27 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/daily_reports.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ env:

jobs:
run-hive:
name: Run engine hive simulator to gather coverage information.
name: Hive - ${{ matrix.name }}
runs-on: ubuntu-latest
strategy:
matrix:
Expand Down Expand Up @@ -45,7 +45,7 @@ jobs:
run: make setup-hive

- name: Run Hive Simulation
run: cd hive && ./hive --client ethrex --sim ${{ matrix.simulation }} --sim.parallelism 4
run: cd hive && ./hive --client ethrex --sim ${{ matrix.simulation }} --sim.parallelism 16
continue-on-error: true

- name: Upload results
Expand Down
90 changes: 81 additions & 9 deletions cmd/hive_report/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,59 @@ struct JsonFile {
test_cases: std::collections::HashMap<String, TestCase>,
}

struct HiveResult {
category: String,
display_name: String,
passed_tests: usize,
total_tests: usize,
success_percentage: f64,
}

impl HiveResult {
fn new(suite: String, passed_tests: usize, total_tests: usize) -> Self {
let success_percentage = (passed_tests as f64 / total_tests as f64) * 100.0;

let (category, display_name) = match suite.as_str() {
"engine-api" => ("Engine", "Paris"),
"engine-auth" => ("Engine", "Auth"),
"engine-cancun" => ("Engine", "Cancun"),
"engine-exchange-capabilities" => ("Engine", "Exchange Capabilities"),
"engine-withdrawals" => ("Engine", "Shanghai"),
"discv4" => ("P2P", "Discovery V4"),
"eth" => ("P2P", "Eth capability"),
"snap" => ("P2P", "Snap capability"),
"rpc-compat" => ("RPC", "RPC API Compatibility"),
"sync" => ("Sync", "Node Syncing"),
other => {
eprintln!("Warn: Unknown suite: {}. Skipping", other);
("", "")
}
};

HiveResult {
category: category.to_string(),
display_name: display_name.to_string(),
passed_tests,
total_tests,
success_percentage,
}
}

fn should_skip(&self) -> bool {
self.category.is_empty()
}
}

impl std::fmt::Display for HiveResult {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}: {}/{} ({:.02}%)",
self.display_name, self.passed_tests, self.total_tests, self.success_percentage
)
}
}

fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut results = Vec::new();

Expand Down Expand Up @@ -54,22 +107,41 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
.filter(|test_case| test_case.summary_result.pass)
.count();

results.push((json_data.name, passed_tests, total_tests));
let result = HiveResult::new(json_data.name, passed_tests, total_tests);
if !result.should_skip() {
results.push(result);
}
}
}

// Sort by file name.
results.sort_by(|a, b| a.0.cmp(&b.0));
// First by category ascending, then by passed tests descending, then by success percentage descending.
results.sort_by(|a, b| {
a.category
.cmp(&b.category)
.then_with(|| b.passed_tests.cmp(&a.passed_tests))
.then_with(|| {
b.success_percentage
.partial_cmp(&a.success_percentage)
.unwrap()
})
});

let results_by_category = results.chunk_by(|a, b| a.category == b.category);

for (file_name, passed, total) in &results {
let success_percentage = (*passed as f64 / *total as f64) * 100.0;
println!("{file_name}: {passed}/{total} ({success_percentage:.02}%)");
for results in results_by_category {
// print category
println!("*{}*", results[0].category);
for result in results {
println!("\t{}", result);
}
println!();
}

println!();
let total_passed = results.iter().map(|(_, p, _)| p).sum::<usize>();
let total_tests = results.iter().map(|(_, _, t)| t).sum::<usize>();
let total_passed = results.iter().map(|r| r.passed_tests).sum::<usize>();
let total_tests = results.iter().map(|r| r.total_tests).sum::<usize>();
let total_percentage = (total_passed as f64 / total_tests as f64) * 100.0;
println!("Total: {total_passed}/{total_tests} ({total_percentage:.02}%)");
println!("*Total: {total_passed}/{total_tests} ({total_percentage:.02}%)*");

Ok(())
}
17 changes: 1 addition & 16 deletions crates/common/types/block.rs
Original file line number Diff line number Diff line change
Expand Up @@ -586,27 +586,12 @@ fn calc_excess_blob_gas(parent_header: &BlockHeader) -> u64 {

#[cfg(test)]
mod test {
use std::{str::FromStr, time::Instant};
use std::str::FromStr;

use super::*;
use ethereum_types::H160;
use hex_literal::hex;

#[test]
fn compute_hash() {
let block = Block::default();

let start = Instant::now();
block.hash();
let duration = start.elapsed();

let start_2 = Instant::now();
block.hash();
let duration_2 = start_2.elapsed();

assert!(duration > 1000 * duration_2);
}

#[test]
fn test_compute_withdrawals_root() {
// Source: https://github.com/ethereum/tests/blob/9760400e667eba241265016b02644ef62ab55de2/BlockchainTests/EIPTests/bc4895-withdrawals/amountIs0.json
Expand Down

0 comments on commit 6c75012

Please sign in to comment.