[WIP] PeerReview

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "autobahn"]
 	path = autobahn
 	url = https://github.com/grapheo12/autobahn-artifact.git
-	branch = autobahn-pirateship-baseline-2
+	branch = autobahn-tputvslatency

Cargo.toml

@@ -103,6 +103,7 @@ evil = []
 extra_2pc = []
 no_pipeline = []
 extra_qc_check = []
+witness_forwarding = []
 
 # Applications
 app_logger = []
@@ -116,8 +117,10 @@ chained_pbft = ["round_robin_leader", "always_sign", "view_change"]
 jolteon = ["round_robin_leader", "always_sign", "view_change", "no_pipeline"]
 hotstuff = ["round_robin_leader", "always_sign", "view_change", "no_pipeline", "extra_qc_check"]
 pirateship = ["round_robin_leader", "dynamic_sign", "view_change"]
+peerreview = ["fixed_leader", "always_sign", "witness_forwarding"]
 
-default = ["pirateship", "app_logger", "storage", "fast_path", "platforms"]
+default = ["peerreview", "app_logger", "storage", "platforms", "fast_path"]
+# default = ["pirateship", "app_logger", "storage", "platforms", "fast_path"]
 # default = ["jolteon", "app_logger", "storage"]
 # default = ["engraft", "app_kvs", "storage", "fast_path", "platforms"]
 

Makefile

@@ -6,6 +6,11 @@
 pirateship_logger:
 	CC=clang CXX=clang++ cargo build --release
 
+
+.PHONY: peerreview
+peerreview:
+	CC=clang CXX=clang++ cargo build --release --features peerreview,app_logger,storage,platforms,fast_path --no-default-features
+
 .PHONY: contrib
 contrib:
 	CC=clang CXX=clang++ cargo build --release --manifest-path contrib/Cargo.toml

deployment/azure-tf/main.tf

@@ -74,31 +74,39 @@ resource "azurerm_public_ip" "sevpool_public_ip" {
   count               = length(local.sevpool_ids_flattened_)
   location            = var.platform_locations[local.sevpool_ids_flattened_[count.index][0]]
   resource_group_name = azurerm_resource_group.rg.name
-  allocation_method   = "Dynamic"
+  # allocation_method   = "Dynamic"
+  allocation_method   = "Static"
+  sku                 = "Standard"
 }
 
 resource "azurerm_public_ip" "tdxpool_public_ip" {
   name                = "tdxpoolPublicIp_${local.tdxpool_ids_flattened_[count.index][0]}_${local.tdxpool_ids_flattened_[count.index][1]}"
   count               = length(local.tdxpool_ids_flattened_)
   location            = var.platform_locations[local.tdxpool_ids_flattened_[count.index][0]]
   resource_group_name = azurerm_resource_group.rg.name
-  allocation_method   = "Dynamic"
+  # allocation_method   = "Dynamic"
+  allocation_method   = "Static"
+  sku                 = "Standard"
 }
 
 resource "azurerm_public_ip" "clientpool_public_ip" {
   name                = "clientpoolPublicIp_${local.clientpool_ids_flattened_[count.index][0]}_${local.clientpool_ids_flattened_[count.index][1]}"
   count               = length(local.clientpool_ids_flattened_)
   location            = var.platform_locations[local.clientpool_ids_flattened_[count.index][0]]
   resource_group_name = azurerm_resource_group.rg.name
-  allocation_method   = "Dynamic"
+  # allocation_method   = "Dynamic"
+  allocation_method   = "Static"
+  sku                 = "Standard"
 }
 
 resource "azurerm_public_ip" "nonteepool_public_ip" {
   name                = "nonteepoolPublicIp_${local.nonteepool_ids_flattened_[count.index][0]}_${local.nonteepool_ids_flattened_[count.index][1]}"
   count               = length(local.nonteepool_ids_flattened_)
   location            = var.platform_locations[local.nonteepool_ids_flattened_[count.index][0]]
   resource_group_name = azurerm_resource_group.rg.name
-  allocation_method   = "Dynamic"
+  # allocation_method   = "Dynamic"
+  allocation_method   = "Static"
+  sku                 = "Standard"
 }
 
 
@@ -255,8 +263,8 @@ resource "azurerm_managed_disk" "sevpool_disk" {
 
 resource "azurerm_managed_disk" "tdxpool_disk" {
   name                 = "tdxpoolDataDisk${count.index}"
-  count                = length(local.sevpool_ids_flattened_)
-  location             = var.platform_locations[local.sevpool_ids_flattened_[count.index][0]]
+  count                = length(local.tdxpool_ids_flattened_)
+  location             = var.platform_locations[local.tdxpool_ids_flattened_[count.index][0]]
   resource_group_name  = azurerm_resource_group.rg.name
   storage_account_type = "Premium_LRS"
   disk_size_gb         = 2048 # P40 ssd Gives enough throughput for NIC bottleneck in 7+ machine clusters.
@@ -265,8 +273,8 @@ resource "azurerm_managed_disk" "tdxpool_disk" {
 
 resource "azurerm_managed_disk" "nonteepool_disk" {
   name                 = "nonteepoolDataDisk${count.index}"
-  count                = length(local.sevpool_ids_flattened_)
-  location             = var.platform_locations[local.sevpool_ids_flattened_[count.index][0]]
+  count                = length(local.nonteepool_ids_flattened_)
+  location             = var.platform_locations[local.nonteepool_ids_flattened_[count.index][0]]
   resource_group_name  = azurerm_resource_group.rg.name
   storage_account_type = "Premium_LRS"
   disk_size_gb         = 2048 # P40 ssd Gives enough throughput for NIC bottleneck in 7+ machine clusters.
@@ -400,7 +408,7 @@ resource "azurerm_linux_virtual_machine" "clientpool_vm" {
   location              = var.platform_locations[local.clientpool_ids_flattened_[count.index][0]]
   resource_group_name   = azurerm_resource_group.rg.name
   network_interface_ids = [azurerm_network_interface.clientpool_nic[count.index].id]
-  size                  = "Standard_D8ds_v5"
+  size                  = "Standard_D8ds_v6"
 
 #   delete_os_disk_on_termination    = true
 #   delete_data_disks_on_termination = true

deployment/azure-tf/setups/lan-2.tfvars

@@ -0,0 +1,7 @@
+platform_locations = ["eastus"]
+sevpool_count = [7]
+# sevpool_count = [1]
+tdxpool_count = [0]
+nonteepool_count = [0]
+clientpool_count = [1]
+# clientpool_count = [1]

deployment/azure-tf/setups/lan.tfvars

@@ -1,5 +1,7 @@
 platform_locations = ["eastus"]
 sevpool_count = [7]
+# sevpool_count = [1]
 tdxpool_count = [0]
 nonteepool_count = [0]
 clientpool_count = [3]
+# clientpool_count = [1]

experiments/lan_manual.toml

@@ -0,0 +1,181 @@
+workdir = "deployment_artifacts"
+project_home = "https://github.com/grapheo12/pirateship"
+
+[deployment_config]
+mode = "manual"
+ssh_key = "id_ed25519"
+ssh_user = "shubham_mishra"
+node_port_base = 3000
+
+
+[deployment_config.node_list]
+[deployment_config.node_list.nodepool_vm0]
+private_ip = "10.0.1.16"
+public_ip = "34.9.210.58"
+tee_type = "sev"
+region_id = 0
+
+[deployment_config.node_list.nodepool_vm1]
+private_ip = "10.0.1.10"
+public_ip = "34.132.89.189"
+tee_type = "sev"
+region_id = 0
+
+[deployment_config.node_list.nodepool_vm2]
+private_ip = "10.0.1.15"
+public_ip = "34.170.163.79"
+tee_type = "sev"
+region_id = 0
+
+[deployment_config.node_list.nodepool_vm3]
+private_ip = "10.0.1.13"
+public_ip = "34.45.236.240"
+tee_type = "sev"
+region_id = 0
+
+[deployment_config.node_list.nodepool_vm4]
+private_ip = "10.0.1.14"
+public_ip = "34.57.92.93"
+tee_type = "sev"
+region_id = 0
+
+[deployment_config.node_list.nodepool_vm5]
+private_ip = "10.0.1.11"
+public_ip = "34.10.69.175"
+tee_type = "sev"
+region_id = 0
+
+[deployment_config.node_list.nodepool_vm6]
+private_ip = "10.0.1.12"
+public_ip = "104.198.56.254"
+tee_type = "sev"
+region_id = 0
+
+[deployment_config.node_list.clientpool_vm0]
+private_ip = "10.0.2.3"
+public_ip = "34.31.76.85"
+[deployment_config.node_list.clientpool_vm1]
+private_ip = "10.0.2.4"
+public_ip = "34.31.167.16"
+[deployment_config.node_list.clientpool_vm2]
+private_ip = "10.0.2.5"
+public_ip = "35.239.110.106"
+
+
+[node_config]
+
+
+[node_config.net_config]
+client_max_retry = 10
+
+[node_config.rpc_config]
+recv_buffer_size = 32768
+channel_depth = 1000
+
+[node_config.consensus_config]
+commit_index_gap_soft = 250
+commit_index_gap_hard = 500
+liveness_u = 1
+max_backlog_batch_size = 1000
+signature_max_delay_blocks = 10
+signature_max_delay_ms = 102 # roughly batch_max_delay_ms * signature_max_delay_blocks
+num_crypto_workers = 5
+view_timeout_ms = 4000
+batch_max_delay_ms = 2
+
+[node_config.consensus_config.log_storage_config.RocksDB]
+write_buffer_size = 67_108_864 # 64 MiB
+max_write_buffer_number = 8
+max_write_buffers_to_merge = 1
+
+[node_config.app_config]
+logger_stats_report_ms = 100
+checkpoint_interval_ms = 1000
+
+[node_config.evil_config]
+simulate_byzantine_behavior = false
+byzantine_start_block = 0
+
+
+[client_config]
+full_duplex = true
+client_sub_id = 0       # This is filled up later by the caller code.
+
+[client_config.net_config]
+client_max_retry = 10
+
+[client_config.workload_config]
+num_requests = 100_000_000
+request_config = "Blanks"
+max_concurrent_requests = 16
+
+
+[[experiments]]
+name = "pirateship"
+repeats = 1
+num_nodes = 4
+node_distribution = "uniform"
+build_command = "make signed_raft_logger"
+duration = 60
+
+[experiments.sweeping_parameters]
+# num_clients = [10, 20, 30, 50, 100, 200, 400, 500, 600]
+num_clients = [400, 500, 600]
+
+# [[experiments]]
+# name = "autobahn"
+# type = "autobahn"
+# repeats = 1
+# num_nodes = 7
+# node_distribution = "uniform"
+# build_command = "make -C autobahn"
+# duration = 60
+
+# [experiments.sweeping_parameters]
+# # num_clients = [100_000, 120_000, 150_000, 200_000, 250_000, 300_000, 310_000, 320_000]
+# num_clients = [250_000, 300_000]
+
+
+# [[experiments]]
+# name = "pirateship_sig_1"
+# repeats = 1
+# num_nodes = 7
+# node_distribution = "uniform"
+# build_command = "make"
+# duration = 60
+
+# [experiments.node_config.consensus_config]
+# signature_max_delay_blocks = 1
+
+# [experiments.sweeping_parameters]
+# num_clients = [10, 20, 30, 50, 100, 200, 400, 500, 600]
+
+
+
+
+[[results]]
+name = "tput_latency_client_sweep"
+plotter = "tput_latency_sweep"
+ramp_up = 5
+ramp_down = 2
+output = "lan_final.pdf"
+force_parse = true
+# skip_indices = [0, 1]
+
+[results.legends]
+# Experiment group to legend mapping
+pirateship = "ps(sig=10)+ps(sig=10)-aud+byz"
+pirateship_sig_1 = "ps(sig=1)+ps(sig=1)-aud+byz"
+hotstuff = "hotstuff+onlybyz"
+engraft = "engraft+onlybyz"
+signed_raft = "signed_raft(n=5)"
+autobahn = "autobahn"
+signed_raft_n7 = "signed_raft(n=7)"
+
+
+# [results.partitions]
+# cft = ["ps(sig=10)", "ps(sig=1)", "engraft", "signed_raft(n=5)", "signed_raft(n=7)"]
+# bft = ["ps(sig=10)-aud", "ps(sig=1)-aud", "hotstuff", "autobahn"]
+
+[results.font]
+size = 55

experiments/lan_tee_experiments.toml

@@ -132,9 +132,9 @@ num_clients = [10, 20, 30, 50, 100, 200, 400, 500, 600]
 [[results]]
 name = "tput_latency_client_sweep"
 plotter = "tput_latency_sweep"
-ramp_up = 15
-ramp_down = 15
-output = "lan_final.pdf"
+ramp_up = 12
+ramp_down = 12
+output = "lan_with_peerreview.pdf"
 force_parse = true
 skip_indices = [0, 1]
 
@@ -147,10 +147,11 @@ engraft = "engraft+onlybyz"
 signed_raft = "signed_raft(n=5)"
 autobahn = "autobahn"
 signed_raft_n7 = "signed_raft(n=7)"
+peerreview = "peerreview"
 
 
 [results.partitions]
-cft = ["ps(sig=10)", "ps(sig=1)", "engraft", "signed_raft(n=5)", "signed_raft(n=7)"]
+cft = ["ps(sig=10)", "ps(sig=1)", "engraft", "signed_raft(n=5)", "signed_raft(n=7)", "peerreview"]
 bft = ["ps(sig=10)-aud", "ps(sig=1)-aud", "hotstuff", "autobahn"]
 
 [results.font]

experiments/lan_tee_experiments4.toml

@@ -64,22 +64,23 @@ build_command = "make"
 duration = 60
 
 [experiments.sweeping_parameters]
-num_clients = [10, 20, 30, 50, 100, 200, 400, 500, 600]
+# num_clients = [10, 20, 30, 50, 100, 200, 400, 500, 600]
+num_clients = [100, 200, 400] 
 
 
-[[experiments]]
-name = "pirateship_sig_1"
-repeats = 1
-num_nodes = 7
-node_distribution = "uniform"
-build_command = "make"
-duration = 60
+# [[experiments]]
+# name = "pirateship_sig_1"
+# repeats = 1
+# num_nodes = 7
+# node_distribution = "uniform"
+# build_command = "make"
+# duration = 60
 
-[experiments.node_config.consensus_config]
-signature_max_delay_blocks = 1
+# [experiments.node_config.consensus_config]
+# signature_max_delay_blocks = 1
 
-[experiments.sweeping_parameters]
-num_clients = [10, 20, 30, 50, 100, 200, 400, 500, 600]
+# [experiments.sweeping_parameters]
+# num_clients = [10, 20, 30, 50, 100, 200, 400, 500, 600]
 
 
 
@@ -91,7 +92,7 @@ ramp_up = 15
 ramp_down = 15
 output = "lan_final.pdf"
 force_parse = true
-skip_indices = [0, 1]
+# skip_indices = [0, 1]
 
 [results.legends]
 # Experiment group to legend mapping