feat(bench): add different certificate signature algorithms to benchm…

…arks (aws#4080)

bindings/rust/bench/.cargo/config.toml

@@ -0,0 +1,3 @@
+[env]
+S2N_TLS_LIB_DIR = "/home/ubuntu/s2n-tls/bindings/rust/bench/target/s2n-tls-build/lib"
+LD_LIBRARY_PATH = "/home/ubuntu/s2n-tls/bindings/rust/bench/target/s2n-tls-build/lib"

bindings/rust/bench/.gitignore

@@ -1,2 +1,3 @@
+*.pem
 *.svg
 !historical-perf-*.svg

bindings/rust/bench/README.md

@@ -4,7 +4,7 @@ We use to Criterion.rs to benchmark s2n-tls against two commonly used TLS librar
 
 ## Setup 
 
-Setup is easy! Just have OpenSSL installed and generate Rust bindings for s2n-tls using `bindings/rust/generate.sh`. 
+Setup is easy! Just have OpenSSL installed, generate Rust bindings for s2n-tls using `../generate.sh`, and generate certs using `certs/generate_certs.sh`. 
 
 Dependencies are the same as with s2n-tls. Currently, this crate has only been tested on Ubuntu (both x86 and ARM), but we expect everything to work with other Unix environments. 
 
@@ -14,6 +14,7 @@ For example, to get started with benching s2n-tls with AWS-LC:
 
 ```
 ../generate.sh
+certs/generate_certs.sh
 ./install-aws-lc.sh
 cargo bench --config aws-lc-config/s2n.toml
 ```
@@ -44,7 +45,7 @@ To remove external factors, we use custom IO with our benchmarks, bypassing the
 
 ### Certificate generation
 
-All certs are stored in `certs/` and can be regenerated using `certs/generate_certs.sh`. There is one root cert that directly signs the server and client certs that are used in benchmarking. Currently, we use ECDSA with `secp384r1`.
+There is one root cert that directly signs the server and client certs that are used in benchmarking. We currently bench RSA and ECDSA certs.
 
 ### Negotiation parameters
 

bindings/rust/bench/benches/handshake.rs

@@ -2,10 +2,12 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use bench::{
-    CryptoConfig,
+    CipherSuite, CryptoConfig,
     ECGroup::{self, *},
     HandshakeType::{self, *},
-    OpenSslHarness, RustlsHarness, S2NHarness, TlsBenchHarness,
+    OpenSslHarness, RustlsHarness, S2NHarness,
+    SigType::{self, *},
+    TlsBenchHarness,
 };
 use criterion::{
     criterion_group, criterion_main, measurement::WallTime, BatchSize, BenchmarkGroup, Criterion,
@@ -17,17 +19,15 @@ pub fn bench_handshake_params(c: &mut Criterion) {
         name: &str,
         handshake_type: HandshakeType,
         ec_group: ECGroup,
+        sig_type: SigType,
     ) {
         // generate all harnesses (TlsBenchHarness structs) beforehand so that benchmarks
         // only include negotiation and not config/connection initialization
         bench_group.bench_function(name, |b| {
             b.iter_batched_ref(
                 || {
                     T::new(
-                        CryptoConfig {
-                            cipher_suite: Default::default(),
-                            ec_group,
-                        },
+                        CryptoConfig::new(CipherSuite::default(), ec_group, sig_type),
                         handshake_type,
                         Default::default(),
                     )
@@ -47,29 +47,35 @@ pub fn bench_handshake_params(c: &mut Criterion) {
 
     for handshake_type in [ServerAuth, MutualAuth] {
         for ec_group in [SECP256R1, X25519] {
-            let mut bench_group =
-                c.benchmark_group(format!("handshake-{:?}-{:?}", handshake_type, ec_group));
-
-            bench_handshake_for_library::<S2NHarness>(
-                &mut bench_group,
-                "s2n-tls",
-                handshake_type,
-                ec_group,
-            );
-            #[cfg(not(feature = "historical-perf"))]
-            {
-                bench_handshake_for_library::<RustlsHarness>(
+            for sig_type in [Rsa2048, Rsa3072, Rsa4096, Ec384] {
+                let mut bench_group = c.benchmark_group(format!(
+                    "handshake-{:?}-{:?}-{:?}",
+                    handshake_type, ec_group, sig_type
+                ));
+                bench_handshake_for_library::<S2NHarness>(
                     &mut bench_group,
-                    "rustls",
-                    handshake_type,
-                    ec_group,
-                );
-                bench_handshake_for_library::<OpenSslHarness>(
-                    &mut bench_group,
-                    "openssl",
+                    "s2n-tls",
                     handshake_type,
                     ec_group,
+                    sig_type,
                 );
+                #[cfg(not(feature = "historical-perf"))]
+                {
+                    bench_handshake_for_library::<RustlsHarness>(
+                        &mut bench_group,
+                        "rustls",
+                        handshake_type,
+                        ec_group,
+                        sig_type,
+                    );
+                    bench_handshake_for_library::<OpenSslHarness>(
+                        &mut bench_group,
+                        "openssl",
+                        handshake_type,
+                        ec_group,
+                        sig_type,
+                    );
+                }
             }
         }
     }

bindings/rust/bench/benches/throughput.rs

@@ -3,7 +3,8 @@
 
 use bench::{
     CipherSuite::{self, *},
-    CryptoConfig, OpenSslHarness, RustlsHarness, S2NHarness, TlsBenchHarness,
+    CryptoConfig, ECGroup, HandshakeType, OpenSslHarness, RustlsHarness, S2NHarness, SigType,
+    TlsBenchHarness, harness::ConnectedBuffer,
 };
 use criterion::{
     criterion_group, criterion_main, measurement::WallTime, BatchSize, BenchmarkGroup, Criterion,
@@ -24,12 +25,9 @@ pub fn bench_throughput_cipher_suite(c: &mut Criterion) {
             b.iter_batched_ref(
                 || {
                     T::new(
-                        CryptoConfig {
-                            cipher_suite,
-                            ec_group: Default::default(),
-                        },
-                        Default::default(),
-                        Default::default(),
+                        CryptoConfig::new(cipher_suite, ECGroup::default(), SigType::default()),
+                        HandshakeType::default(),
+                        ConnectedBuffer::default(),
                     )
                     .map(|mut h| {
                         let _ = h.handshake();

bindings/rust/bench/certs/generate_certs.sh

@@ -3,39 +3,72 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
+# Usage: ./generate_certs.sh [clean]
+# Generates all necessary certs for benching
+# Use argument "clean" to remove all generated certs
+
 # immediately bail if any command fails
 set -e
 
-pushd "$(dirname "$0")"
+# go to directory script is located in
+pushd "$(dirname "$0")" > /dev/null
+
+# Generates certs with given algorithms and bits in $1$2/, ex. ec384/
+# $1: rsa or ec
+# $2: number of bits
+cert-gen () {
+    echo -e "\n----- generating certs for $1$2 -----\n"
+
+    key_family=$1
+    key_size=$2
+
+    # set openssl argument name
+    if [[ $key_family == rsa ]]; then
+        local argname=rsa_keygen_bits:
+    elif [[ $key_family == ec ]]; then
+        local argname=ec_paramgen_curve:P-
+    fi
+
+    # make directory for certs
+    mkdir -p $key_family$key_size
+    cd $key_family$key_size
+
+    echo "generating CA private key and certificate"
+    openssl req -new -nodes -x509 -newkey $key_family -pkeyopt $argname$key_size -keyout  ca-key.pem -out ca-cert.pem -days 65536 -config ../config/ca.cnf
 
-echo "generating CA private key and certificate"
-openssl req -nodes -new -x509 -keyout ca-key.pem -out ca-cert.pem -days 65536 -config config/ca.cnf
+    echo "generating server private key and CSR"
+    openssl req  -new -nodes -newkey $key_family -pkeyopt $argname$key_size -keyout server-key.pem -out server.csr -config ../config/server.cnf
 
-# secp384r1 is an arbitrarily chosen curve that is supported by the default
-# security policy in s2n-tls.
-# https://github.com/aws/s2n-tls/blob/main/docs/USAGE-GUIDE.md#chart-security-policy-version-to-supported-curvesgroups
-echo "generating server private key and CSR"
-openssl req  -new -nodes -newkey ec -pkeyopt ec_paramgen_curve:secp384r1 -keyout server-key.pem -out server.csr -config config/server.cnf
+    echo "generating client private key and CSR"
+    openssl req  -new -nodes -newkey $key_family -pkeyopt $argname$key_size -keyout client-key.pem -out client.csr -config ../config/client.cnf
 
-echo "generating client private key and CSR"
-openssl req  -new -nodes -newkey ec -pkeyopt ec_paramgen_curve:secp384r1 -keyout client-key.pem -out client.csr -config config/client.cnf
+    echo "generating server certificate and signing it"
+    openssl x509 -days 65536 -req -in server.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out server-cert.pem -extensions req_ext -extfile ../config/server.cnf
 
-echo "generating server certificate and signing it"
-openssl x509 -days 65536 -req -in server.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out server-cert.pem -extensions req_ext -extfile config/server.cnf
+    echo "generating client certificate and signing it"
+    openssl x509 -days 65536 -req -in client.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client-cert.pem -extensions req_ext -extfile ../config/client.cnf
 
-echo "generating client certificate and signing it"
-openssl x509 -days 65536 -req -in client.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client-cert.pem -extensions req_ext -extfile config/client.cnf
+    echo "verifying generated certificates"
+    openssl verify -CAfile ca-cert.pem server-cert.pem
+    openssl verify -CAfile ca-cert.pem client-cert.pem
 
-echo "verifying generated certificates"
-openssl verify -CAfile ca-cert.pem server-cert.pem
-openssl verify -CAfile ca-cert.pem client-cert.pem
+    echo "cleaning up temporary files"
+    rm server.csr
+    rm client.csr
+    rm ca-key.pem
 
-cat server-cert.pem ca-cert.pem > server-fullchain.pem
-cat client-cert.pem ca-cert.pem > client-fullchain.pem
+    cd ..
+}
 
-echo "cleaning up temporary files"
-rm server.csr
-rm client.csr
-rm ca-key.pem
+if [[ $1 != "clean" ]] 
+then
+    cert-gen ec 384
+    cert-gen rsa 2048
+    cert-gen rsa 3072
+    cert-gen rsa 4096
+else 
+    echo "cleaning certs"
+    rm -rf ec*/ rsa*/
+fi
 
-popd
+popd > /dev/null