add automated benchmarking

eriptic · Dec 28, 2023 · 3b74d30 · 3b74d30
1 parent 9c18a35
commit 3b74d30
Show file tree

Hide file tree

Showing 29 changed files with 1,110 additions and 484 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -52,7 +52,7 @@ jobs:
            source ~/zephyrproject/zephyr/zephyr-env.sh
            source ~/zephyrproject/.venv/bin/activate
            cd test/
-           bash test_build_options.sh
+           bash ci_scripts/test_build_options.sh
 
       - name: Archive static lib
         uses: actions/upload-artifact@v2
@@ -77,7 +77,7 @@ jobs:
           name: uoscore-uedhoc-x86-64
           path: test/build/uoscore_uedhoc/libuoscore-uedhoc.a
 
-### Generate test coverage report
+### Generate test coverage/stack/flash report
       - name: Build native_posix with coverage information
         run: |
            source ~/zephyrproject/zephyr/zephyr-env.sh
@@ -90,13 +90,27 @@ jobs:
            lcov --capture --directory ./ --output-file lcov.info -q --rc lcov_branch_coverage=1
            genhtml lcov.info --output-directory lcov_html -q --ignore-errors source --branch-coverage --highlight --legend
            tar -zcvf github-pages.tar.gz lcov_html
+           bash ci_scripts/flash_report_gen.sh
+           bash ci_scripts/stack_report_gen.sh
 
       - name: Upload test coverage report
         uses: actions/upload-pages-artifact@v1
         with: 
           name: github-pages  
           path: test/lcov_html
-
+
+      - name: Upload rom report
+        uses: actions/upload-pages-artifact@v1
+        with: 
+          name: github-pages  
+          path: build_reports/rom_report_nrf91_with_tinycrypt.txt
+
+      - name: Upload stack report
+        uses: actions/upload-pages-artifact@v1
+        with: 
+          name: github-pages  
+          path: build_reports/stack_report.html
+
 
   # Deploy job
   deploy:

diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,7 @@
 misk
 build
 build_lib_test
+build_reports
 *.log
 .vscode
 *.cproject

diff --git a/Makefile b/Makefile
@@ -37,36 +37,6 @@ C_SOURCES += $(wildcard src/cbor/*.c)
 #$(info    \n C_SOURCES is $(C_SOURCES))
 vpath %.c $(sort $(dir $(C_SOURCES)))
 
-################################################################################
-# libraries 
-
-
-################################################################################
-# C includes
-C_INCLUDES += -Iinc
-
-# Crypto engine
-ifeq ($(findstring COMPACT25519,$(CRYPTO_ENGINE)),COMPACT25519) 
-C_INCLUDES += -Iexternals/compact25519/src/c25519/ 
-C_INCLUDES += -Iexternals/compact25519/src/ 
-endif
-
-ifeq ($(findstring TINYCRYPT,$(CRYPTO_ENGINE)),TINYCRYPT)
-C_INCLUDES += -Iexternals/tinycrypt/lib/include
-endif
-
-ifeq ($(findstring MBEDTLS,$(CRYPTO_ENGINE)),MBEDTLS)
-C_INCLUDES += -Iexternals/mbedtls/library 
-C_INCLUDES += -Iexternals/mbedtls/include 
-C_INCLUDES += -Iexternals/mbedtls/include/mbedtls 
-C_INCLUDES += -Iexternals/mbedtls/include/psa 
-endif
-
-# CBOR engine
-ifeq ($(findstring ZCBOR,$(CBOR_ENGINE)),ZCBOR)
-C_INCLUDES += -Iexternals/zcbor/include
-endif
-
 ################################################################################
 # CFLAGS
 ################################################################################
@@ -89,9 +59,6 @@ ifeq ($(findstring UNIT_TEST,$(DUNIT_TEST)),UNIT_TEST)
 EXTENDED_CFLAGS += -DOSCORE_NVM_SUPPORT
 endif
 
-#add include paths
-EXTENDED_CFLAGS += $(C_INCLUDES)
-
 #generate debug symbols
 EXTENDED_CFLAGS += -g3 -gdwarf-4
 
@@ -156,6 +123,36 @@ ifeq ($(findstring ASAN,$(ASAN)),ASAN)
 EXTENDED_CFLAGS += -fsanitize=address -fomit-frame-pointer
 endif
 
+################################################################################
+# C includes
+################################################################################
+C_INCLUDES += -Iinc
+
+# Crypto engine
+ifeq ($(findstring COMPACT25519,$(EXTENDED_CFLAGS)),COMPACT25519) 
+C_INCLUDES += -Iexternals/compact25519/src/c25519/ 
+C_INCLUDES += -Iexternals/compact25519/src/ 
+endif
+
+ifeq ($(findstring TINYCRYPT,$(EXTENDED_CFLAGS)),TINYCRYPT)
+C_INCLUDES += -Iexternals/tinycrypt/lib/include
+endif
+
+ifeq ($(findstring MBEDTLS,$(EXTENDED_CFLAGS)),MBEDTLS)
+C_INCLUDES += -Iexternals/mbedtls/library 
+C_INCLUDES += -Iexternals/mbedtls/include 
+C_INCLUDES += -Iexternals/mbedtls/include/mbedtls 
+C_INCLUDES += -Iexternals/mbedtls/include/psa 
+endif
+
+# CBOR engine
+ifeq ($(findstring ZCBOR,$(EXTENDED_CFLAGS)),ZCBOR)
+C_INCLUDES += -Iexternals/zcbor/include
+endif
+
+#add include paths
+EXTENDED_CFLAGS += $(C_INCLUDES)
+
 
 $(info    EXTENDED_CFLAGS are $(EXTENDED_CFLAGS))
 ################################################################################

diff --git a/benchmarks.md b/benchmarks.md
@@ -0,0 +1,54 @@
+# Benchmark and comparison of OSCORE+EDHOC vs. (D)TLS
+
+## Flash
+
+| Platform             | uoscore-uedhoc 3.0.x | TLS 1.3     | DTLS 1.3    |
+| -------------------- | -------------------- | ----------- | ----------- |
+| nRF52840 (Cortex M4) | 41,156 Byte          | 52,023 Byte | 58,097 Byte |
+
+Notes:
+
+1) uOSCORE-uEDHOC and (D)TLS are used with P256/AES-128-CCM and X509 certificates. 
+2) usocore-uedhoc uses [tinycrypt](https://github.com/intel/tinycrypt) as crypto engine for this evaluation and [zcbor](https://github.com/NordicSemiconductor/zcbor) as CBOR engine. The table shows the total footprint including tinycrypt and zcbor. 
+3) The (D)TLS data is obtained with [mbedTLS](https://github.com/Mbed-TLS/mbedtls) and published in Table II in  [Low-Power IoT Communication Security: On the Performance of DTLS and TLS 1.3](https://inria.hal.science/hal-03035402/document)
+
+## RAM
+
+| uoscore-uedhoc 3.0.x with FLA                 | uoscore-uedhoc 3.0.x with VLA                 | TLS 1.3    | DTLS 1.3   |
+| --------------------------------------------- | --------------------------------------------- | ---------- | ---------- |
+| 6916 Byte (initiator) / 6656 Byte (responder) | 6440 Byte (initiator) / 6160 Byte (responder) | 21692 Byte | 22026 Byte |
+
+Notes:
+
+1. uOSCORE-uEDHOC uses only stack memory. The (D)TLS data includes stack and heap memory as given in Table V in [Low-Power IoT Communication Security: On the Performance of DTLS and TLS 1.3](https://inria.hal.science/hal-03035402/document)
+
+## Bytes-Over-Air
+
+| EDHOC                                                       | TLS 1.3 | DTLS 1.3 |
+| ----------------------------------------------------------- | ------- | -------- |
+| 808 Byte (exchanging two 293 Byte X.509 certificates)       | 1371    | 1500     |
+| 242 Byte (Using pre-established  X.509 certificates)        | -       | -        |
+| 101 Byte (Using pre-established static Diffie-Hellman keys) | -       | -        |
+
+## Latency
+
+### Latency for a key exchange with uEDHOC
+
+| Platform                    | EDHOC  exchanging X.509 certificates | EDHOC Using pre-established  X.509 certificates |
+| --------------------------- | ------------------------------------ | ----------------------------------------------- |
+| ESP32 (Xtensa LX6 @ 160MHz) | 0,402 s                              | 0,294 s                                         |
+| nRF52840 (Cortex M4 @64MHz) | 1,1735 s                             | 0,8615 s                                        |
+| nRF51822 (Cortex M0 @16MHz) | 11,7205 s                            | 8,622 s                                         |
+
+### Latency for encrypting/decrypting payload data with uOSCORE
+
+| Platform                    | Payload 20 Byte | Payload 50 Byte | Payload 100 Byte | Payload 200 Byte | Payload 500 Byte | Payload 1000 Byte |
+| --------------------------- | --------------- | --------------- | ---------------- | ---------------- | ---------------- | ----------------- |
+| ESP32 (Xtensa LX6 @ 160MHz) | 475 us          | 671 us          | 977 us           | 1585 us          | 3500 us          | 6600 us           |
+| nRF52840 (Cortex M4 @64MHz) | 1801 us         | 2533 us         | 3723 us          | 6073 us          | 13519 us         | 25665 us          |
+| nRF51822 (Cortex M0 @16MHz) | 9613 us         | 14069 us        | 20508 us         | 33356 us         | 74005 us         | 140381 us         |
+
+Notes:
+
+1. All latency numbers are pure latency caused by computations. No data was send or received. The sending/receiving was emulated off-line.
+2. [Tinycrypt](https://github.com/intel/tinycrypt) was used as cryptographic engine.
diff --git a/inc/cbor/edhoc_decode_cert.h b/inc/cbor/edhoc_decode_cert.h
@@ -21,12 +21,8 @@ extern "C" {
 #error "The type file was generated with a different default_max_qty than this file"
 #endif
 
-
-int cbor_decode_cert(
-		const uint8_t *payload, size_t payload_len,
-		struct cert *result,
-		size_t *payload_len_out);
-
+int cbor_decode_cert(const uint8_t *payload, size_t payload_len,
+		     struct cert *result, size_t *payload_len_out);
 
 #ifdef __cplusplus
 }

diff --git a/inc/common/print_util.h b/inc/common/print_util.h
@@ -32,7 +32,7 @@ void print_array(const uint8_t *in_data, uint32_t in_len);
  */
 void handle_runtime_error(int error_code, const char *file_name,
 			  const int line);
-		
+
 /**
  * @brief 			In case of an error in a function belonging to 
  * 				an external library this function prints where 
@@ -41,10 +41,23 @@ void handle_runtime_error(int error_code, const char *file_name,
  * @param error_code 		The error code to be printed.
  * @param file_name 		The file name where the error occurred.
  * @param line 			The line at which the error occurred.
- */		
+ */
 void handle_external_runtime_error(int error_code, const char *file_name,
 				   const int line);
+
 #ifdef DEBUG_PRINT
+#define RED "\x1B[31m"
+#define RESET "\033[0m"
+static const char transport_deinit_message[] = {
+	RESET "Transport deinitialized at %s:%d\n\n"
+};
+static const char runtime_error_message[] = {
+	RED "Runtime error: code %d at %s:%d\n\n" RESET
+};
+static const char external_runtime_error_message[] = {
+	RED "External lib runtime error: code %d at %s:%d\n\n" RESET
+};
+
 #define PRINT_ARRAY(msg, a, a_len)                                             \
 	printf(msg);                                                           \
 	print_array(a, a_len);

diff --git a/inc/oscore.h b/inc/oscore.h
@@ -110,9 +110,7 @@ enum err oscore_context_init(struct oscore_init_params *params,
  * @param 	buf_out when a OSCORE packet is found and decrypted the 
  * 		resulting CoAP is saved in buf_out
  * @param 	buf_out_len length of the CoAP packet
- * @param	flag  indicates if the 
  * @param 	c pointer to a security context
- * @param 	oscore_pkg indicates if an incoming packet is OSCORE
  * @return	err
  */
 enum err oscore2coap(uint8_t *buf_in, uint32_t buf_in_len, uint8_t *buf_out,

diff --git a/makefile_config.mk b/makefile_config.mk
@@ -25,7 +25,7 @@ OPT = -O0
 ################################################################################
 # Print helpful debug messages
 ################################################################################
-DEBUG_PRINT += -DDEBUG_PRINT
+#DEBUG_PRINT += -DDEBUG_PRINT
 
 ################################################################################
 # Use Address Sanitizer, e.g. with native_posix
@@ -157,5 +157,5 @@ FEATURES += -DI_OPTIONS_BUFF_MAX_LEN=100
 
 
 #CRYPTO_ENGINE += -DTINYCRYPT
-CRYPTO_ENGINE += -DCOMPACT25519
-CRYPTO_ENGINE += -DMBEDTLS
+#CRYPTO_ENGINE += -DCOMPACT25519
+#CRYPTO_ENGINE += -DMBEDTLS
diff --git a/src/common/crypto_wrapper.c b/src/common/crypto_wrapper.c
@@ -506,11 +506,12 @@ enum err WEAK verify(enum sign_alg alg, const struct byte_array *pk,
 		TRY_EXPECT(tc_sha256_update(&ctx_sha256, msg->ptr, msg->len),
 			   1);
 		TRY_EXPECT(tc_sha256_final(hash, &ctx_sha256), 1);
+		uint8_t *pk_ptr = pk->ptr;
 		if ((P_256_PUB_KEY_UNCOMPRESSED_SIZE == pk->len) &&
 		    (0x04 == *pk->ptr)) {
-			pk->ptr++;
+			pk_ptr++;
 		}
-		TRY_EXPECT(uECC_verify(pk->ptr, hash, NUM_ECC_BYTES, sgn->ptr,
+		TRY_EXPECT(uECC_verify(pk_ptr, hash, NUM_ECC_BYTES, sgn->ptr,
 				       p256),
 			   1);
 		*result = true;

diff --git a/src/common/print_util.c b/src/common/print_util.c
@@ -14,20 +14,7 @@
 
 #include "common/print_util.h"
 #include "common/oscore_edhoc_error.h"
-
-#ifdef DEBUG_PRINT
-#define RED "\x1B[31m"
-#define RESET "\033[0m"
-static const char transport_deinit_message[] = {
-	RESET "Transport deinitialized at %s:%d\n\n"
-};
-static const char runtime_error_message[] = {
-	RED "Runtime error: code %d at %s:%d\n\n" RESET
-};
-static const char external_runtime_error_message[] = {
-	RED "External lib runtime error: code %d at %s:%d\n\n" RESET
-};
-#endif
+#include "common/print_util.h"
 
 void print_array(const uint8_t *in_data, uint32_t in_len)
 {