[Pal/LibOS] Fix overflow checks in calloc()

- Fixes a bug in PAL's calloc() (missing overflow check, most likely a
  security issue).
- Changes LibOS's calloc() to use overflow builtins for better
  performance (no division needed; we use GCC/clang builtins everywhere
  anyways).

Signed-off-by: Michał Kowalczyk <mkow@invisiblethingslab.com>

LibOS/shim/src/shim_malloc.c

@@ -101,11 +101,11 @@ void* malloc(size_t size) {
     return mem;
 }
 
-void* calloc(size_t nmemb, size_t size) {
-    // This overflow checking is not a UB, because the operands are unsigned.
-    size_t total = nmemb * size;
-    if (total / size != nmemb)
+void* calloc(size_t num, size_t size) {
+    size_t total;
+    if (__builtin_mul_overflow(num, size, &total))
         return NULL;
+
     void* ptr = malloc(total);
     if (ptr)
         memset(ptr, 0, total);

Pal/include/pal_internal.h

@@ -250,7 +250,7 @@ int _DkSetProtectedFilesKey(const char* pf_key_hex);
 void init_slab_mgr(char* mem_pool, size_t mem_pool_size);
 void* malloc(size_t size);
 void* malloc_copy(const void* mem, size_t size);
-void* calloc(size_t nmem, size_t size);
+void* calloc(size_t num, size_t size);
 void free(void* mem);
 
 #ifdef __GNUC__

Pal/src/slab.c

@@ -164,12 +164,14 @@ void* malloc_copy(const void* mem, size_t size) {
     return nmem;
 }
 
-void* calloc(size_t nmem, size_t size) {
-    void* ptr = malloc(nmem * size);
+void* calloc(size_t num, size_t size) {
+    size_t total;
+    if (__builtin_mul_overflow(num, size, &total))
+        return NULL;
 
+    void* ptr = malloc(total);
     if (ptr)
-        memset(ptr, 0, nmem * size);
-
+        memset(ptr, 0, total);
     return ptr;
 }
 

subprojects/packagefiles/mbedtls/include/mbedtls/config-pal.h

@@ -48,7 +48,7 @@
 #include <limits.h>
 #include <stddef.h>
 
-void* calloc(size_t nmem, size_t size);
+void* calloc(size_t num, size_t size);
 void free(void*);
 
 #define MBEDTLS_PLATFORM_STD_CALLOC   calloc