Eliminate cyclic locks in runtime

This change introduces a new deadlock detector for Cosmo's POSIX threads
implementation. Error check mutexes will now track a DAG of nested locks
and report EDEADLK when a deadlock is theoretically possible. These will
occur rarely, but it's important for production hardening your code. You
don't even need to change your mutexes to use the POSIX error check mode
because `cosmocc -mdbg` will enable error checking on mutexes by default
globally. When cycles are found, an error message showing your demangled
symbols describing the strongly connected component are printed and then
the SIGTRAP is raised, which means you'll also get a backtrace if you're
using ShowCrashReports() too. This new error checker is so low-level and
so pure that it's able to verify the relationships of every libc runtime
lock, including those locks upon which the mutex implementation depends.

Author: jart

libc/calls/getloadavg-nt.c

@@ -21,24 +21,23 @@
 #include "libc/calls/syscall_support-nt.internal.h"
 #include "libc/dce.h"
 #include "libc/fmt/conv.h"
+#include "libc/intrin/cxaatexit.h"
 #include "libc/macros.h"
 #include "libc/nt/accounting.h"
 #include "libc/runtime/runtime.h"
-#include "libc/thread/thread.h"
 
+#define CTOR  __attribute__((__constructor__(99)))
 #define FT(x) (x.dwLowDateTime | (uint64_t)x.dwHighDateTime << 32)
 
 static int cpus;
 static double load;
-static pthread_spinlock_t lock;
 static struct NtFileTime idle1, kern1, user1;
 
 textwindows int sys_getloadavg_nt(double *a, int n) {
   int i, rc;
   uint64_t elapsed, used;
   struct NtFileTime idle, kern, user;
-  BLOCK_SIGNALS;
-  pthread_spin_lock(&lock);
+  __cxa_lock();
   if (GetSystemTimes(&idle, &kern, &user)) {
     elapsed = (FT(kern) - FT(kern1)) + (FT(user) - FT(user1));
     if (elapsed) {
@@ -54,12 +53,11 @@ textwindows int sys_getloadavg_nt(double *a, int n) {
   } else {
     rc = __winerr();
   }
-  pthread_spin_unlock(&lock);
-  ALLOW_SIGNALS;
+  __cxa_unlock();
   return rc;
 }
 
-__attribute__((__constructor__(40))) static textstartup void ntinitload(void) {
+CTOR static textstartup void sys_getloadavg_nt_init(void) {
   if (IsWindows()) {
     load = 1;
     cpus = __get_cpu_count() / 2;

libc/calls/getprogramexecutablename.greg.c

@@ -96,9 +96,8 @@ static int OldApeLoader(char *s) {
 static int CopyWithCwd(const char *q, char *p, char *e) {
   char c;
   if (*q != '/') {
-    if (q[0] == '.' && q[1] == '/') {
+    if (q[0] == '.' && q[1] == '/')
       q += 2;
-    }
     int got = __getcwd(p, e - p - 1 /* '/' */);
     if (got != -1) {
       p += got - 1;
@@ -118,9 +117,10 @@ static int CopyWithCwd(const char *q, char *p, char *e) {
 
 // if q exists then turn it into an absolute path.
 static int TryPath(const char *q) {
-  if (!CopyWithCwd(q, g_prog.u.buf, g_prog.u.buf + sizeof(g_prog.u.buf))) {
+  if (!q)
+    return 0;
+  if (!CopyWithCwd(q, g_prog.u.buf, g_prog.u.buf + sizeof(g_prog.u.buf)))
     return 0;
-  }
   return !sys_faccessat(AT_FDCWD, g_prog.u.buf, F_OK, 0);
 }
 
@@ -129,9 +129,8 @@ static int TryPath(const char *q) {
 void __init_program_executable_name(void) {
   if (__program_executable_name && *__program_executable_name != '/' &&
       CopyWithCwd(__program_executable_name, g_prog.u.buf,
-                  g_prog.u.buf + sizeof(g_prog.u.buf))) {
+                  g_prog.u.buf + sizeof(g_prog.u.buf)))
     __program_executable_name = g_prog.u.buf;
-  }
 }
 
 static inline void InitProgramExecutableNameImpl(void) {
@@ -212,14 +211,12 @@ static inline void InitProgramExecutableNameImpl(void) {
   }
 
   // don't trust argv or envp if set-id.
-  if (issetugid()) {
+  if (issetugid())
     goto UseEmpty;
-  }
 
   // try argv[0], then then $_.
-  if (TryPath(__argv[0]) || TryPath(__getenv(__envp, "_").s)) {
+  if (TryPath(__argv[0]) || TryPath(__getenv(__envp, "_").s))
     goto UseBuf;
-  }
 
   // give up and just copy argv[0] into it
   if ((q = __argv[0])) {

libc/calls/state.internal.h

@@ -13,7 +13,6 @@ extern unsigned __sighandflags[NSIG + 1];
 extern uint64_t __sighandmask[NSIG + 1];
 extern const struct NtSecurityAttributes kNtIsInheritable;
 
-void __fds_wipe(void);
 void __fds_lock(void);
 void __fds_unlock(void);
 

libc/calls/struct/sigset.internal.h

@@ -5,27 +5,15 @@
 #include "libc/sysv/consts/sig.h"
 COSMOPOLITAN_C_START_
 
-#ifndef MODE_DBG
-/* block sigs because theoretical edge cases */
 #define BLOCK_SIGNALS  \
   do {                 \
     sigset_t _SigMask; \
   _SigMask = __sig_block()
+
 #define ALLOW_SIGNALS      \
   __sig_unblock(_SigMask); \
   }                        \
   while (0)
-#else
-/* doesn't block signals so we can get a crash
-   report, when a core runtime library crashes */
-#define BLOCK_SIGNALS  \
-  do {                 \
-    sigset_t _SigMask; \
-  sigprocmask(SIG_SETMASK, 0, &_SigMask)
-#define ALLOW_SIGNALS \
-  }                   \
-  while (0)
-#endif
 
 sigset_t __sig_block(void);
 void __sig_unblock(sigset_t);

libc/cosmo.h

@@ -22,5 +22,11 @@ int cosmo_futex_wake(_COSMO_ATOMIC(int) *, int, char);
 int cosmo_futex_wait(_COSMO_ATOMIC(int) *, int, char, int,
                      const struct timespec *);
 
+int __deadlock_check(void *, int) libcesque;
+int __deadlock_tracked(void *) libcesque;
+void __deadlock_record(void *, int) libcesque;
+void __deadlock_track(void *, int) libcesque;
+void __deadlock_untrack(void *) libcesque;
+
 COSMOPOLITAN_C_END_
 #endif /* COSMOPOLITAN_LIBC_COSMO_H_ */

libc/errno.h

@@ -26,11 +26,11 @@ COSMOPOLITAN_C_START_
 /* this header is included by 700+ files; therefore we */
 /* hand-roll &__get_tls()->tib_errno to avoid #include */
 /* cosmopolitan uses x28 as the tls register b/c apple */
-#define errno                                      \
-  (*__extension__({                                \
-    errno_t *__ep;                                 \
-    __asm__("sub\t%0,x28,#512-0x3c" : "=r"(__ep)); \
-    __ep;                                          \
+#define errno                                       \
+  (*__extension__({                                 \
+    errno_t *__ep;                                  \
+    __asm__("sub\t%0,x28,#1024-0x3c" : "=r"(__ep)); \
+    __ep;                                           \
   }))
 #else
 #define errno (*__errno_location())

libc/integral/c.inc

@@ -135,7 +135,7 @@ typedef struct {
 #define strftimeesque(n) __attribute__((__format__(__strftime__, n, 0)))
 
 #ifndef privileged
-#define privileged _Section(".privileged") dontinline dontinstrument dontubsan
+#define privileged _Section(".privileged") dontinstrument dontubsan
 #endif
 
 #ifndef wontreturn

libc/intrin/__getenv.c

@@ -20,7 +20,7 @@
 #include "libc/intrin/getenv.h"
 #include "libc/intrin/kprintf.h"
 
-privileged struct Env __getenv(char **p, const char *k) {
+privileged optimizesize struct Env __getenv(char **p, const char *k) {
   char *t;
   int i, j;
   for (i = 0; (t = p[i]); ++i) {

libc/intrin/bzero.c

@@ -16,155 +16,18 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
-#include "libc/assert.h"
-#include "libc/dce.h"
-#include "libc/nexgen32e/nexgen32e.h"
-#include "libc/nexgen32e/x86feature.h"
 #include "libc/str/str.h"
 
-typedef char xmm_t __attribute__((__vector_size__(16), __aligned__(1)));
-typedef long long xmm_a __attribute__((__vector_size__(16), __aligned__(16)));
-
-static void bzero128(char *p, size_t n) {
-  xmm_t v = {0};
-  if (n <= 32) {
-    *(xmm_t *)(p + n - 16) = v;
-    *(xmm_t *)p = v;
-  } else {
-    do {
-      n -= 32;
-      *(xmm_t *)(p + n) = v;
-      *(xmm_t *)(p + n + 16) = v;
-    } while (n > 32);
-    *(xmm_t *)(p + 16) = v;
-    *(xmm_t *)p = v;
-  }
-}
-
-#if defined(__x86_64__) && !defined(__chibicc__)
-_Microarchitecture("avx") static void bzero_avx(char *p, size_t n) {
-  xmm_t v = {0};
-  if (n <= 32) {
-    *(xmm_t *)(p + n - 16) = v;
-    *(xmm_t *)p = v;
-  } else if (n >= 1024 && X86_HAVE(ERMS)) {
-    asm("rep stosb" : "+D"(p), "+c"(n), "=m"(*(char(*)[n])p) : "a"(0));
-  } else {
-    if (n < kHalfCache3 || !kHalfCache3) {
-      do {
-        n -= 32;
-        *(xmm_t *)(p + n) = v;
-        *(xmm_t *)(p + n + 16) = v;
-      } while (n > 32);
-    } else {
-      while ((uintptr_t)(p + n) & 15) {
-        p[--n] = 0;
-      }
-      do {
-        n -= 32;
-        __builtin_ia32_movntdq((xmm_a *)(p + n), (xmm_a)v);
-        __builtin_ia32_movntdq((xmm_a *)(p + n + 16), (xmm_a)v);
-      } while (n > 32);
-      asm("sfence");
-    }
-    *(xmm_t *)(p + 16) = v;
-    *(xmm_t *)p = v;
-  }
-}
-#endif
-
 /**
  * Sets memory to zero.
  *
- *     bzero n=0                          661 picoseconds
- *     bzero n=1                          661 ps/byte          1,476 mb/s
- *     bzero n=2                          330 ps/byte          2,952 mb/s
- *     bzero n=3                          220 ps/byte          4,428 mb/s
- *     bzero n=4                          165 ps/byte          5,904 mb/s
- *     bzero n=7                           94 ps/byte         10,333 mb/s
- *     bzero n=8                           41 ps/byte         23,618 mb/s
- *     bzero n=15                          44 ps/byte         22,142 mb/s
- *     bzero n=16                          20 ps/byte         47,236 mb/s
- *     bzero n=31                          21 ps/byte         45,760 mb/s
- *     bzero n=32                          20 ps/byte         47,236 mb/s
- *     bzero n=63                          10 ps/byte         92,997 mb/s
- *     bzero n=64                          15 ps/byte         62,982 mb/s
- *     bzero n=127                         15 ps/byte         62,490 mb/s
- *     bzero n=128                         10 ps/byte         94,473 mb/s
- *     bzero n=255                         14 ps/byte         68,439 mb/s
- *     bzero n=256                          9 ps/byte            105 gb/s
- *     bzero n=511                         15 ps/byte         62,859 mb/s
- *     bzero n=512                         11 ps/byte         83,976 mb/s
- *     bzero n=1023                        15 ps/byte         61,636 mb/s
- *     bzero n=1024                        10 ps/byte         88,916 mb/s
- *     bzero n=2047                         9 ps/byte            105 gb/s
- *     bzero n=2048                         8 ps/byte            109 gb/s
- *     bzero n=4095                         8 ps/byte            115 gb/s
- *     bzero n=4096                         8 ps/byte            118 gb/s
- *     bzero n=8191                         7 ps/byte            129 gb/s
- *     bzero n=8192                         7 ps/byte            130 gb/s
- *     bzero n=16383                        6 ps/byte            136 gb/s
- *     bzero n=16384                        6 ps/byte            137 gb/s
- *     bzero n=32767                        6 ps/byte            140 gb/s
- *     bzero n=32768                        6 ps/byte            141 gb/s
- *     bzero n=65535                       15 ps/byte         64,257 mb/s
- *     bzero n=65536                       15 ps/byte         64,279 mb/s
- *     bzero n=131071                      15 ps/byte         63,166 mb/s
- *     bzero n=131072                      15 ps/byte         63,115 mb/s
- *     bzero n=262143                      15 ps/byte         62,052 mb/s
- *     bzero n=262144                      15 ps/byte         62,097 mb/s
- *     bzero n=524287                      15 ps/byte         61,699 mb/s
- *     bzero n=524288                      15 ps/byte         61,674 mb/s
- *     bzero n=1048575                     16 ps/byte         60,179 mb/s
- *     bzero n=1048576                     15 ps/byte         61,330 mb/s
- *     bzero n=2097151                     15 ps/byte         61,071 mb/s
- *     bzero n=2097152                     15 ps/byte         61,065 mb/s
- *     bzero n=4194303                     16 ps/byte         60,942 mb/s
- *     bzero n=4194304                     16 ps/byte         60,947 mb/s
- *     bzero n=8388607                     16 ps/byte         60,872 mb/s
- *     bzero n=8388608                     16 ps/byte         60,879 mb/s
- *
  * @param p is memory address
  * @param n is byte length
  * @return p
  * @asyncsignalsafe
  */
 void bzero(void *p, size_t n) {
-  char *b;
-  uint64_t x;
-  b = p;
-#ifdef __x86_64__
-  asm("xorl\t%k0,%k0" : "=r"(x));
-#else
-  if (1) {
-    memset(p, 0, n);
-    return;
-  }
-  x = 0;
-#endif
-  if (n <= 16) {
-    if (n >= 8) {
-      __builtin_memcpy(b, &x, 8);
-      __builtin_memcpy(b + n - 8, &x, 8);
-    } else if (n >= 4) {
-      __builtin_memcpy(b, &x, 4);
-      __builtin_memcpy(b + n - 4, &x, 4);
-    } else if (n) {
-      do {
-        asm volatile("" ::: "memory");
-        b[--n] = x;
-      } while (n);
-    }
-#if defined(__x86_64__) && !defined(__chibicc__)
-  } else if (IsTiny()) {
-    asm("rep stosb" : "+D"(b), "+c"(n), "=m"(*(char(*)[n])b) : "a"(0));
-    return;
-  } else if (X86_HAVE(AVX)) {
-    bzero_avx(b, n);
-#endif
-  } else {
-    bzero128(b, n);
-  }
+  memset(p, 0, n);
 }
 
 __weak_reference(bzero, explicit_bzero);

libc/intrin/cxalock.c

@@ -19,11 +19,7 @@
 #include "libc/intrin/cxaatexit.h"
 #include "libc/thread/thread.h"
 
-static pthread_mutex_t __cxa_lock_obj;
-
-void __cxa_wipe(void) {
-  pthread_mutex_init(&__cxa_lock_obj, 0);
-}
+pthread_mutex_t __cxa_lock_obj = PTHREAD_MUTEX_INITIALIZER;
 
 void __cxa_lock(void) {
   pthread_mutex_lock(&__cxa_lock_obj);
@@ -32,7 +28,3 @@ void __cxa_lock(void) {
 void __cxa_unlock(void) {
   pthread_mutex_unlock(&__cxa_lock_obj);
 }
-
-__attribute__((__constructor__(60))) static textstartup void __cxa_init() {
-  pthread_atfork(__cxa_lock, __cxa_unlock, __cxa_wipe);
-}