Support process shared pthread_rwlock

Cosmo now has a non-nsync implementation of POSIX read-write locks. It's
possible to call pthread_rwlockattr_setpshared in PTHREAD_PROCESS_SHARED
mode. Furthermore, if cosmo is built with PTHREAD_USE_NSYNC set to zero,
then Cosmo shouldn't use nsync at all. That's helpful if you want to not
link any Apache 2.0 licensed code.

Author: jart

libc/thread/pthread_rwlock_destroy.c

@@ -16,16 +16,30 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/errno.h"
+#include "libc/intrin/atomic.h"
 #include "libc/str/str.h"
 #include "libc/thread/thread.h"
+#include "third_party/nsync/mu.h"
 
 /**
  * Destroys read-write lock.
  *
  * @return 0 on success, or error number on failure
- * @raise EINVAL if any threads still hold the lock
+ * @raise EBUSY if any threads still hold the lock
  */
 errno_t pthread_rwlock_destroy(pthread_rwlock_t *rwlock) {
+
+  // check if lock is held
+  if (!rwlock->_pshared) {
+    nsync_mu *mu = (nsync_mu *)rwlock->_nsync;
+    if (atomic_load_explicit(&mu->word, memory_order_relaxed))
+      return EBUSY;
+  } else {
+    if (atomic_load_explicit(&rwlock->_word, memory_order_relaxed))
+      return EBUSY;
+  }
+
   memset(rwlock, -1, sizeof(*rwlock));
   return 0;
 }

libc/thread/pthread_rwlock_init.c

@@ -26,6 +26,8 @@
  */
 errno_t pthread_rwlock_init(pthread_rwlock_t *rwlock,
                             const pthread_rwlockattr_t *attr) {
-  *rwlock = (pthread_rwlock_t){0};
+  *rwlock = (pthread_rwlock_t){
+      ._pshared = attr ? *attr : PTHREAD_PROCESS_PRIVATE,
+  };
   return 0;
 }

libc/thread/pthread_rwlock_rdlock.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/intrin/atomic.h"
 #include "libc/thread/thread.h"
 #include "third_party/nsync/mu.h"
 
@@ -24,7 +25,25 @@
  *
  * @return 0 on success, or errno on error
  */
-errno_t pthread_rwlock_rdlock(pthread_rwlock_t *rwlock) {
-  nsync_mu_rlock((nsync_mu *)rwlock);
-  return 0;
+errno_t pthread_rwlock_rdlock(pthread_rwlock_t *lk) {
+
+#if PTHREAD_USE_NSYNC
+  // use nsync if possible
+  if (!lk->_pshared) {
+    nsync_mu_rlock((nsync_mu *)lk->_nsync);
+    return 0;
+  }
+#endif
+
+  // naive implementation
+  uint32_t w = 0;
+  for (;;) {
+    if (w & 1)
+      for (;;)
+        if (~(w = atomic_load_explicit(&lk->_word, memory_order_relaxed)) & 1)
+          break;
+    if (atomic_compare_exchange_weak_explicit(
+            &lk->_word, &w, w + 2, memory_order_acquire, memory_order_relaxed))
+      return 0;
+  }
 }

libc/thread/pthread_rwlock_tryrdlock.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/errno.h"
+#include "libc/intrin/atomic.h"
 #include "libc/thread/thread.h"
 #include "third_party/nsync/mu.h"
 
@@ -29,9 +30,26 @@
  * @raise EINVAL if `rwlock` doesn't refer to an initialized r/w lock
  */
 errno_t pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock) {
-  if (nsync_mu_rtrylock((nsync_mu *)rwlock)) {
-    return 0;
-  } else {
-    return EBUSY;
+
+#if PTHREAD_USE_NSYNC
+  // use nsync if possible
+  if (!rwlock->_pshared) {
+    if (nsync_mu_rtrylock((nsync_mu *)rwlock->_nsync)) {
+      return 0;
+    } else {
+      return EBUSY;
+    }
+  }
+#endif
+
+  // naive implementation
+  uint32_t word = 0;
+  for (;;) {
+    if (word & 1)
+      return EBUSY;
+    if (atomic_compare_exchange_weak_explicit(&rwlock->_word, &word, word + 2,
+                                              memory_order_acquire,
+                                              memory_order_relaxed))
+      return 0;
   }
 }

libc/thread/pthread_rwlock_trywrlock.c

@@ -17,6 +17,7 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/errno.h"
+#include "libc/intrin/atomic.h"
 #include "libc/thread/thread.h"
 #include "third_party/nsync/mu.h"
 
@@ -28,10 +29,23 @@
  * @raise EINVAL if `rwlock` doesn't refer to an initialized r/w lock
  */
 errno_t pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock) {
-  if (nsync_mu_trylock((nsync_mu *)rwlock)) {
-    rwlock->_iswrite = 1;
-    return 0;
-  } else {
-    return EBUSY;
+
+#if PTHREAD_USE_NSYNC
+  // use nsync if possible
+  if (!rwlock->_pshared) {
+    if (nsync_mu_trylock((nsync_mu *)rwlock->_nsync)) {
+      rwlock->_iswrite = 1;
+      return 0;
+    } else {
+      return EBUSY;
+    }
   }
+#endif
+
+  // naive implementation
+  uint32_t word = 0;
+  if (atomic_compare_exchange_strong_explicit(
+          &rwlock->_word, &word, 1, memory_order_acquire, memory_order_relaxed))
+    return 0;
+  return EBUSY;
 }

libc/thread/pthread_rwlock_unlock.c

@@ -16,6 +16,8 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/errno.h"
+#include "libc/intrin/atomic.h"
 #include "libc/thread/thread.h"
 #include "third_party/nsync/mu.h"
 
@@ -26,11 +28,33 @@
  * @raise EINVAL if lock is in a bad state
  */
 errno_t pthread_rwlock_unlock(pthread_rwlock_t *rwlock) {
-  if (rwlock->_iswrite) {
-    rwlock->_iswrite = 0;
-    nsync_mu_unlock((nsync_mu *)rwlock);
-  } else {
-    nsync_mu_runlock((nsync_mu *)rwlock);
+
+#if PTHREAD_USE_NSYNC
+  // use nsync if possible
+  if (!rwlock->_pshared) {
+    if (rwlock->_iswrite) {
+      rwlock->_iswrite = 0;
+      nsync_mu_unlock((nsync_mu *)rwlock->_nsync);
+    } else {
+      nsync_mu_runlock((nsync_mu *)rwlock->_nsync);
+    }
+    return 0;
+  }
+#endif
+
+  // naive implementation
+  uint32_t word = atomic_load_explicit(&rwlock->_word, memory_order_relaxed);
+  for (;;) {
+    if (word & 1) {
+      atomic_store_explicit(&rwlock->_word, 0, memory_order_release);
+      return 0;
+    } else if (word) {
+      if (atomic_compare_exchange_weak_explicit(&rwlock->_word, &word, word - 2,
+                                                memory_order_release,
+                                                memory_order_relaxed))
+        return 0;
+    } else {
+      return EPERM;
+    }
   }
-  return 0;
 }

libc/thread/pthread_rwlock_wrlock.c

@@ -16,6 +16,7 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/intrin/atomic.h"
 #include "libc/thread/thread.h"
 #include "third_party/nsync/mu.h"
 
@@ -25,7 +26,24 @@
  * @return 0 on success, or errno on error
  */
 errno_t pthread_rwlock_wrlock(pthread_rwlock_t *rwlock) {
-  nsync_mu_lock((nsync_mu *)rwlock);
-  rwlock->_iswrite = 1;
-  return 0;
+
+#if PTHREAD_USE_NSYNC
+  // use nsync if possible
+  if (!rwlock->_pshared) {
+    nsync_mu_lock((nsync_mu *)rwlock->_nsync);
+    rwlock->_iswrite = 1;
+    return 0;
+  }
+#endif
+
+  // naive implementation
+  uint32_t w = 0;
+  for (;;) {
+    if (atomic_compare_exchange_weak_explicit(
+            &rwlock->_word, &w, 1, memory_order_acquire, memory_order_relaxed))
+      return 0;
+    for (;;)
+      if (!(w = atomic_load_explicit(&rwlock->_word, memory_order_relaxed)))
+        break;
+  }
 }

libc/thread/pthread_rwlockattr_getpshared.c

@@ -23,7 +23,7 @@
  *
  * @param pshared is set to one of the following
  *     - `PTHREAD_PROCESS_PRIVATE` (default)
- *     - `PTHREAD_PROCESS_SHARED` (unsupported)
+ *     - `PTHREAD_PROCESS_SHARED`
  * @return 0 on success, or error on failure
  */
 errno_t pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *attr,

libc/thread/pthread_rwlockattr_setpshared.c

@@ -24,13 +24,14 @@
  *
  * @param pshared can be one of
  *     - `PTHREAD_PROCESS_PRIVATE` (default)
- *     - `PTHREAD_PROCESS_SHARED` (unsupported)
+ *     - `PTHREAD_PROCESS_SHARED`
  * @return 0 on success, or error on failure
  * @raises EINVAL if `pshared` is invalid
  */
 errno_t pthread_rwlockattr_setpshared(pthread_rwlockattr_t *attr, int pshared) {
   switch (pshared) {
     case PTHREAD_PROCESS_PRIVATE:
+    case PTHREAD_PROCESS_SHARED:
       *attr = pshared;
       return 0;
     default:

libc/thread/thread.h

@@ -107,8 +107,15 @@ typedef struct pthread_cond_s {
 } pthread_cond_t;
 
 typedef struct pthread_rwlock_s {
-  void *_nsync[2];
-  char _iswrite;
+  union {
+    void *_nsync[2];
+    struct {
+      uint32_t _nsync_word;
+      char _pshared;
+      char _iswrite;
+      _PTHREAD_ATOMIC(uint32_t) _word;
+    };
+  };
 } pthread_rwlock_t;
 
 typedef struct pthread_barrier_s {

test/libc/thread/pthread_rwlock_rdlock_test.c

@@ -17,23 +17,48 @@
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
 #include "libc/atomic.h"
+#include "libc/calls/calls.h"
 #include "libc/mem/gc.h"
 #include "libc/mem/mem.h"
 #include "libc/testlib/testlib.h"
 #include "libc/thread/thread.h"
 
-#define ITERATIONS 50000
-#define READERS    8
-#define WRITERS    2
+#define READERS           8
+#define WRITERS           2
+#define READER_ITERATIONS 10000
+#define WRITER_ITERATIONS 1000
 
+int writes;
 atomic_int reads;
-atomic_int writes;
 pthread_rwlock_t lock;
+pthread_rwlockattr_t attr;
 pthread_barrier_t barrier;
 
+FIXTURE(pthread_rwlock, private) {
+  reads = 0;
+  writes = 0;
+  ASSERT_EQ(0, pthread_rwlockattr_init(&attr));
+  ASSERT_EQ(0, pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_PRIVATE));
+  ASSERT_EQ(0, pthread_rwlock_init(&lock, &attr));
+  ASSERT_EQ(0, pthread_rwlockattr_destroy(&attr));
+}
+
+FIXTURE(pthread_rwlock, pshared) {
+  reads = 0;
+  writes = 0;
+  ASSERT_EQ(0, pthread_rwlockattr_init(&attr));
+  ASSERT_EQ(0, pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED));
+  ASSERT_EQ(0, pthread_rwlock_init(&lock, &attr));
+  ASSERT_EQ(0, pthread_rwlockattr_destroy(&attr));
+}
+
+void TearDown(void) {
+  ASSERT_EQ(0, pthread_rwlock_destroy(&lock));
+}
+
 void *Reader(void *arg) {
   pthread_barrier_wait(&barrier);
-  for (int i = 0; i < ITERATIONS; ++i) {
+  for (int i = 0; i < READER_ITERATIONS; ++i) {
     ASSERT_EQ(0, pthread_rwlock_rdlock(&lock));
     ++reads;
     ASSERT_EQ(0, pthread_rwlock_unlock(&lock));
@@ -43,10 +68,12 @@ void *Reader(void *arg) {
 
 void *Writer(void *arg) {
   pthread_barrier_wait(&barrier);
-  for (int i = 0; i < ITERATIONS; ++i) {
+  for (int i = 0; i < WRITER_ITERATIONS; ++i) {
     ASSERT_EQ(0, pthread_rwlock_wrlock(&lock));
     ++writes;
     ASSERT_EQ(0, pthread_rwlock_unlock(&lock));
+    for (volatile int i = 0; i < 100; ++i)
+      pthread_pause_np();
   }
   return 0;
 }
@@ -62,7 +89,7 @@ TEST(pthread_rwlock_rdlock, test) {
   for (i = 0; i < READERS + WRITERS; ++i) {
     EXPECT_SYS(0, 0, pthread_join(t[i], 0));
   }
-  EXPECT_EQ(READERS * ITERATIONS, reads);
-  EXPECT_EQ(WRITERS * ITERATIONS, writes);
+  EXPECT_EQ(READERS * READER_ITERATIONS, reads);
+  EXPECT_EQ(WRITERS * WRITER_ITERATIONS, writes);
   ASSERT_EQ(0, pthread_barrier_destroy(&barrier));
 }