allow malloc outside of GVL

We currently do not support calling ruby_xmalloc out of GVL.  OTOH
oftentimes people do `#define malloc ruby_xmalloc`.  Under GVL that
has no practical problem but without one it must be a nightmare.

Because whether a runtime context has GVL or not is not fixed when
the macro is expanded, why don't we actively check that property
on the fly and (instead of abnormal end of the process) return NULL.

Better safe than sorry I guess.

Author: shyouhei

gc.c

@@ -4208,6 +4208,19 @@ rb_malloc_info_show_results(void)
 void *
 ruby_xmalloc(size_t size)
 {
+    if (! ruby_thread_has_gvl_p()) {
+        /* if (ruby_native_thread_p()) {
+         *     // We could reacquire the GVL here using rb_thread_call_with_gvl
+         *     // but what if that raised an  exception?  There would be no way
+         *     // to recover.
+         *     //
+         *     // This saves no one from nothing.
+         *     return rb_thread_call_with_gvl(ruby_xmalloc, (void*)size);
+         * }
+         */
+        return ruby_mimmalloc(size);
+    }
+
     if ((ssize_t)size < 0) {
         negative_size_allocation_error("too large allocation size");
     }
@@ -4226,12 +4239,38 @@ ruby_malloc_size_overflow(size_t count, size_t elsize)
 void *
 ruby_xmalloc2(size_t n, size_t size)
 {
+    if (! ruby_thread_has_gvl_p()) {
+        /* Out of GVL.  No GC, and no way to inform the integer overflow. */
+        struct rbimpl_size_mul_overflow_tag of =
+            rbimpl_size_mul_overflow(n, size);
+
+        if (of.left) {
+            return NULL;
+        }
+        else {
+            return ruby_mimmalloc(of.right);
+        }
+    }
+
     return rb_gc_impl_malloc(rb_gc_get_objspace(), xmalloc2_size(n, size));
 }
 
 void *
 ruby_xcalloc(size_t n, size_t size)
 {
+    if (! ruby_thread_has_gvl_p()) {
+        /* Out of GVL.  No GC, and no way to inform the integer overflow. */
+        struct rbimpl_size_mul_overflow_tag of =
+            rbimpl_size_mul_overflow(n, size);
+
+        if (of.left) {
+            return NULL;
+        }
+        else {
+            return ruby_mimcalloc(of.right, 1);
+        }
+    }
+
     return rb_gc_impl_calloc(rb_gc_get_objspace(), xmalloc2_size(n, size));
 }
 
@@ -4241,6 +4280,12 @@ ruby_xcalloc(size_t n, size_t size)
 void *
 ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size)
 {
+    if (! ruby_thread_has_gvl_p()) {
+        /* Out of GVL, no GC. */
+        /* Also there is no such thing like ruby_mimrealloc. */
+        return realloc(ptr, new_size);
+    }
+
     if ((ssize_t)new_size < 0) {
         negative_size_allocation_error("too large allocation size");
     }
@@ -4260,6 +4305,24 @@ ruby_xrealloc(void *ptr, size_t new_size)
 void *
 ruby_sized_xrealloc2(void *ptr, size_t n, size_t size, size_t old_n)
 {
+    if (! ruby_thread_has_gvl_p()) {
+        /* Out of GVL.  No GC, and no way to inform the integer overflow. */
+        struct rbimpl_size_mul_overflow_tag new_o, old_o;
+
+        new_o = rbimpl_size_mul_overflow(n, size);
+        old_o = rbimpl_size_mul_overflow(old_n, size);
+
+        if (new_o.left) {
+            return NULL;
+        }
+        else if (old_o.left) {
+            return NULL;
+        }
+        else {
+            return realloc(ptr, new_o.right);
+        }
+    }
+
     size_t len = xmalloc2_size(n, size);
     return rb_gc_impl_realloc(rb_gc_get_objspace(), ptr, len, old_n * size);
 }

include/ruby/internal/xmalloc.h

@@ -61,21 +61,23 @@ RBIMPL_SYMBOL_EXPORT_BEGIN()
 
 RBIMPL_ATTR_NODISCARD()
 RBIMPL_ATTR_RESTRICT()
-RBIMPL_ATTR_RETURNS_NONNULL()
 RBIMPL_ATTR_ALLOC_SIZE((1))
 /**
- * Allocates a  storage instance.  It is  largely the same as  system malloc(),
- * except:
+ * Allocates a  storage instance.  It is  largely the same as  system malloc().
+ * Especially when called from outside of GVL this is identical to it, at least
+ * API-wise.   But  when under  GVL,  it  exercises some  additional  manoeuvre
+ * namely:
  *
  *   - It raises Ruby exceptions instead of returning NULL, and
  *   - In case of `ENOMEM` it tries to GC to make some room.
  *
  * @param[in]  size            Requested amount of memory.
  * @exception  rb_eNoMemError  No space left for `size` bytes allocation.
- * @return     A valid pointer  to an allocated storage instance;  which has at
- *             least `size` bytes width, with appropriate alignment detected by
- *             the underlying malloc() routine.
- * @note       It doesn't return NULL.
+ * @retval     NULL            Allocation failed but no way to raise anything.
+ * @retval     otherwise       A   valid  pointer   to  an   allocated  storage
+ *                             instance; which has at least `size` bytes width,
+ *                             with  appropriate  alignment   detected  by  the
+ *                             underlying malloc() routine.
  * @note       Unlike some malloc() implementations, it allocates something and
  *             returns a meaningful value even when `size` is equal to zero.
  * @warning    The return  value shall  be invalidated  exactly once  by either
@@ -89,7 +91,6 @@ RBIMPL_ATTR_NOEXCEPT(malloc(size))
 
 RBIMPL_ATTR_NODISCARD()
 RBIMPL_ATTR_RESTRICT()
-RBIMPL_ATTR_RETURNS_NONNULL()
 RBIMPL_ATTR_ALLOC_SIZE((1,2))
 /**
  * Identical to ruby_xmalloc(), except it allocates `nelems` * `elemsiz` bytes.
@@ -102,10 +103,12 @@ RBIMPL_ATTR_ALLOC_SIZE((1,2))
  * @param[in]  elemsiz         Size of an element.
  * @exception  rb_eNoMemError  No space left for allocation.
  * @exception  rb_eArgError    `nelems` * `elemsiz` would overflow.
- * @return     A valid pointer  to an allocated storage instance;  which has at
- *             least  `nelems`  *  `elemsiz`   bytes  width,  with  appropriate
- *             alignment detected by the underlying malloc() routine.
- * @note       It doesn't return NULL.
+ * @retval     NULL            Allocation failed but no way to raise anything.
+ * @retval     otherwise       A   valid  pointer   to  an   allocated  storage
+ *                             instance;   which  has   at  least   `nelems`  *
+ *                             `elemsiz`   bytes    width,   with   appropriate
+ *                             alignment  detected by  the underlying  malloc()
+ *                             routine.
  * @note       Unlike some malloc() implementations, it allocates something and
  *             returns a  meaningful value even  when `nelems` or  `elemsiz` or
  *             both are zero.
@@ -120,7 +123,6 @@ RBIMPL_ATTR_NOEXCEPT(malloc(nelems * elemsiz))
 
 RBIMPL_ATTR_NODISCARD()
 RBIMPL_ATTR_RESTRICT()
-RBIMPL_ATTR_RETURNS_NONNULL()
 RBIMPL_ATTR_ALLOC_SIZE((1,2))
 /**
  * Identical  to  ruby_xmalloc2(),  except  it returns  a  zero-filled  storage
@@ -131,11 +133,13 @@ RBIMPL_ATTR_ALLOC_SIZE((1,2))
  * @param[in]  elemsiz         Size of an element.
  * @exception  rb_eNoMemError  No space left for allocation.
  * @exception  rb_eArgError    `nelems` * `elemsiz` would overflow.
- * @return     A valid pointer  to an allocated storage instance;  which has at
- *             least  `nelems`  *  `elemsiz`   bytes  width,  with  appropriate
- *             alignment detected by the underlying calloc() routine.
+ * @retval     NULL            Allocation failed but no way to raise anything.
+ * @retval     otherwise       A   valid  pointer   to  an   allocated  storage
+ *                             instance;   which  has   at  least   `nelems`  *
+ *                             `elemsiz`   bytes    width,   with   appropriate
+ *                             alignment  detected by  the underlying  calloc()
+ *                             routine.
  * @post       The returned storage instance is filled with zeros.
- * @note       It doesn't return NULL.
  * @note       Unlike some calloc() implementations, it allocates something and
  *             returns a  meaningful value even  when `nelems` or  `elemsiz` or
  *             both are zero.
@@ -149,7 +153,6 @@ RBIMPL_ATTR_NOEXCEPT(calloc(nelems, elemsiz))
 ;
 
 RBIMPL_ATTR_NODISCARD()
-RBIMPL_ATTR_RETURNS_NONNULL()
 RBIMPL_ATTR_ALLOC_SIZE((2))
 /**
  * Resize the storage instance.
@@ -163,10 +166,11 @@ RBIMPL_ATTR_ALLOC_SIZE((2))
  *                               - ruby_xrealloc2().
  * @param[in]  newsiz          Requested new amount of memory.
  * @exception  rb_eNoMemError  No space left for `newsiz` bytes allocation.
- * @return     A  valid  pointer  to   a  (possibly  newly  allocated)  storage
- *             instance;  which  has  at   least  `newsiz`  bytes  width,  with
- *             appropriate  alignment  detected  by  the  underlying  realloc()
- *             routine.
+ * @retval     NULL            Allocation failed but no way to raise anything.
+ * @retval     otherwise       A valid pointer to  a (possibly newly allocated)
+ *                             storage  instance; which  has at  least `newsiz`
+ *                             bytes width, with appropriate alignment detected
+ *                             by the underlying realloc() routine.
  * @pre        The passed pointer must point  to a valid live storage instance.
  *             It is a failure to pass an already freed pointer.
  * @post       In  case the  function  returns the  passed  pointer as-is,  the
@@ -175,7 +179,6 @@ RBIMPL_ATTR_ALLOC_SIZE((2))
  *             pointer to a  newly allocated storage instance  is returned.  In
  *             this  case  `ptr`  is  invalidated   as  if  it  was  passed  to
  *             ruby_xfree().
- * @note       It doesn't return NULL.
  * @warning    Unlike some realloc() implementations,  passing zero to `newsiz`
  *             is not the  same as calling ruby_xfree(),  because this function
  *             never returns NULL.  Something meaningful still returns then.
@@ -195,7 +198,6 @@ RBIMPL_ATTR_NOEXCEPT(realloc(ptr, newsiz))
 ;
 
 RBIMPL_ATTR_NODISCARD()
-RBIMPL_ATTR_RETURNS_NONNULL()
 RBIMPL_ATTR_ALLOC_SIZE((2,3))
 /**
  * Identical to ruby_xrealloc(),  except it resizes the  given storage instance
@@ -219,10 +221,12 @@ RBIMPL_ATTR_ALLOC_SIZE((2,3))
  * @param[in]  newsiz          Requested new size of each element.
  * @exception  rb_eNoMemError  No space left for  allocation.
  * @exception  rb_eArgError    `newelems` * `newsiz` would overflow.
- * @return     A  valid  pointer  to   a  (possibly  newly  allocated)  storage
- *             instance; which has at least  `newelems` * `newsiz` bytes width,
- *             with appropriate alignment detected  by the underlying realloc()
- *             routine.
+ * @retval     NULL            Allocation failed but no way to raise anything.
+ * @retval     otherwise       A valid pointer to  a (possibly newly allocated)
+ *                             storage instance; which  has at least `newelems`
+ *                             * `newsiz`   bytes   width,   with   appropriate
+ *                             alignment detected  by the  underlying realloc()
+ *                             routine.
  * @pre        The passed pointer must point  to a valid live storage instance.
  *             It is a failure to pass an already freed pointer.
  * @post       In  case the  function  returns the  passed  pointer as-is,  the
@@ -231,7 +235,6 @@ RBIMPL_ATTR_ALLOC_SIZE((2,3))
  *             Otherwise a valid pointer to  a newly allocated storage instance
  *             is returned.   In this case  `ptr` is  invalidated as if  it was
  *             passed to ruby_xfree().
- * @note       It doesn't return NULL.
  * @warning    Unlike some  realloc() implementations,  passing zero  to either
  *             `newelems`   or  `elemsiz`   are   not  the   same  as   calling
  *             ruby_xfree(),   because  this   function  never   returns  NULL.

internal/gc.h

@@ -245,8 +245,8 @@ size_t rb_obj_gc_flags(VALUE, ID[], size_t);
 void rb_gc_mark_values(long n, const VALUE *values);
 void rb_gc_mark_vm_stack_values(long n, const VALUE *values);
 void rb_gc_update_values(long n, VALUE *values);
-void *ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2));
-void *ruby_sized_xrealloc2(void *ptr, size_t new_count, size_t element_size, size_t old_count) RUBY_ATTR_RETURNS_NONNULL RUBY_ATTR_ALLOC_SIZE((2, 3));
+void *ruby_sized_xrealloc(void *ptr, size_t new_size, size_t old_size) RUBY_ATTR_ALLOC_SIZE((2));
+void *ruby_sized_xrealloc2(void *ptr, size_t new_count, size_t element_size, size_t old_count) RUBY_ATTR_ALLOC_SIZE((2, 3));
 void ruby_sized_xfree(void *x, size_t size);
 
 const char * rb_gc_active_gc_name(void);