Fix issues with Lua preprocessor of call-in files

- Lua now permits all the types that yottadb permits; notably the ones not beginning with `ydb_`
- Change in call-in file preprocessor:
  - now auto-converts illegal non-pointer return types to pointer-types
  - no longer substitutes ydb types within function and routine names (e.g. previously func/routines named my_ydb_string_t could bechanged to my_ydb_buffer_t. This is more important now that `int` is handled properly as a valid type
  - add optional `debug` parameter to `yottadb.require()` to make for easier call-in file problem solving. Document it.
- Updated documentation in examples/arithmetic.ci to match these changes.

callins.c

@@ -133,16 +133,16 @@ int block_M_signals(lua_State *L) {
 
 // Make table of constants for export to Lua
 const const_Reg yottadb_types[] = {
-  {"ydb_long_t", YDB_LONG_T}, {"ydb_ulong_t", YDB_ULONG_T},
-  {"ydb_int_t", YDB_INT_T}, {"ydb_uint_t", YDB_UINT_T},
-  {"ydb_int64_t", YDB_INT64_T}, {"ydb_uint64_t", YDB_UINT64_T},
-  {"ydb_float_t", YDB_FLOAT_T}, {"ydb_double_t", YDB_DOUBLE_T},
-  {"ydb_long_t*", YDB_LONG_T_PTR}, {"ydb_ulong_t*", YDB_ULONG_T_PTR},
-  {"ydb_int_t*", YDB_INT_T_PTR}, {"ydb_uint_t*", YDB_UINT_T_PTR},
-  {"ydb_int64_t*", YDB_INT64_T_PTR}, {"ydb_uint64_t*", YDB_UINT64_T_PTR},
-  {"ydb_float_t*", YDB_FLOAT_T_PTR}, {"ydb_double_t*", YDB_DOUBLE_T_PTR},
-  {"ydb_char_t*", YDB_CHAR_T_PTR},
-  {"ydb_string_t*", YDB_STRING_T_PTR},
+  {"ydb_long_t", YDB_LONG_T}, {"ydb_ulong_t", YDB_ULONG_T}, {"long", YDB_LONG_T}, {"ulong", YDB_ULONG_T},
+  {"ydb_int_t", YDB_INT_T}, {"ydb_uint_t", YDB_UINT_T}, {"int", YDB_INT_T}, {"uint", YDB_UINT_T},
+  {"ydb_int64_t", YDB_INT64_T}, {"ydb_uint64_t", YDB_UINT64_T}, {"int64", YDB_INT64_T}, {"uint64", YDB_UINT64_T},
+  {"ydb_float_t", YDB_FLOAT_T}, {"ydb_double_t", YDB_DOUBLE_T}, {"float", YDB_FLOAT_T}, {"double", YDB_DOUBLE_T},
+  {"ydb_long_t*", YDB_LONG_T_PTR}, {"ydb_ulong_t*", YDB_ULONG_T_PTR}, {"long*", YDB_LONG_T_PTR}, {"ulong*", YDB_ULONG_T_PTR},
+  {"ydb_int_t*", YDB_INT_T_PTR}, {"ydb_uint_t*", YDB_UINT_T_PTR}, {"int*", YDB_INT_T_PTR}, {"uint*", YDB_UINT_T_PTR},
+  {"ydb_int64_t*", YDB_INT64_T_PTR}, {"ydb_uint64_t*", YDB_UINT64_T_PTR}, {"int64*", YDB_INT64_T_PTR}, {"uint64*", YDB_UINT64_T_PTR},
+  {"ydb_float_t*", YDB_FLOAT_T_PTR}, {"ydb_double_t*", YDB_DOUBLE_T_PTR}, {"float*", YDB_FLOAT_T_PTR}, {"double*", YDB_DOUBLE_T_PTR},
+  {"ydb_char_t*", YDB_CHAR_T_PTR}, {"char*", YDB_CHAR_T_PTR},
+  {"ydb_string_t*", YDB_STRING_T_PTR}, {"string*", YDB_STRING_T_PTR},
   {"ydb_buffer_t*", YDB_BUFFER_T_PTR},
   {"void", VOID},
   {NULL, 0},

callins.h

@@ -12,7 +12,7 @@ typedef struct const_Reg {
   int value;
 } const_Reg;
 
-extern const const_Reg yottadb_types[21];
+extern const const_Reg yottadb_types[39];
 extern const char *LUA_YDB_ERR_PREFIX;
 
 // Make enum that expresses a bitfield for speedy category lookup when processing

docs/lua-yottadb-ydbdocs.rst

@@ -1133,11 +1133,11 @@ Dump the specified node tree.
 
 
 
-~~~~~~~~~~~~~~~~~~~~~~~
-require (Mprototypes)
-~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+require (Mprototypes, debug)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Import M routines as Lua functions specified in ydb 'call-in' file.
+Import M routines as Lua functions specified in a ydb 'call-in' file.
 
 See example call-in file `arithmetic.ci <https://github.com/anet-be/lua-yottadb/blob/master/examples/arithmetic.ci>`_
 and matching M file `arithmetic.m <https://github.com/anet-be/lua-yottadb/blob/master/examples/arithmetic.m>`_.
@@ -1149,6 +1149,10 @@ and matching M file `arithmetic.m <https://github.com/anet-be/lua-yottadb/blob/m
   If the string contains ``:`` it is considered to be the call-in specification itself;
   otherwise it is treated as the filename of a call-in file to be opened and read.
 
+* ``debug``:
+  When neither false nor nil, tell Lua not to delete the temporary preprocessed call-in table file it created.
+  The name of this file will be stored in the ``__ci_filename`` field of the returned table.
+
 
 :Returns:
     A table of functions analogous to a Lua module.
@@ -1163,7 +1167,7 @@ and matching M file `arithmetic.m <https://github.com/anet-be/lua-yottadb/blob/m
     :dedent: 2
     :force:
 
-      $ export ydb_routines=examples   # put arithmetic.m (below) into ydb path
+      $ export ydb_routines=examples   # put arithmetic.m into ydb path
       $ lua -lyottadb
       arithmetic = yottadb.require('examples/arithmetic.ci')
       arithmetic.add_verbose("Sum is:", 2, 3)

docs/yottadb.html

@@ -68,7 +68,7 @@ <h2>Contents</h2>
 <br/>
 <li><a href="#High_level_functions"><b>High level functions </b></a></li>
     <li><a href="#dump">dump (node[, ...[, maxlines=30]])</a></li>
-    <li><a href="#require">require (Mprototypes)</a></li>
+    <li><a href="#require">require (Mprototypes, debug)</a></li>
 <br/>
 <li><a href="#Node_class_operations"><b>Node class operations </b></a></li>
     <li><a href="#inherit">inherit (node_func[, metatable])</a></li>
@@ -248,8 +248,8 @@ <h2><a href="#High_level_functions">High level functions </a></h2>
 	<td class="summary">Dump the specified node tree.</td>
 	</tr>
 	<tr>
-	<td class="name" nowrap><a href="#require">require (Mprototypes)</a></td>
-	<td class="summary">Import M routines as Lua functions specified in ydb 'call-in' file.</td>
+	<td class="name" nowrap><a href="#require">require (Mprototypes, debug)</a></td>
+	<td class="summary">Import M routines as Lua functions specified in a ydb 'call-in' file.</td>
 	</tr>
 </table>
 <h2><a href="#Node_class_operations">Node class operations </a></h2>
@@ -1684,10 +1684,10 @@ <h3>Examples:</h3>
 </dd>
     <dt>
     <a name = "require"></a>
-    <strong>require (Mprototypes)</strong>
+    <strong>require (Mprototypes, debug)</strong>
     </dt>
     <dd>
-    Import M routines as Lua functions specified in ydb 'call-in' file.  <br>
+    Import M routines as Lua functions specified in a ydb 'call-in' file.  <br>
  See example call-in file <a href="https://github.com/anet-be/lua-yottadb/blob/master/examples/arithmetic.ci">arithmetic.ci</a>
  and matching M file <a href="https://github.com/anet-be/lua-yottadb/blob/master/examples/arithmetic.m">arithmetic.m</a>.
 
@@ -1701,6 +1701,10 @@ <h3>Parameters:</h3>
  If the string contains <code>:</code> it is considered to be the call-in specification itself;
  otherwise it is treated as the filename of a call-in file to be opened and read.
         </li>
+        <li><span class="parameter">debug</span>
+         When neither false nor nil, tell Lua not to delete the temporary preprocessed call-in table file it created.
+ The name of this file will be stored in the <code>__ci_filename</code> field of the returned table.
+        </li>
     </ul>
 
     <h3>Returns:</h3>
@@ -1714,7 +1718,7 @@ <h3>Returns:</h3>
 
     <h3>Example:</h3>
     <ul>
-        <pre class="example">$ export ydb_routines=examples   # put arithmetic.m (below) into ydb path
+        <pre class="example">$ export ydb_routines=examples   # put arithmetic.m into ydb path
 $ lua -lyottadb
 arithmetic = yottadb.<span class="global">require</span>(<span class="string">'examples/arithmetic.ci'</span>)
 arithmetic.add_verbose(<span class="string">"Sum is:"</span>, <span class="number">2</span>, <span class="number">3</span>)
@@ -2867,7 +2871,7 @@ <h3>See also:</h3>
 </div> <!-- id="main" -->
 <div id="about">
 <i>generated by <a href="http://github.com/stevedonovan/LDoc">LDoc 1.4.6</a></i>
-<i style="float:right;">Last updated 2024-07-22 17:59:27 </i>
+<i style="float:right;">Last updated 2024-08-02 20:00:50 </i>
 </div> <!-- id="about" -->
 </div> <!-- id="container" -->
 </body>

examples/arithmetic.ci

@@ -2,26 +2,29 @@
 
 // Format of lines in this file:
 //<c-call-name> : <ret-type> <label-ref> ([<direction>:<param-type>,...])
-// where <ret-type> may only be:
-//      void
-//      [unsigned]integer: (ydb_[u]long_t*)
-//      null-terminated string: ydb_char_t*  (see note below re use as an IO)
-//      string: ydb_string_t* [preallocation]
+// where <ret-type> and <param-type> may be:
+//      [unsigned]integer: ydb_[u]long_t OR [u]long
+//      [unsigned]int32: ydb_[u]int_t OR [u]int (32-bit),
+//      [unsigned]int64: ydb_[u]int64_t OR [u]int64 (64-bit platforms only)
+//      [unsigned]long: ydb_[u]long_t OR [u]long (32 or 64-bit depending on platform)
+//      floating point: ydb_float_t, ydb_double_t OR float, double
+//      null-terminated string: ydb_char_t* OR char*  (see note below re use as an IO)
+//      string: ydb_string_t* [preallocation] OR ydb_buffer_t* [preallocation]
 //        (preallocation is the amount of bytes Lua should allocate for output strings from M)
-// where <param_type> may be any of the above, plus:
-//      [unsigned]int32: ydb_[u]int_t (32-bit),
-//      [unsigned]int64: ydb_[u]int64_t (64-bit platforms only)
-//      [unsigned]long: ydb_[u]long_t (32 or 64-bit depending on platform)
-//      floating point: ydb_float_t, ydb_double_t
-//    each type may optionally be followed by '*' for pointer type (required for O and IO direction)
+// and <ret-type> may also be:
+//      void
+//    for O and IO direction, each type must be followed by '*' to specify pointer type
 // where <direction> may be: I, O, or IO (in/out to M; outputs may only be pointer types)
 // Notes:
 //   - between YDB r1.26 - r1.35, ydb_string_t* as an IO cannot output more characters than were input (but ydb_char_t* can).
 //   - ydb_char_t* as an output/retval is discouraged as preallocation is always max (1MB) to prevent overruns.
 //   - ydb_char_t* cannot return data containing '\0' characters since '\0' determines its end.
-//   - ydb_buffer_t* is also accepted but lua-yottadb automatically converts between it and ydb_string_t* for best efficiency/support.
-//   - double/int/float don't actually work with Lua and so are automatically converted to pointer types
+//   - ydb_buffer_t*/ydb_string_t*: lua-yottadb automatically converts between these for best efficiency/support.
+//   - float is converted to double since Lua doesn't use floats (this avoids unexpected noise in the insignificant bits).
+//   - double/int/float don't actually work with Lua and so are automatically converted to double*/int*/double*.
+//   - non-pointer return types are not allowed by YottaDB, so lua-yottadb automatically converts them to pointer types.
 
-add_verbose: ydb_string_t*[1024] addVerbose^arithmetic(I:ydb_string_t*, I:ydb_long_t, I:ydb_long_t)
-add: ydb_long_t* add^arithmetic(I:ydb_long_t, I:ydb_long_t)
-sub: ydb_long_t* sub^arithmetic(I:ydb_long_t, I:ydb_long_t)
+add_verbose: string*[1024] addVerbose^arithmetic(I:string*, I:long, I:long)
+add: long* add^arithmetic(I:long, I:long)
+sub: long* sub^arithmetic(I:long, I:long)
+addfloats: float* add^arithmetic(I:float, I:float)

yottadb.c

@@ -737,6 +737,8 @@ int luaopen__yottadb(lua_State *L) {
   lua_pushboolean(L, true), lua_setfield(L, -2, "YDB_DEL_TREE");  // these 2 values are changed from libyottadb.h so delete()
   lua_pushboolean(L, false), lua_setfield(L, -2, "YDB_DEL_NODE");  // can detect their type is not string/number. No API change.
   lua_pushstring(L, LUA_YOTTADB_VERSION_STRING), lua_setfield(L, -2, "_VERSION");
+
+  // Create Lua table YDB_CI_PARAM_TYPES to hold constants specified by yottadb_types
   lua_createtable(L, 0, (sizeof(yottadb_types)/sizeof(const_Reg))-1);
   for (const_Reg *c = &yottadb_types[0]; c->name; c++) {
     lua_pushinteger(L, c->value), lua_setfield(L, -2, c->name);

yottadb.h

@@ -7,6 +7,7 @@
 v3.0 Introduce inheritable nodes using yottadb.inherit()
  - Update examples/startup.lua to properly detect inherited nodes
  - Breaking change to lock() and lock_incr() which now wait forever with nil timeout, like the M LOCK command
+ - Support all call-in table types, now including those not starting with `ydb_`, like `int`
  - Docs:
    - Document `yottadb.inherit()`
    - Update documentation with new node methods: `kill`, `grab`, `release`, and `__repr`.

yottadb.lua

@@ -821,24 +821,26 @@ local function parse_prototype(line, ci_handle)
   return routine_name, func
 end
 
---- Import M routines as Lua functions specified in ydb 'call-in' file. <br>
+--- Import M routines as Lua functions specified in a ydb 'call-in' file. <br>
 -- See example call-in file [arithmetic.ci](https://github.com/anet-be/lua-yottadb/blob/master/examples/arithmetic.ci)
 -- and matching M file [arithmetic.m](https://github.com/anet-be/lua-yottadb/blob/master/examples/arithmetic.m).
 -- @param Mprototypes A list of lines in the format of ydb 'call-in' files required by `ydb_ci()`.
 -- If the string contains `:` it is considered to be the call-in specification itself;
 -- otherwise it is treated as the filename of a call-in file to be opened and read.
+-- @param debug When neither false nor nil, tell Lua not to delete the temporary preprocessed call-in table file it created.
+-- The name of this file will be stored in the `__ci_filename` field of the returned table.
 -- @return A table of functions analogous to a Lua module.
 -- Each function in the table will call an M routine specified in `Mprototypes`.
 -- @example
--- $ export ydb_routines=examples   # put arithmetic.m (below) into ydb path
+-- $ export ydb_routines=examples   # put arithmetic.m into ydb path
 -- $ lua -lyottadb
 -- arithmetic = yottadb.require('examples/arithmetic.ci')
 -- arithmetic.add_verbose("Sum is:", 2, 3)
 -- -- Sum is: 5
 -- -- Sum is: 5
 -- arithmetic.sub(5,7)
 -- -- -2
-function M.require(Mprototypes)
+function M.require(Mprototypes,debug)
   local routines = {}
   if not Mprototypes:find(':', 1, true) then
     -- read call-in file
@@ -847,38 +849,60 @@ function M.require(Mprototypes)
     Mprototypes = f:read('a')
     f:close()
   end
-  -- preprocess call-in types that can't be used with wrapper caller ydb_call_variadic_plist_func()
-  Mprototypes = Mprototypes:gsub('ydb_double_t%s*%*?', 'ydb_double_t*')
-  Mprototypes = Mprototypes:gsub('ydb_float_t%s*%*?', 'ydb_float_t*')
-  Mprototypes = Mprototypes:gsub('ydb_int_t%s*%*?', 'ydb_int_t*')
-  if ydb_release < 1.36 then  Mprototypes = Mprototypes:gsub('ydb_buffer_t%s*%*', 'ydb_string_t*')  end
-  if ydb_release >= 1.36 then
-    -- Convert between buffer/string types for efficiency (lets user just use ydb_string_t* all the time without thinking about it
-    Mprototypes = Mprototypes:gsub('IO:ydb_string_t%s*%*', 'IO:ydb_buffer_t*')
-    Mprototypes = Mprototypes:gsub('([^IO][IO]):ydb_buffer_t%s*%*', '%1:ydb_string_t*')
-    -- Also replace buffer with string in the retval (first typespec in the line)
-    -- (note: %f[^\n\0] below, captures beginning of line or string)
-    Mprototypes = Mprototypes:gsub('(\n%s*[A-Za-z0-9_]+%s*:%s*)ydb_buffer_t%s*%*', '%1ydb_string_t*')
-    Mprototypes = Mprototypes:gsub('^(%s*[A-Za-z0-9_]+%s*:%s*)ydb_buffer_t%s*%*', '%1ydb_string_t*')
+
+  -- now preprocess the call-in table to make it more suited to lua-yottadb
+  -- first, declare a function that will be passed every string of types, for potential substitution
+  local function type_replacer(before,types,after)
+    local isretval = not types:find(':')  -- if we're replacing the retval, there is no ':' in types
+    -- convert floats to doubles since Lua doesn't use floats. Avoids unexpected junk in the insignificant figures
+    types = types:gsub('float', 'double')
+    -- any non-pointer types that can't be used with ydb_call_variadic_plist_func(), convert to pointer types
+    for _,typ in pairs{'ydb_double_t','ydb_int_t','double','int'} do
+      types = types:gsub('(' .. typ .. '%s*)[*]*([,) ])', '%1*%2')
+    end
+    -- convert any non-pointer retvals (which are illegal) to pointer types
+    if isretval and not types:find('void') then  types=types:gsub('([A-Za-z0-9_]+)[%s*]+', '%1* ')  end
+    if ydb_release < 1.36 then
+      -- make ydb <1.36 emulate 'buffer' type, even though it's not as good as string for output-only (O:) parameters
+      types = types:gsub('ydb_buffer_t%s*%*', 'ydb_string_t*')
+    end
+    if ydb_release >= 1.36 then
+      -- convert between buffer/string types for best efficiency (lets user use either one without thinking about it)
+      types = types:gsub('IO%s*:%s*ydb_string_t%s*%*', 'IO:ydb_buffer_t*')  -- IO: params
+      types = types:gsub('IO%s*:%s*string%s*%*', 'IO:ydb_buffer_t*')
+      types = types:gsub('([^IO][IO])%s*:%s*ydb_buffer_t%s*%*', '%1:ydb_string_t*') -- I: or O: params
+      -- always replace 'buffer' with 'string' in the retval as it is equivalent of an O: param
+      if isretval then  types=types:gsub('ydb_buffer_t%s*%*', 'ydb_string_t*')  end
+    end
+    return before..types..after
   end
-  -- remove preallocation specs for ydb which (as of r1.34) can only process them in call-out tables
-  local ydb_prototypes = Mprototypes:gsub('%b[]', '')
+  -- replace things in parameters, i.e. between parentheses, and then replace things in retvals, i.e. immediately after first ':'
+  -- note: `(\1?)` is my substitute for an empty capture since `()` is a special Lua code that returns the position in the string
+  Mprototypes = Mprototypes:gsub('(\1?)(%([^\n]*%))(\1?)', type_replacer )
+  Mprototypes = Mprototypes:gsub('^(%s*[A-Za-z0-9_]+%s*:%s*)([^%s*]+[%s*]+)(\1?)', type_replacer) -- first line's retval
+  Mprototypes = Mprototypes:gsub('(\n%s*[A-Za-z0-9_]+%s*:%s*)([^%s*]+[%s*]+)(\1?)', type_replacer) -- other lines' retval
+  -- remove preallocation specs for ydb which can only process them in call-out tables not call-in tables
+  local ydb_prototypes = Mprototypes  -- take a copy for our own parser which requires preallocation specs
+  Mprototypes = Mprototypes:gsub('%b[]', '')
+
   -- write call-in file and load into ydb
   local filename = os.tmpname()
   local f = assert(io.open(filename, 'w'), string.format("cannot open temporary call-in file %q", filename))
-  assert(f:write(ydb_prototypes), string.format("cannot write to temporary call-in file %q", filename))
+  assert(f:write(Mprototypes), string.format("cannot write to temporary call-in file %q", filename))
   f:close()
   local ci_handle = _yottadb.ci_tab_open(filename)
-  routines.__ci_filename = filename
+  if debug then  routines.__ci_filename = filename  end
   routines.__ci_table_handle = ci_handle
+
   -- process ci-table ourselves to get routine names and types to cast
+  -- do this after ydb has handled it to let ydb catch any errors in the table
   local line_no = 0
-  for line in Mprototypes:gmatch('([^\n]*)\n?') do
+  for line in ydb_prototypes:gmatch('([^\n]*)\n?') do
     line_no = line_no+1
     local ok, routine, func = pcall(parse_prototype, line, ci_handle)
     if routine then  routines[routine] = func end
   end
-  os.remove(filename) -- cleanup
+  if not debug then  os.remove(filename)  end  -- cleanup
   return routines
 end