tests: improve test runner and testsuite isolation (#2016)

- initial LUA paths are now setup by the runner itself, so it can
  be executed directly without needing to manually set them before
- patch the LUA paths before actually executing the tests to remove
  the testsuite specific entries (e.g. `spec/rocks/…`), ensuring
  tests only have access to shipped packages

Makefile

@@ -134,20 +134,18 @@ download-all: test-data
 $(OUTPUT_DIR)/spec/base: | $(OUTPUT_DIR)/spec/
 	$(SYMLINK) $(KOR_BASE)/spec $@
 
-$(OUTPUT_DIR)/spec/config.lua: | $(OUTPUT_DIR)/spec/
-	$(SYMLINK) $(KOR_BASE)/test-runner/busted_config.lua $@
+$(addprefix $(OUTPUT_DIR)/spec/,config.lua helper.lua): | $(OUTPUT_DIR)/spec/
+	$(SYMLINK) $(KOR_BASE)/test-runner/busted_$(notdir $@) $@
 
-$(OUTPUT_DIR)/spec/meson.build: | $(OUTPUT_DIR)/spec/
-	$(SYMLINK) $(KOR_BASE)/test-runner/meson.build $@
-
-$(OUTPUT_DIR)/spec/runtests: | $(OUTPUT_DIR)/spec/
-	$(SYMLINK) $(KOR_BASE)/test-runner/runtests $@
+$(addprefix $(OUTPUT_DIR)/spec/,meson.build runtests): | $(OUTPUT_DIR)/spec/
+	$(SYMLINK) $(KOR_BASE)/test-runner/$(notdir $@) $@
 
 $(BASE_PREFIX)test: $(BASE_PREFIX)all test-data
 	$(RUNTESTS) $(OUTPUT_DIR) base $T
 
 define test_data_common
 $(OUTPUT_DIR)/spec/config.lua
+$(OUTPUT_DIR)/spec/helper.lua
 $(OUTPUT_DIR)/spec/meson.build
 $(OUTPUT_DIR)/spec/runtests
 endef

Makefile.defs

@@ -1082,9 +1082,6 @@ define runtests_fn
 runtests() {(
 kodir="$$1";
 shift 1;
-export LUA_CPATH='./?.so;$(abspath $(SPEC_ROCKS_DIR)/lib/lua/5.1/?.so)';
-export LUA_PATH='./?.lua;$(abspath $(SPEC_ROCKS_DIR)/share/lua/5.1/?.lua);$(abspath $(SPEC_ROCKS_DIR)/share/lua/5.1/?/init.lua)';
-export TESSDATA_DIR="$$PWD/data";
 "$$kodir/spec/runtests" $(PARALLEL_JOBS:%=-j%) "$$@";
 )}
 endef

spec/unit/common_spec.lua

@@ -1,5 +1,3 @@
-package.path = "common/?.lua;" .. package.path
-package.cpath = "common/?.so;" .. package.cpath
 require("ffi_wrapper")
 
 local url = require("socket.url")

spec/unit/spore_spec.lua

@@ -1,6 +1,3 @@
-package.path = "common/?.lua;" .. package.path
-package.cpath = "common/?.so;" .. package.cpath
-
 local service = [[
 {
     "base_url" : "http://services.org:9999/restapi/",

test-runner/busted_config.lua

@@ -4,7 +4,8 @@ local testsuites = {}
 local roots = {}
 local lpaths = {}
 for entry in lfs.dir("spec") do
-    if not string.match(entry, "^[.]") then
+    local attr = lfs.attributes("spec/" .. entry .. "/unit")
+    if attr and attr.mode == "directory" then
         local testroot = "spec/" .. entry .. "/unit"
         local testpath = testroot .. "/?.lua"
         testsuites[entry] = {}

test-runner/busted_helper.lua

@@ -0,0 +1,21 @@
+-- Preload necessary busted modules and their dependencies (which
+-- are normally dynamically loaded during the testsuite execution).
+require("pl.dir")
+require("busted.execute")
+require("busted.modules.files.lua")
+require("busted.modules.test_file_loader")
+-- Patch `package.path / package.cpath`: filter-out paths
+-- specific to the test framework (e.g. `spec/rocks/…`).
+local function filter_rocks(path)
+    local filtered = {}
+    for spec in string.gmatch(path, "([^;]+)") do
+        if not spec:match("spec/rocks/") then
+            table.insert(filtered, spec)
+        end
+    end
+    return table.concat(filtered, ';')
+end
+package.path = filter_rocks(package.path)
+package.cpath = filter_rocks(package.cpath)
+-- Setup `ffi.loadlib` support.
+require("ffi/loadlib")

test-runner/runtests

@@ -118,7 +118,8 @@ kodir="$(cd "$(dirname "$0")/.." && pwd -P)"
 busted_args=(
     --config-file=spec/config.lua
     --exclude-tags=notest
-    --helper=ffi/loadlib.lua
+    --helper=spec/helper.lua
+    --loaders=lua
     --lazy
 )
 debug=0
@@ -217,6 +218,8 @@ case "$1" in
         ;;
 esac
 
+export LUA_CPATH='?.so;common/?.so;spec/rocks/lib/lua/5.1/?.so'
+export LUA_PATH='?.lua;common/?.lua;frontend/?.lua;spec/rocks/share/lua/5.1/?.lua;spec/rocks/share/lua/5.1/?/init.lua'
 export TESSDATA_PREFIX="${kodir}/data"
 
 if [[ "${use_meson}" -ne 0 ]]; then