Tests: automatically run all test files (#2810)

.codecov.yml

@@ -1,7 +1,7 @@
 codecov:
   notify:
     # Keep this in sync with the number of CI jobs uploading coverage.
-    after_n_builds: 3
+    after_n_builds: 4
 coverage:
   status:
     project:

.github/workflows/CI.yml

@@ -27,6 +27,7 @@ jobs:
           - '1.9'
           - '~1.10.0-0'
           - 'nightly'
+        group: [ 'short', 'long' ]
         os:
           - ubuntu-latest
         include:
@@ -50,6 +51,9 @@ jobs:
         if: runner.os == 'macOS'
         # restrict number of openMP threads on macOS due to oversubscription
         run: echo "OMP_NUM_THREADS=1" >> $GITHUB_ENV
+      - name: "set test subgroup"
+        if: ${{ matrix.group }} != ''
+        run: echo "OSCAR_TEST_SUBSET=${{matrix.group}}" >> $GITHUB_ENV
       - name: "Run tests"
         uses: julia-actions/julia-runtest@latest
         with:

docs/src/DeveloperDocumentation/new_developers.md

@@ -106,6 +106,11 @@ Oscar.test_module
 Oscar.get_seeded_rng
 ```
 
+If a test folder contains a file called `setup_tests.jl` it is included
+automatically before each file (directly) in this directory. This can be used
+to define helper functions that are used in multiple test files, for example
+`test_save_load_roundtrip` for serialization.
+
 ### Adding documentation
 For more information on docstrings, please read our page on [Documenting OSCAR
 code](@ref).  There are two places where documentation can be added:

docs/src/Experimental/intro.md

@@ -34,14 +34,18 @@ experimental/PACKAGE_NAME/
 ├── src
 │   └── PACKAGE_NAME.jl
 └── test
-    └── runtests.jl
+    └── *.jl
 ```
-The files `src/PACKAGE_NAME.jl` and `test/runtests.jl` are mandatory as they
-are used by Oscar.jl to find your code and tests. The file `docs/doc.main` is
-used for integrating your documentation in the Oscar manual under the
-`Experimental` section. Optionally please provide a `README.md` describing your
-project and its goals, especially if you are starting from scratch and don't
-have any documentation yet.
+The file `src/PACKAGE_NAME.jl` and at least one `.jl` file in the `test/`
+directory are mandatory and are used by Oscar.jl to find your code and tests.
+If there is a `test/runtests.jl` then only this file is executed during
+testing, otherwise all `.jl` files will be run automatically (in a random
+order).
+
+The file `docs/doc.main` is used for integrating your documentation in the
+Oscar manual under the `Experimental` section. Optionally please provide a
+`README.md` describing your project and its goals, especially if you are
+starting from scratch and don't have any documentation yet.
 
 !!! note
     There are still older projects in `experimental` from before the

experimental/Experimental.jl

@@ -28,8 +28,9 @@ for pkg in exppkgs
   if !isfile(joinpath(expdir, pkg, "src", "$pkg.jl"))
     error("experimental/$pkg is incomplete: $pkg/src/$pkg.jl missing. See the documentation at https://docs.oscar-system.org/dev/Experimental/intro/ for details.")
   end
-  if !isfile(joinpath(expdir, pkg, "test", "runtests.jl"))
-    error("experimental/$pkg is incomplete: $pkg/test/runtests.jl missing. See the documentation at https://docs.oscar-system.org/dev/Experimental/intro/ for details.")
+  path = joinpath(expdir, pkg, "test")
+  if !isdir(path) || length(filter(endswith(".jl"), readdir(path))) == 0
+    error("experimental/$pkg is incomplete: $pkg/test/ missing or empty. See the documentation at https://docs.oscar-system.org/dev/Experimental/intro/ for details.")
   end
 end
 

experimental/FTheoryTools/test/setup_tests.jl

@@ -0,0 +1,2 @@
+set_verbosity_level(:FTheoryConstructorInformation, -1)
+include(joinpath(Oscar.oscardir, "test", "Serialization", "setup_tests.jl"))

experimental/FTheoryTools/test/tate.jl

@@ -30,10 +30,6 @@ t = global_tate_model(base; completeness_check = false)
   @test is_smooth(ambient_space(t)) == false
   @test toric_variety(calabi_yau_hypersurface(t)) == ambient_space(t)
 
-  isdefined(Main, :test_save_load_roundtrip) || include(
-    joinpath(Oscar.oscardir, "test", "Serialization", "test_save_load_roundtrip.jl")
-  )
-
   mktempdir() do path
     test_save_load_roundtrip(path, t) do loaded
       @test tate_polynomial(t) == tate_polynomial(loaded)

experimental/QuadFormAndIsom/test/runtests.jl

@@ -98,10 +98,6 @@ end
   @test is_elementary_with_prime(integer_lattice_with_isometry(root_lattice(:E, 7)))[1]
   @test is_unimodular(integer_lattice_with_isometry(hyperbolic_plane_lattice()))
 
-  isdefined(Main, :test_save_load_roundtrip) || include(
-    joinpath(Oscar.oscardir, "test", "Serialization", "test_save_load_roundtrip.jl")
-  )
-
   mktempdir() do path
     test_save_load_roundtrip(path, Lf) do loaded
       @test Lf == loaded

experimental/QuadFormAndIsom/test/setup_tests.jl

@@ -0,0 +1 @@
+../../../test/Serialization/setup_tests.jl

src/utils/tests.jl

@@ -1,46 +1,198 @@
+function _timed_include(str::String, mod::Module=Main; use_ctime::Bool=VERSION >= v"1.9.0")
+  if use_ctime
+    compile_elapsedtimes = Base.cumulative_compile_time_ns()
+  end
+  stats = @timed Base.include(identity, mod, str)
+  fullpath = abspath(joinpath(Base.source_dir(), str))
+  # skip files which just include other files and ignore
+  # files outside of the oscar folder
+  if startswith(fullpath, Oscar.oscardir)
+    path = relpath(fullpath, Oscar.oscardir)
+    if use_ctime
+      compile_elapsedtimes = Base.cumulative_compile_time_ns() .- compile_elapsedtimes
+      compile_elapsedtimes = compile_elapsedtimes ./ 10^9
+    end
+    rtime=NaN
+    if use_ctime
+      comptime = first(compile_elapsedtimes)
+      rcomptime = last(compile_elapsedtimes)
+      println("-> Testing $path took: runtime $(round(stats.time-comptime; digits=3)) seconds + compilation $(round(comptime-rcomptime; digits=3)) seconds + recompilation $(round(rcomptime; digits=3)) seconds, $(Base.format_bytes(stats.bytes))")
+      return (path=>(time=stats.time-comptime, ctime=comptime-rcomptime, rctime=rcomptime, alloc=stats.bytes/2^20))
+    else
+      println("-> Testing $path took: $(round(stats.time; digits=3)) seconds, $(Base.format_bytes(stats.bytes))")
+      return (path=>(time=stats.time, alloc=stats.bytes/2^20))
+    end
+  else
+    return ()
+  end
+end
+
+function _gather_tests(path::AbstractString; ignore=[])
+  # default ignore patterns
+  ignorepatterns = Regex[
+                     # these two files seem obsolete
+                     r"Modules/GradedModules(\.jl)?$",
+                     r"Modules/FreeModules-graded(\.jl)?$",
+                     # FIXME: temporarily disable AlgClosureFp tests until we resolve
+                     # issue https://github.com/oscar-system/Oscar.jl/issues/2691
+                     r"Rings/AlgClosureFp(\.jl)?$",
+                     # this can only run on the main process and not on distributed workers
+                     # so it is included directly in runtests
+                     r"Serialization/IPC(\.jl)?$",
+                   ]
+  for i in ignore
+    if i isa Regex
+      push!(ignorepatterns, i)
+    elseif i isa AbstractString
+      if endswith(i, ".jl")
+        push!(ignorepatterns, Regex("$i\$"))
+      else
+        push!(ignorepatterns, Regex("$i(\\.jl)?\$"))
+      end
+    else
+      throw(ArgumentError("invalid ignore pattern $i"))
+    end
+  end
+
+  if any(p->contains(path, p), ignorepatterns)
+    @info "ignore: $(relpath(path, Oscar.oscardir))"
+    return String[]
+  end
+
+  if !isabspath(path)
+    path = joinpath(Oscar.oscardir, path)
+  end
+
+  isfile(path) && return [path]
+  isfile("$path.jl") && return ["$path.jl"]
+
+  # if there is a runtests.jl we ignore everything else in that folder
+  # except for the main Oscar test dir
+  isfile(joinpath(path, "runtests.jl")) &&
+    path != joinpath(Oscar.oscardir, "test") &&
+    return [joinpath(path, "runtests.jl")]
+
+  tests = String[]
+  for entry in readdir(path; join=true)
+    if any(s->contains(relpath(entry, Oscar.oscardir), s), ignorepatterns)
+      @info "ignore: $(relpath(entry, Oscar.oscardir))"
+      continue
+    end
+    endswith(entry, "setup_tests.jl") && continue
+    # this is only for the main test/runtests.jl
+    endswith(entry, "runtests.jl") && continue
+    if isdir(entry)
+      append!(tests, _gather_tests(entry; ignore=ignore))
+    elseif isfile(entry) && endswith(entry, ".jl")
+      push!(tests, entry)
+    end
+  end
+  return tests
+end
+
+
+
+
 @doc raw"""
-    test_module(file::AbstractString; new::Bool = true)
+    test_module(path::AbstractString; new::Bool = true, timed::Bool=false, ignore=[])
 
-Run the Oscar tests in the file `test/<file>.jl` where `file` may be a path.
+Run the Oscar tests in `path`:
+- if `path` is relative then it will be set to `<oscardir>/test/<path>`
+- if `path` is a directory, run all test files in that directory and below
+- if `path` or `path.jl` is a file, run this file
+
+If a directory contains a `runtests.jl` file only this file will be executed, otherwise
+all files will be included independently.
 
 The optional parameter `new` takes the values `false` and `true` (default). If
 `true`, then the tests are run in a new session, otherwise the currently active
 session is used.
 
+With the optional parameter `timed` the function will return a dict mapping file
+names to a named tuple with compilation times and allocations.
+This only works for `new=false`.
+
+The parameter `ignore` can be used to pass a list of `String` or `Regex` patterns.
+Test files or folders matching these will be skipped. Strings will be compared as
+suffixes.
+This only works for `new=false`.
+
 For experimental modules, use [`test_experimental_module`](@ref) instead.
 """
-function test_module(file::AbstractString; new::Bool=true)
+function test_module(path::AbstractString; new::Bool=true, timed::Bool=false, ignore=[])
   julia_exe = Base.julia_cmd()
   project_path = Base.active_project()
-  rel_test_file = normpath("test", "$file.jl")
-  test_file = joinpath(oscardir, rel_test_file)
-
+  if !isabspath(path)
+    if !startswith(path, "test")
+      path = joinpath("test", path)
+    end
+    rel_test_path = normpath(path)
+    path = joinpath(oscardir, rel_test_path)
+  end
   if new
-    cmd = "using Test; using Oscar; Hecke.assertions(true); include(\"$test_file\");"
+    @req isempty(ignore) && !timed "The `timed` and `ignore` options only work for `new=false`."
+    cmd = "using Test; using Oscar; Hecke.assertions(true); Oscar.test_module(\"$path\"; new=false);"
     @info("spawning ", `$julia_exe --project=$project_path -e \"$cmd\"`)
     run(`$julia_exe --project=$project_path -e $cmd`)
   else
+    testlist = _gather_tests(path; ignore=ignore)
+    @req !isempty(testlist) "no such file or directory: $path[.jl]"
+
     @req isdefined(Base.Main, :Test) "You need to do \"using Test\""
-    @info("Running tests for $rel_test_file in same session")
-    Base.include(Base.Main, test_file)
+
+    use_ctime = timed && VERSION >= v"1.9.0-DEV"
+    if use_ctime
+      Base.cumulative_compile_timing(true)
+    end
+    stats = Dict{String,NamedTuple}()
+    for entry in testlist
+      dir = dirname(entry)
+      if isfile(joinpath(dir,"setup_tests.jl"))
+        Base.include(identity, Main, joinpath(dir,"setup_tests.jl"))
+      end
+      if timed
+        push!(stats, _timed_include(entry; use_ctime=use_ctime))
+      else
+        Base.include(identity, Main, entry)
+      end
+    end
+
+    if timed
+      use_ctime && Base.cumulative_compile_timing(false)
+      return stats
+    else
+      return nothing
+    end
   end
 end
 
 @doc raw"""
-    test_experimental_module(project::AbstractString; file::AbstractString="runtests", new::Bool = true)
+    test_experimental_module(project::AbstractString; file::AbstractString="",
+      new::Bool=true, timed::Bool=false, ignore=[])
+
+Run the Oscar tests in `experimental/<project>/test/<path>`:
+- if `path` is empty then all tests in that module are run, either via `runtests.jl` or directly.
+- if `path` or `path.jl` is a file in that directory only this file is run.
 
-Run the Oscar tests in the file `experimental/<project>/test/<file>.jl`
-where `file` may be a path.
 The default is to run the entire test suite of the module `project`.
 
 The optional parameter `new` takes the values `false` and `true` (default). If
 `true`, then the tests are run in a new session, otherwise the currently active
 session is used.
+
+With the optional parameter `timed` the function will return a dict mapping file
+names to a named tuple with compilation times and allocations.
+This only works for `new=false`.
+
+The parameter `ignore` can be used to pass a list of `String` or `Regex` patterns.
+Test files or folders matching these will be skipped. Strings will be compared as
+suffixes.
+This only works for `new=false`.
 """
 function test_experimental_module(
-  project::AbstractString; file::AbstractString="runtests", new::Bool=true
+  project::AbstractString; file::AbstractString="", new::Bool=true, timed::Bool=false, ignore=[]
 )
   test_file = "../experimental/$project/test/$file"
-  test_module(test_file; new)
+  test_module(test_file; new, timed=timed, ignore=ignore)
 end
 

system/precompile.jl

@@ -1,7 +1,6 @@
 import Pkg
 Pkg.add("Documenter")
 Pkg.add("PrettyTables")
-Pkg.add("Printf")
 Pkg.add("Aqua")
 
 Pkg.precompile()