Skip to content

Commit 2826591

Browse files
SeelengrabSeelengrab
committed
Add tests for reporting results
This accomplishes two things: * Firstly, actually test that what is reported is accurate * Secondly, make sure that the expanded macro doesn't collide with variables from the environment The former is important for UX - if we support user reporting, we should also make sure that what we report is something the user can act on, if they need to. The latter just ensures that there's no bad state floating around.
1 parent 34955a6 commit 2826591

File tree

3 files changed

+113
-41
lines changed

3 files changed

+113
-41
lines changed

‎src/api.jl

Lines changed: 33 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -247,53 +247,54 @@ function check_call(e::Expr, tsargs)
247247
end
248248

249249
function final_check_block(namestr, run_input, gen_input, tsargs)
250-
ts = gensym()
251-
sr = gensym()
250+
@gensym(ts, sr, report, previous_failure, got_res, got_err, got_score,
251+
res, choices, n_tc, obj, exc, trace, len, err, fail,
252+
pass, score)
252253

253254
return quote
254255
# need this for backwards compatibility
255256
$sr = $SuppositionReport
256257
$Test.@testset $sr $(tsargs...) $namestr begin
257-
report = $Test.get_testset()
258-
previous_failure = $retrieve(report.database, $record_name(report))
259-
$ts = $TestState(report.config, $run_input, previous_failure)
258+
$report = $Test.get_testset()
259+
$previous_failure = $retrieve($report.database, $record_name($report))
260+
$ts = $TestState($report.config, $run_input, $previous_failure)
260261
$Supposition.run($ts)
261-
$Test.record(report, $ts)
262-
got_res = !isnothing($ts.result)
263-
got_err = !isnothing($ts.target_err)
264-
got_score = !isnothing($ts.best_scoring)
265-
@debug "Any result?" Res=got_res Err=got_err Score=got_score
266-
if got_res | got_err | got_score
267-
res = @something $ts.target_err $ts.best_scoring $ts.result
268-
choices = if got_err | got_score
269-
last(res)
262+
$Test.record($report, $ts)
263+
$got_res = !isnothing($ts.result)
264+
$got_err = !isnothing($ts.target_err)
265+
$got_score = !isnothing($ts.best_scoring)
266+
$Logging.@debug "Any result?" Res=$got_res Err=$got_err Score=$got_score
267+
if $got_res | $got_err | $got_score
268+
$res = $Base.@something $ts.target_err $ts.best_scoring $ts.result
269+
$choices = if $got_err | $got_score
270+
$last($res)
270271
else
271-
res
272+
$res
272273
end
273-
n_tc = $Supposition.for_choices(choices, copy($ts.rng))
274-
obj = $ScopedValues.@with $Supposition.CURRENT_TESTCASE => n_tc begin
275-
$gen_input(n_tc)
274+
$n_tc = $Supposition.for_choices($choices, $copy($ts.rng))
275+
$obj = $ScopedValues.@with $Supposition.CURRENT_TESTCASE => $n_tc begin
276+
$gen_input($n_tc)
276277
end
277-
@debug "Recording result in testset"
278-
if got_err
278+
$Logging.@debug "Recording result in testset"
279+
if $got_err
279280
# This is an unexpected error, report as `Error`
280-
exc, trace, len = res
281-
err = $Error(obj, exc, trace[begin:len-2])
282-
$Test.record(report, err)
283-
elseif got_res # res
281+
$exc, $trace, $len = $res
282+
$err = $Error($obj, $exc, $trace[begin:$len-2])
283+
$Test.record($report, $err)
284+
elseif $got_res # res
284285
# This is an unexpected failure, report as `Fail`
285-
fail = $Fail(obj, nothing)
286-
$Test.record(report, fail)
287-
elseif got_score
286+
$fail = $Fail($obj, $nothing)
287+
$Test.record($report, $fail)
288+
elseif $got_score
288289
# This means we didn't actually get a result, so report as `Pass`
289290
# Also mark this, so we can display this correctly during `finish`
290-
score = first(res)
291-
pass = $Pass(Some(obj), Some(score))
292-
$Test.record(report, pass)
291+
$score = $first($res)
292+
$pass = $Pass($Some($obj), $Some($score))
293+
$Test.record($report, $pass)
293294
end
294295
else
295-
pass = $Supposition.Pass(nothing, nothing)
296-
$Test.record(report, pass)
296+
$pass = $Supposition.Pass($nothing, $nothing)
297+
$Test.record($report, $pass)
297298
end
298299
end
299300
end

‎src/testset.jl

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,13 @@ struct Error <: Result
3030
end
3131

3232
function results(sr::SuppositionReport)
33-
ispass = @something(sr.result) isa Pass
34-
isfail = @something(sr.result) isa Fail
35-
iserror = @something(sr.result) isa Error
36-
isbroken = sr.expect_broken && (isfail | iserror)
33+
res_pass = @something(sr.result) isa Pass
34+
res_fail = @something(sr.result) isa Fail
35+
res_error = @something(sr.result) isa Error
36+
ispass = res_pass && !sr.expect_broken
37+
iserror = res_error && !sr.expect_broken
38+
isfail = (res_pass && sr.expect_broken) || (res_fail && !sr.expect_broken)
39+
isbroken = !(ispass | iserror | isfail)
3740
(;ispass,isfail,iserror,isbroken)
3841
end
3942

@@ -59,13 +62,14 @@ end
5962
Test.format_duration(sr::SuppositionReport) = _format_duration(sr)
6063

6164
function Test.get_test_counts(sr::SuppositionReport)
62-
(;ispass,isfail,iserror,isbroken) = results(sr)
63-
@assert count((ispass, isfail, iserror, isbroken)) in (1,2)
65+
res = results(sr)
66+
(;ispass,isfail,iserror,isbroken) = res
67+
@assert isone(count(values(res))) values(res)
6468
return Test.TestCounts(
6569
true,
66-
!isbroken & ispass,
67-
!isbroken & isfail,
68-
!isbroken & iserror,
70+
ispass,
71+
isfail,
72+
iserror,
6973
isbroken,
7074
0,
7175
0,

‎test/runtests.jl

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -644,4 +644,71 @@ const verb = VERSION.major == 1 && VERSION.minor < 11
644644
@test data_rng_5[] == data_rng_6[]
645645
@test default_rng_5[] == default_rng_6[]
646646
end
647+
648+
@testset "Reporting behavior" begin
649+
pass(_) = true
650+
fail(_) = false
651+
err(_) = error()
652+
db = Supposition.NoRecordDB()
653+
654+
pass_sr = @check db=db record=false pass(Data.Just("Dummy"))
655+
# silence the expected printing
656+
fail_sr, err_sr, broke_pass_sr = redirect_stderr(devnull) do
657+
fail_sr = @check db=db record=false fail(Data.Just("Dummy"))
658+
err_sr = @check db=db record=false err(Data.Just("Dummy"))
659+
broke_pass_sr = @check db=db record=false broken=true pass(Data.Just("Dummy"))
660+
fail_sr, err_sr, broke_pass_sr
661+
end
662+
broke_fail_sr = @check db=db record=false broken=true fail(Data.Just("Dummy"))
663+
broke_err_sr = @check db=db record=false broken=true err(Data.Just("Dummy"))
664+
665+
@test @something(pass_sr.result) isa Supposition.Pass
666+
@test @something(broke_pass_sr.result) isa Supposition.Pass
667+
@test @something(fail_sr.result) isa Supposition.Fail
668+
@test @something(broke_fail_sr.result) isa Supposition.Fail
669+
@test @something(err_sr.result) isa Supposition.Error
670+
@test @something(broke_err_sr.result) isa Supposition.Error
671+
672+
@testset "Pass" begin
673+
@test Supposition.results(pass_sr).ispass
674+
@test !Supposition.results(pass_sr).isfail
675+
@test !Supposition.results(pass_sr).iserror
676+
@test !Supposition.results(pass_sr).isbroken
677+
end
678+
679+
@testset "Fail" begin
680+
@test !Supposition.results(fail_sr).ispass
681+
@test Supposition.results(fail_sr).isfail
682+
@test !Supposition.results(fail_sr).iserror
683+
@test !Supposition.results(fail_sr).isbroken
684+
end
685+
686+
@testset "Error" begin
687+
@test !Supposition.results(err_sr).ispass
688+
@test !Supposition.results(err_sr).isfail
689+
@test Supposition.results(err_sr).iserror
690+
@test !Supposition.results(err_sr).isbroken
691+
end
692+
693+
@testset "Broken Pass" begin
694+
@test !Supposition.results(broke_pass_sr).ispass
695+
@test Supposition.results(broke_pass_sr).isfail
696+
@test !Supposition.results(broke_pass_sr).iserror
697+
@test !Supposition.results(broke_pass_sr).isbroken
698+
end
699+
700+
@testset "Broken Fail" begin
701+
@test !Supposition.results(broke_fail_sr).ispass
702+
@test !Supposition.results(broke_fail_sr).isfail
703+
@test !Supposition.results(broke_fail_sr).iserror
704+
@test Supposition.results(broke_fail_sr).isbroken
705+
end
706+
707+
@testset "Broken Error" begin
708+
@test !Supposition.results(broke_err_sr).ispass
709+
@test !Supposition.results(broke_err_sr).isfail
710+
@test !Supposition.results(broke_err_sr).iserror
711+
@test Supposition.results(broke_err_sr).isbroken
712+
end
713+
end
647714
end

0 commit comments

Comments
 (0)