Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Ignore excluded/skipped tests and unnecessary setup/teardown using SkipRemainingOnFailure #2442

Merged
merged 8 commits into from
May 17, 2024
Merged

Ignore excluded/skipped tests and unnecessary setup/teardown using SkipRemainingOnFailure #2442

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4b8c99d
Refactor GetSkipRemainingOnFailure and ignore excluded and skipped tests
fflaten May 4, 2024
2e45b89
Skip blocks to avoid running BeforeAll/AfterAll
fflaten May 4, 2024
69f2256
Fix tests
fflaten May 4, 2024
0e8d82f
Skip root-level BeforeAll/AfterAll
fflaten May 5, 2024
974c27a
Fix Skipped container result
fflaten May 5, 2024
2e29633
Add tests for block-level skip
fflaten May 5, 2024
26da4f9
Cleanup
fflaten May 5, 2024
9126740
Cleanup test
fflaten May 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Pester.RSpec.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ function PostProcess-RspecTestRun ($TestRun) {
## decorate

# here we add result
$b.result = if ($b.Skipped) {
$b.result = if ($b.Skip) {
"Skipped"
}
elseif ($b.Passed) {
Expand Down
2 changes: 1 addition & 1 deletion src/Pester.Utility.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ function Fold-Block {
foreach ($b in $Block) {
$Accumulator = & $OnBlock $Block $Accumulator
foreach ($test in $Block.Tests) {
$Accumulator = &$OnTest $test $Accumulator
$Accumulator = & $OnTest $test $Accumulator
}

foreach ($b in $Block.Blocks) {
Expand Down
128 changes: 55 additions & 73 deletions src/functions/Get-SkipRemainingOnFailurePlugin.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,21 +1,46 @@
function New-SkippedTestMessage {
[OutputType([string])]
[CmdletBinding()]
param (
[Parameter(Mandatory)]
[Pester.Test]
$Test
)
"Skipped due to previous failure at '$($Test.ExpandedPath)' and Run.SkipRemainingOnFailure set to '$($PesterPreference.Run.SkipRemainingOnFailure.Value)'"
}

function Resolve-SkipRemainingOnFailureConfiguration {
$supportedValues = 'None', 'Block', 'Container', 'Run'
if ($PesterPreference.Run.SkipRemainingOnFailure.Value -notin $supportedValues) {
throw (Get-StringOptionErrorMessage -OptionPath 'Run.SkipRemainingOnFailure' -SupportedValues $supportedValues -Value $PesterPreference.Run.SkipRemainingOnFailure.Value)
}
}

function Set-RemainingAsSkipped {
param(
[Parameter(Mandatory)]
[Pester.Test]
$FailedTest,

[Parameter(Mandatory)]
[Pester.Block]
$Block
)

$errorRecord = [Pester.Factory]::CreateErrorRecord(
'PesterTestSkipped',
"Skipped due to previous failure at '$($FailedTest.ExpandedPath)' and Run.SkipRemainingOnFailure set to '$($PesterPreference.Run.SkipRemainingOnFailure.Value)'",
$null,
$null,
$null,
$false
)

Fold-Block -Block $Block -OnTest {
param ($test)
if ($test.ShouldRun -and -not $test.Skip -and -not $test.Executed) {
# Skipping and counting remaining unexecuted tests
$Context.Configuration.SkipRemainingOnFailureCount += 1
$test.Skip = $true
$test.ErrorRecord.Add($errorRecord)
}
} -OnBlock {
param($block)
if ($block.ShouldRun -and -not $block.Skip -and -not $block.Executed) {
# Marking remaining blocks as Skip to avoid executing BeforeAll/AfterAll
$block.Skip = $true
}
fflaten marked this conversation as resolved.
Show resolved Hide resolved
}
}

function Get-SkipRemainingOnFailurePlugin {
# Validate configuration
Expand All @@ -29,41 +54,18 @@ function Get-SkipRemainingOnFailurePlugin {
$p.Start = {
param ($Context)

# TODO: Use $Context.GlobalPluginData.SkipRemainingOnFailure.SkippedCount when exposed in $Context
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this something you will address in this PR or there should be issue created for this?

Copy link
Collaborator Author

@fflaten fflaten May 14, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

New issue. Didn't want to touch plugin interface as it wasn't critical.

I'm happy to provide a follow-up PR when name and steps (all?) are decided.

$Context.Configuration.SkipRemainingOnFailureCount = 0
}

if ($PesterPreference.Run.SkipRemainingOnFailure.Value -eq 'Block') {
$p.EachTestTeardownEnd = {
param($Context)

# If test is not marked skipped and failed
# Go through block tests and child tests and mark unexecuted tests as skipped
# If test was not skipped and failed
if (-not $Context.Test.Skipped -and -not $Context.Test.Passed) {

$errorRecord = [Pester.Factory]::CreateErrorRecord(
'PesterTestSkipped',
(New-SkippedTestMessage -Test $Context.Test),
$null,
$null,
$null,
$false
)

foreach ($test in $Context.Block.Tests) {
if (-not $test.Executed) {
$Context.Configuration.SkipRemainingOnFailureCount += 1
$test.Skip = $true
$test.ErrorRecord.Add($errorRecord)
}
}

foreach ($test in ($Context.Block | View-Flat)) {
if (-not $test.Executed) {
$Context.Configuration.SkipRemainingOnFailureCount += 1
$test.Skip = $true
$test.ErrorRecord.Add($errorRecord)
}
}
# Skip all remaining tests in the block recursively
Set-RemainingAsSkipped -FailedTest $Context.Test -Block $Context.Block
}
}
}
Expand All @@ -72,57 +74,37 @@ function Get-SkipRemainingOnFailurePlugin {
$p.EachTestTeardownEnd = {
param($Context)

# If test is not marked skipped and failed
# Go through every test in container from block root and marked unexecuted tests as skipped
# If test was not skipped and failed
if (-not $Context.Test.Skipped -and -not $Context.Test.Passed) {

$errorRecord = [Pester.Factory]::CreateErrorRecord(
'PesterTestSkipped',
(New-SkippedTestMessage -Test $Context.Test),
$null,
$null,
$null,
$false
)

foreach ($test in ($Context.Block.Root | View-Flat)) {
if (-not $test.Executed) {
$Context.Configuration.SkipRemainingOnFailureCount += 1
$test.Skip = $true
$test.ErrorRecord.Add($errorRecord)
}
}
# Skip all remaining tests in the container recursively
Set-RemainingAsSkipped -FailedTest $Context.Test -Block $Context.Block.Root
}
}
}

elseif ($PesterPreference.Run.SkipRemainingOnFailure.Value -eq 'Run') {
$p.EachTestSetupStart = {
$p.ContainerRunStart = {
param($Context)

# If a test has failed at some point during the run
# Skip the test before it runs
# This handles skipping tests that failed from different containers in the same run
# If a test failed in a previous container, skip all tests
if ($Context.Configuration.SkipRemainingFailedTest) {
$Context.Configuration.SkipRemainingOnFailureCount += 1
$Context.Test.Skip = $true

$errorRecord = [Pester.Factory]::CreateErrorRecord(
'PesterTestSkipped',
(New-SkippedTestMessage -Test $Context.Configuration.SkipRemainingFailedTest),
$null,
$null,
$null,
$false
)
$Context.Test.ErrorRecord.Add($errorRecord)
# Skip container root block to avoid root-level BeforeAll/AfterAll from running. Only applicable in this mode
$Context.Block.Root.Skip = $true
# Skip all remaining tests in current container
Set-RemainingAsSkipped -FailedTest $Context.Configuration.SkipRemainingFailedTest -Block $Context.Block
}
}

$p.EachTestTeardownEnd = {
param($Context)

# If test was not skipped but failed
if (-not $Context.Test.Skipped -and -not $Context.Test.Passed) {
# Skip all remaining tests in current container
Set-RemainingAsSkipped -FailedTest $Context.Test -Block $Context.Block.Root

# Store failed test so we can skip remaining containers in ContainerRunStart-step
# TODO: Use $Context.GlobalPluginData.SkipRemainingOnFailure.FailedTest when exposed in $Context
$Context.Configuration.SkipRemainingFailedTest = $Context.Test
}
}
Expand Down
180 changes: 180 additions & 0 deletions tst/Pester.RSpec.ts.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2591,6 +2591,186 @@ i -PassThru:$PassThru {
$r.Containers[1].Blocks[0].Tests[0].ErrorRecord.FullyQualifiedErrorID | Verify-Equal 'PesterTestSkipped'
$r.Containers[1].Blocks[0].Tests[0].ErrorRecord.TargetObject.Message | Verify-Equal "Skipped due to previous failure at 'a.b' and Run.SkipRemainingOnFailure set to 'Run'"
}

foreach ($mode in 'Block', 'Container', 'Run') {
t "Ignore tests with -Skip or excluded by filter in mode '$mode'" {
$sb1 = {
Describe 'a' {
It 'Included - fails' -Tag 'Demo' {
$false | Should -BeTrue
}
It 'Excluded - ignore' {
$true | Should -BeTrue
}
Describe 'b' {
It 'Included - skip' -Tag 'Demo' {
$true | Should -BeTrue
}
It 'Included but skipped - ignore' -Tag 'Demo' -Skip {
$true | Should -BeTrue
}
}
}
Describe 'c' {
It 'Included - skip on Container and Run' -Tag 'Demo' {
$true | Should -BeTrue
}
}
}

$sb2 = {
Describe 'd' {
It 'Included - skip on Run' -Tag 'Demo' {
$true | Should -BeTrue
}
}
}

$c = [PesterConfiguration] @{
Filter = @{
Tag = 'Demo'
}
Run = @{
ScriptBlock = $sb1, $sb2
PassThru = $true
SkipRemainingOnFailure = $mode
}
Output = @{
CIFormat = 'None'
}
}

$r = Invoke-Pester -Configuration $c

$r.Tests[0].Skipped | Verify-False
$r.Tests[0].Result | Verify-Equal 'Failed'

# Should not mark excluded tests as Skipped
$r.Tests[1].Skipped | Verify-False
$r.Tests[1].Result | Verify-Equal 'NotRun'

# Should mark included test as Skipped
$r.Tests[2].Skipped | Verify-True
$r.Tests[2].Result | Verify-Equal 'Skipped'
$r.Tests[2].ErrorRecord.TargetObject.Message -match '^Skipped due to previous failure' | Verify-True

# Should not modify explicitly skipped tests
$r.Tests[3].Skipped | Verify-True
$r.Tests[3].Result | Verify-Equal 'Skipped'
$r.Tests[3].ErrorRecord | Verify-Null

switch ($mode) {
'Block' { $r.PluginConfiguration.SkipRemainingOnFailureCount | Verify-Equal 1 }
'Container' { $r.PluginConfiguration.SkipRemainingOnFailureCount | Verify-Equal 2 }
'Run' { $r.PluginConfiguration.SkipRemainingOnFailureCount | Verify-Equal 3 }
}
}
}

foreach ($mode in 'Block', 'Container', 'Run') {
t "Remaining blocks are skipped and setup/teardowns are not executed in mode '$mode'" {
$container = [ordered]@{
RootBeforeAll = 0
RootAfterAll = 0
BlockBeforeAll = 0
BlockAfterAll = 0
}

$sb1 = {
BeforeAll { $container.RootBeforeAll++ }
AfterAll { $container.RootAfterAll++ }

Describe 'd1' {
BeforeAll { $container.BlockBeforeAll++ }
AfterAll { $container.BlockAfterAll++ }

It 'Fails' { $false | Should -BeTrue }

Context 'c1' {
BeforeAll { $container.BlockBeforeAll++ }
AfterAll { $container.BlockAfterAll++ }
It 'Skipped' { $true | Should -BeTrue }
}
}
Describe 'd2' {
BeforeAll { $container.BlockBeforeAll++ }
AfterAll { $container.BlockAfterAll++ }

It 'Skipped' { $true | Should -BeTrue }
}
}

$sb2 = {
BeforeAll { $container.RootBeforeAll++ }
AfterAll { $container.RootAfterAll++ }

Describe 'd1' {
BeforeAll { $container.BlockBeforeAll++ }
AfterAll { $container.BlockAfterAll++ }

It 'Skipped' { $true | Should -BeTrue }
}
}

$c = [PesterConfiguration] @{
Run = @{
ScriptBlock = $sb1, $sb2
PassThru = $true
SkipRemainingOnFailure = $mode
}
Output = @{
CIFormat = 'None'
}
}

$r = Invoke-Pester -Configuration $c

switch ($mode) {
'Block' {
$r.Containers[0].Result | Verify-Equal 'Failed'
$r.Containers[0].Blocks[0].Result | Verify-Equal 'Failed'
$r.Containers[0].Blocks[0].Blocks[0].Result | Verify-Equal 'Skipped'
$r.Containers[0].Blocks[1].Result | Verify-Equal 'Passed'

$r.Containers[1].Result | Verify-Equal 'Passed'
$r.Containers[1].Blocks[0].Result | Verify-Equal 'Passed'

$container.RootBeforeAll | Verify-Equal 2
$container.RootAfterAll | Verify-Equal 2
$container.BlockBeforeAll | Verify-Equal 3
$container.BlockAfterAll | Verify-Equal 3
}
'Container' {
$r.Containers[0].Result | Verify-Equal 'Failed'
$r.Containers[0].Blocks[0].Result | Verify-Equal 'Failed'
$r.Containers[0].Blocks[0].Blocks[0].Result | Verify-Equal 'Skipped'
$r.Containers[0].Blocks[1].Result | Verify-Equal 'Skipped'

$r.Containers[1].Result | Verify-Equal 'Passed'
$r.Containers[1].Blocks[0].Result | Verify-Equal 'Passed'

$container.RootBeforeAll | Verify-Equal 2
$container.RootAfterAll | Verify-Equal 2
$container.BlockBeforeAll | Verify-Equal 2
$container.BlockAfterAll | Verify-Equal 2
}
'Run' {
$r.Containers[0].Result | Verify-Equal 'Failed'
$r.Containers[0].Blocks[0].Result | Verify-Equal 'Failed'
$r.Containers[0].Blocks[0].Blocks[0].Result | Verify-Equal 'Skipped'
$r.Containers[0].Blocks[1].Result | Verify-Equal 'Skipped'

$r.Containers[1].Result | Verify-Equal 'Skipped'
$r.Containers[1].Blocks[0].Result | Verify-Equal 'Skipped'

$container.RootBeforeAll | Verify-Equal 1
$container.RootAfterAll | Verify-Equal 1
$container.BlockBeforeAll | Verify-Equal 1
$container.BlockAfterAll | Verify-Equal 1
}
}
}
}
}

b 'Changes to CWD are reverted on exit' {
Expand Down