diff --git a/CHANGELOG.md b/CHANGELOG.md index a9e5b5c4..74d655a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable changes to this project will be documented in this file. - Add order option to all hooks, fixes [#481](https://github.com/badeball/cypress-cucumber-preprocessor/issues/481). +- Add a [`filterSpecsMixedMode`](docs/tags.md#tag-filters-and-non-cucumber-specs) option, fixes [#1125](https://github.com/badeball/cypress-cucumber-preprocessor/issues/1125). + ## v19.1.1 - Mock and imitate Cypress globals during diagnostics / dry run, fixes [#1120](https://github.com/badeball/cypress-cucumber-preprocessor/issues/1120). diff --git a/docs/configuration.md b/docs/configuration.md index 36fc8da7..663b42ad 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -71,18 +71,19 @@ $ CYPRESS_filterSpecs=true cypress run Every configuration option has a similar key which can be use to override it, shown in the table below. -| JSON path | Environment key | Example(s) | -|--------------------|-------------------|------------------------------------------| -| `stepDefinitions` | `stepDefinitions` | `[filepath].{js,ts}` | -| `messages.enabled` | `messagesEnabled` | `true`, `false` | -| `messages.output` | `messagesOutput` | `cucumber-messages.ndjson` | -| `json.enabled` | `jsonEnabled` | `true`, `false` | -| `json.output` | `jsonOutput` | `cucumber-report.json` | -| `html.enabled` | `htmlEnabled` | `true`, `false` | -| `html.output` | `htmlOutput` | `cucumber-report.html` | -| `pretty.enabled` | `prettyEnabled` | `true`, `false` | -| `filterSpecs` | `filterSpecs` | `true`, `false` | -| `omitFiltered` | `omitFiltered` | `true`, `false` | +| JSON path | Environment key | Example(s) | +|------------------------|------------------------|------------------------------------------| +| `stepDefinitions` | `stepDefinitions` | `[filepath].{js,ts}` | +| `messages.enabled` | `messagesEnabled` | `true`, `false` | +| `messages.output` | `messagesOutput` | `cucumber-messages.ndjson` | +| `json.enabled` | `jsonEnabled` | `true`, `false` | +| `json.output` | `jsonOutput` | `cucumber-report.json` | +| `html.enabled` | `htmlEnabled` | `true`, `false` | +| `html.output` | `htmlOutput` | `cucumber-report.html` | +| `pretty.enabled` | `prettyEnabled` | `true`, `false` | +| `filterSpecsMixedMode` | `filterSpecsMixedMode` | `hide`, `show`, `empty-set` | +| `filterSpecs` | `filterSpecs` | `true`, `false` | +| `omitFiltered` | `omitFiltered` | `true`, `false` | ## Test configuration diff --git a/docs/tags.md b/docs/tags.md index 67994518..d445e3be 100644 --- a/docs/tags.md +++ b/docs/tags.md @@ -41,6 +41,14 @@ Tags are inherited by child elements. Tags that are placed above a `Feature` wil Normally when running a subset of scenarios using `cypress run --env tags=@foo`, you could potentially encounter files containing no matching scenarios. These can be pre-filtered away by setting `filterSpecs` to `true`, thus saving you execution time. +### Tag filters and non-Cucumber specs + +If you are mixing Cucumber and non-Cucumber specs, you can control how non-Cucumber specs are filtered when using `filterSpecs` and tag expressions. Filtering non-Cucumber specs (which doesn't contain tags) is not straight forward and there's not a single behavior that's more intuitive than others. Hence there's a `filterSpecsMixedMode` option. Valid options are: + +- "**hide**" (default): non-Cucumber specs are hidden regardless of your tag expression +- "**show**": non-Cucumber specs are shown regardless of your tag expression +- "**empty-set**": non-Cucumber specs are filtered as if having empty set of tags, meaning that positive expressions (`tags=@foo`) will discard these specs, while negative expressions (`tags='not @foo'`) will select them + ## Omit filtered tests By default, all filtered tests are made *pending* using `it.skip` method. If you want to completely omit them, set `omitFiltered` to `true`. diff --git a/features/step_definitions/cli_steps.ts b/features/step_definitions/cli_steps.ts index 527abed3..bf0b4d00 100644 --- a/features/step_definitions/cli_steps.ts +++ b/features/step_definitions/cli_steps.ts @@ -74,21 +74,22 @@ Then("it should appear as if both tests were skipped", function () { ); }); +const ranTestExpr = (spec: string) => + new RegExp("Running:\\s+" + rescape(spec)); + +Then("it should appear to have ran spec {string}", function (spec) { + assert.match(this.lastRun.stdout, ranTestExpr(spec)); +}); + Then("it should appear to not have ran spec {string}", function (spec) { - assert.doesNotMatch( - this.lastRun.stdout, - new RegExp("Running:\\s+" + rescape(spec)) - ); + assert.doesNotMatch(this.lastRun.stdout, ranTestExpr(spec)); }); Then( "it should appear to have ran spec {string} and {string}", function (a, b) { for (const spec of [a, b]) { - assert.match( - this.lastRun.stdout, - new RegExp("Running:\\s+" + rescape(spec)) - ); + assert.match(this.lastRun.stdout, ranTestExpr(spec)); } } ); diff --git a/features/tags/spec_filter.feature b/features/tags/spec_filter.feature index 9d7349df..cbf441c4 100644 --- a/features/tags/spec_filter.feature +++ b/features/tags/spec_filter.feature @@ -1,73 +1,106 @@ Feature: filter spec Background: - Given additional preprocessor configuration + Given additional Cypress configuration + """ + { + "e2e": { + "specPattern": "**/*.{spec.js,feature}" + } + } + """ + And additional preprocessor configuration """ { "filterSpecs": true } """ + And a file named "cypress/e2e/foo.feature" with: + """ + @foo + Feature: some feature + Scenario: first scenario + Given a step + """ + And a file named "cypress/e2e/bar.feature" with: + """ + @bar + Feature: some other feature + Scenario: second scenario + Given a step + """ + And a file named "cypress/support/step_definitions/steps.js" with: + """ + const { Given } = require("@badeball/cypress-cucumber-preprocessor"); + Given("a step", function() {}) + """ + And a file named "cypress/e2e/baz.spec.js" with: + """ + it("should work", () => {}); + """ Rule: it should filter features based on whether they contain a matching scenario Scenario: 1 / 2 specs matching - Given a file named "cypress/e2e/a.feature" with: - """ - @foo - Feature: some feature - Scenario: first scenario - Given a step - """ - And a file named "cypress/e2e/b.feature" with: - """ - @bar - Feature: some other feature - Scenario: second scenario - Given a step - """ - And a file named "cypress/support/step_definitions/steps.js" with: - """ - const { Given } = require("@badeball/cypress-cucumber-preprocessor"); - Given("a step", function() {}) - """ When I run cypress with "--env tags=@foo" Then it passes - And it should appear to not have ran spec "b.feature" + And it should appear to not have ran spec "bar.feature" + But it should appear to have ran spec "foo.feature" + + Scenario: 2 / 2 specs matching + When I run cypress with "--env tags='@foo or @bar'" + Then it passes + And it should appear to have ran spec "foo.feature" and "bar.feature" - Rule: non-feature specs should be filtered as if they have tags equalling the empty set + Rule: filterSpecsMixedMode: hide (default) should hide non-feature specs regardless of tag expression + + Scenario: positive tag expression + When I run cypress with "--env tags=@foo" + Then it passes + And it should appear to not have ran spec "baz.spec.js" + + Scenario: negative tag expression + When I run cypress with "--env 'tags=not @foo'" + Then it passes + And it should appear to not have ran spec "baz.spec.js" + + Rule: filterSpecsMixedMode: show should show non-feature specs regardless of tag expression Background: - Given additional Cypress configuration + Given additional preprocessor configuration """ { - "e2e": { - "specPattern": "**/*.{spec.js,feature}" - } + "filterSpecsMixedMode": "show" } """ - And a file named "cypress/e2e/a.feature" with: - """ - @bar - Feature: some feature - Scenario: first scenario - Given a step - """ - And a file named "cypress/support/step_definitions/steps.js" with: - """ - const { Given } = require("@badeball/cypress-cucumber-preprocessor"); - Given("a step", function() {}) - """ - And a file named "cypress/e2e/b.spec.js" with: + + Scenario: positive tag expression + When I run cypress with "--env tags=@foo" + Then it passes + And it should appear to have ran spec "baz.spec.js" + + Scenario: negative tag expression + When I run cypress with "--env 'tags=not @foo'" + Then it passes + And it should appear to have ran spec "baz.spec.js" + + + Rule: filterSpecsMixedMode: empty-set should filter non-feature specs as if they have tags equalling the empty set + + Background: + Given additional preprocessor configuration """ - it("should work", () => {}); + { + "filterSpecsMixedMode": "empty-set" + } """ - Scenario: logical not - When I run cypress with "--env 'tags=not @foo'" + Scenario: positive tag expression + When I run cypress with "--env tags=@foo" Then it passes - And it should appear to have ran spec "a.feature" and "b.spec.js" + And it should appear to not have ran spec "baz.spec.js" - Scenario: not logical not - When I run cypress with "--env tags=@bar" + Scenario: negative tag expression + When I run cypress with "--env 'tags=not @foo'" Then it passes - And it should appear as if only a single test ran + And it should appear to have ran spec "baz.spec.js" diff --git a/lib/add-cucumber-preprocessor-plugin.ts b/lib/add-cucumber-preprocessor-plugin.ts index 05186d1e..bd500063 100644 --- a/lib/add-cucumber-preprocessor-plugin.ts +++ b/lib/add-cucumber-preprocessor-plugin.ts @@ -45,6 +45,8 @@ import { getTags } from "./helpers/environment"; import { memoize } from "./helpers/memoize"; +import { assertNever } from "./helpers/assertions"; + const resolve = memoize(origResolve); export type AddOptions = { @@ -137,7 +139,16 @@ export async function addCucumberPreprocessorPlugin( config as unknown as ICypressConfiguration ).filter((testFile) => { if (!testFile.endsWith(".feature")) { - return node.evaluate([]); + switch (preprocessor.filterSpecsMixedMode) { + case "hide": + return false; + case "show": + return true; + case "empty-set": + return node.evaluate([]); + default: + assertNever(preprocessor.filterSpecsMixedMode); + } } const content = fs.readFileSync(testFile).toString("utf-8"); diff --git a/lib/helpers/assertions.ts b/lib/helpers/assertions.ts index a16d0a97..f7e7b113 100644 --- a/lib/helpers/assertions.ts +++ b/lib/helpers/assertions.ts @@ -2,6 +2,10 @@ import { createError } from "./error"; import { isString } from "./type-guards"; +export function assertNever(value: never): never { + throw new Error("Illegal value: " + value); +} + export function fail(message: string) { throw createError(message); } diff --git a/lib/preprocessor-configuration.test.ts b/lib/preprocessor-configuration.test.ts index 6e871e80..ca7127c4 100644 --- a/lib/preprocessor-configuration.test.ts +++ b/lib/preprocessor-configuration.test.ts @@ -7,6 +7,7 @@ import assert from "assert"; import { COMPILED_REPORTER_ENTRYPOINT, + FilterSpecsMixedMode, IBaseUserConfiguration, IPreprocessorConfiguration, IUserConfiguration, @@ -633,6 +634,117 @@ describe("resolve()", () => { }); }); + describe("filterSpecsMixedMode", () => { + const getValueFn = ( + configuration: IPreprocessorConfiguration + ): FilterSpecsMixedMode => configuration.filterSpecsMixedMode; + + const setValueFn = ( + configuration: IBaseUserConfiguration, + value: FilterSpecsMixedMode + ) => (configuration.filterSpecsMixedMode = value); + + it("default", () => + test({ + testingType, + getValueFn, + environment: {}, + configuration: {}, + expectedValue: "hide", + })); + + it("override by explicit, type-unspecific configuration", () => + test({ + testingType, + getValueFn, + environment: {}, + configuration: createUserConfiguration({ + setValueFn, + value: "show", + }), + expectedValue: "show", + })); + + it("override by explicit, type-specific configuration", () => + test({ + testingType, + getValueFn, + environment: {}, + configuration: { + [testingType]: createUserConfiguration({ + setValueFn, + value: "show", + }), + }, + expectedValue: "show", + })); + + it("override by environment", () => + test({ + testingType, + getValueFn, + environment: { filterSpecsMixedMode: "show" }, + configuration: {}, + expectedValue: "show", + })); + + it("precedence (environment over explicit, type-unspecific)", () => + test({ + testingType, + getValueFn, + environment: { filterSpecsMixedMode: "empty-set" }, + configuration: createUserConfiguration({ + setValueFn, + value: "show", + }), + expectedValue: "empty-set", + })); + + it("precedence (environment over explicit, type-specific)", () => + test({ + testingType, + getValueFn, + environment: { filterSpecsMixedMode: "empty-set" }, + configuration: { + [testingType]: createUserConfiguration({ + setValueFn, + value: "show", + }), + }, + expectedValue: "empty-set", + })); + + it("precedence (explicit, type-specific over type-unspecific)", () => + test({ + testingType, + getValueFn, + environment: {}, + configuration: { + [testingType]: createUserConfiguration({ + setValueFn, + value: "empty-set", + }), + ...createUserConfiguration({ setValueFn, value: "show" }), + }, + expectedValue: "empty-set", + })); + + it("should fail when configured using non-recognized mode", () => + assert.rejects( + () => + resolve( + Object.assign( + { testingType: testingType }, + DUMMY_POST10_CONFIG + ), + {}, + "cypress/e2e", + () => ({ filterSpecsMixedMode: "foobar" }) + ), + 'Unrecognize filterSpecsMixedMode: foobar (valid options are "hide", "show" and "empty-set")' + )); + }); + describe("filterSpecs", () => { const getValueFn = ( configuration: IPreprocessorConfiguration diff --git a/lib/preprocessor-configuration.ts b/lib/preprocessor-configuration.ts index 968725c5..7995754a 100644 --- a/lib/preprocessor-configuration.ts +++ b/lib/preprocessor-configuration.ts @@ -25,6 +25,12 @@ function isPlainObject(value: any): value is object { return value?.constructor === Object; } +function isFilterSpecsMixedMode(value: any): value is FilterSpecsMixedMode { + const availablesModes = ["hide", "show", "empty-set"]; + + return typeof value === "string" && availablesModes.indexOf(value) !== -1; +} + function validateUserConfigurationEntry( key: string, value: Record @@ -162,6 +168,16 @@ function validateUserConfigurationEntry( }; return { [key]: prettyConfig }; } + case "filterSpecsMixedMode": { + if (!isFilterSpecsMixedMode(value)) { + throw new Error( + `Unrecognize filterSpecsMixedMode: ${util.inspect( + value + )} (valid options are "hide", "show" and "empty-set")` + ); + } + return { [key]: value }; + } case "filterSpecs": { if (!isBoolean(value)) { throw new Error( @@ -321,6 +337,20 @@ function validateEnvironmentOverrides( } } + if (hasOwnProperty(environment, "filterSpecsMixedMode")) { + const { filterSpecsMixedMode } = environment; + + if (isFilterSpecsMixedMode(filterSpecsMixedMode)) { + overrides.filterSpecsMixedMode = filterSpecsMixedMode; + } else { + throw new Error( + `Unrecognize filterSpecsMixedMode: ${util.inspect( + filterSpecsMixedMode + )} (valid options are "hide", "show" and "empty-set")` + ); + } + } + if (hasOwnProperty(environment, "filterSpecs")) { const { filterSpecs } = environment; @@ -368,6 +398,8 @@ function stringToMaybeBoolean(value: string): boolean | undefined { } } +export type FilterSpecsMixedMode = "hide" | "show" | "empty-set"; + interface IEnvironmentOverrides { stepDefinitions?: string | string[]; messagesEnabled?: boolean; @@ -377,6 +409,7 @@ interface IEnvironmentOverrides { htmlEnabled?: boolean; htmlOutput?: string; prettyEnabled?: boolean; + filterSpecsMixedMode?: FilterSpecsMixedMode; filterSpecs?: boolean; omitFiltered?: boolean; } @@ -398,6 +431,7 @@ export interface IBaseUserConfiguration { pretty?: { enabled: boolean; }; + filterSpecsMixedMode?: FilterSpecsMixedMode; filterSpecs?: boolean; omitFiltered?: boolean; } @@ -424,6 +458,7 @@ export interface IPreprocessorConfiguration { readonly pretty: { enabled: boolean; }; + readonly filterSpecsMixedMode: FilterSpecsMixedMode; readonly filterSpecs: boolean; readonly omitFiltered: boolean; readonly implicitIntegrationFolder: string; @@ -509,6 +544,12 @@ export function combineIntoConfiguration( false, }; + const filterSpecsMixedMode: IPreprocessorConfiguration["filterSpecsMixedMode"] = + overrides.filterSpecsMixedMode ?? + specific?.filterSpecsMixedMode ?? + unspecific.filterSpecsMixedMode ?? + "hide"; + const filterSpecs: IPreprocessorConfiguration["filterSpecs"] = overrides.filterSpecs ?? specific?.filterSpecs ?? @@ -527,6 +568,7 @@ export function combineIntoConfiguration( json, html, pretty, + filterSpecsMixedMode, filterSpecs, omitFiltered, implicitIntegrationFolder,