Add a filterSpecsMixedMode option

Filtering non-Cucumber specs (which doesn't contain tags) is not straight forward and there's not a single behavior that's more intuitive than others. Hence a `filterSpecsMixedMode` option. This fixes #1125 [1], essentially reverts 0b2702b [2] by retaining original behavior by default, and also relates to #1116 [3]. [1] #1125 [2] 0b2702b [3] #1116
badeball · Nov 26, 2023 · a9f5b46 · a9f5b46
1 parent 1e3643a
commit a9f5b46
Show file tree

Hide file tree

Showing 9 changed files with 281 additions and 67 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,8 @@ All notable changes to this project will be documented in this file.
 
 - Add order option to all hooks, fixes [#481](https://github.com/badeball/cypress-cucumber-preprocessor/issues/481).
 
+- Add a [`filterSpecsMixedMode`](docs/tags.md#tag-filters-and-non-cucumber-specs) option, fixes [#1125](https://github.com/badeball/cypress-cucumber-preprocessor/issues/1125).
+
 ## v19.1.1
 
 - Mock and imitate Cypress globals during diagnostics / dry run, fixes [#1120](https://github.com/badeball/cypress-cucumber-preprocessor/issues/1120).

diff --git a/docs/configuration.md b/docs/configuration.md
@@ -71,18 +71,19 @@ $ CYPRESS_filterSpecs=true cypress run
 
 Every configuration option has a similar key which can be use to override it, shown in the table below.
 
-| JSON path          | Environment key   | Example(s)                               |
-|--------------------|-------------------|------------------------------------------|
-| `stepDefinitions`  | `stepDefinitions` | `[filepath].{js,ts}`                     |
-| `messages.enabled` | `messagesEnabled` | `true`, `false`                          |
-| `messages.output`  | `messagesOutput`  | `cucumber-messages.ndjson`               |
-| `json.enabled`     | `jsonEnabled`     | `true`, `false`                          |
-| `json.output`      | `jsonOutput`      | `cucumber-report.json`                   |
-| `html.enabled`     | `htmlEnabled`     | `true`, `false`                          |
-| `html.output`      | `htmlOutput`      | `cucumber-report.html`                   |
-| `pretty.enabled`   | `prettyEnabled`   | `true`, `false`                          |
-| `filterSpecs`      | `filterSpecs`     | `true`, `false`                          |
-| `omitFiltered`     | `omitFiltered`    | `true`, `false`                          |
+| JSON path              | Environment key        | Example(s)                               |
+|------------------------|------------------------|------------------------------------------|
+| `stepDefinitions`      | `stepDefinitions`      | `[filepath].{js,ts}`                     |
+| `messages.enabled`     | `messagesEnabled`      | `true`, `false`                          |
+| `messages.output`      | `messagesOutput`       | `cucumber-messages.ndjson`               |
+| `json.enabled`         | `jsonEnabled`          | `true`, `false`                          |
+| `json.output`          | `jsonOutput`           | `cucumber-report.json`                   |
+| `html.enabled`         | `htmlEnabled`          | `true`, `false`                          |
+| `html.output`          | `htmlOutput`           | `cucumber-report.html`                   |
+| `pretty.enabled`       | `prettyEnabled`        | `true`, `false`                          |
+| `filterSpecsMixedMode` | `filterSpecsMixedMode` | `hide`, `show`, `empty-set`              |
+| `filterSpecs`          | `filterSpecs`          | `true`, `false`                          |
+| `omitFiltered`         | `omitFiltered`         | `true`, `false`                          |
 
 ## Test configuration
 

diff --git a/docs/tags.md b/docs/tags.md
@@ -41,6 +41,14 @@ Tags are inherited by child elements. Tags that are placed above a `Feature` wil
 
 Normally when running a subset of scenarios using `cypress run --env tags=@foo`, you could potentially encounter files containing no matching scenarios. These can be pre-filtered away by setting `filterSpecs` to `true`, thus saving you execution time.
 
+### Tag filters and non-Cucumber specs
+
+If you are mixing Cucumber and non-Cucumber specs, you can control how non-Cucumber specs are filtered when using `filterSpecs`  and tag expressions. Filtering non-Cucumber specs (which doesn't contain tags) is not straight forward and there's not a single behavior that's more intuitive than others. Hence there's a `filterSpecsMixedMode` option. Valid options are:
+
+- "**hide**" (default): non-Cucumber specs are hidden regardless of your tag expression
+- "**show**": non-Cucumber specs are shown regardless of your tag expression
+- "**empty-set**": non-Cucumber specs are filtered as if having empty set of tags, meaning that positive expressions (`tags=@foo`) will discard these specs, while negative expressions (`tags='not @foo'`) will select them
+
 ## Omit filtered tests
 
 By default, all filtered tests are made *pending* using `it.skip` method. If you want to completely omit them, set `omitFiltered` to `true`.

diff --git a/features/step_definitions/cli_steps.ts b/features/step_definitions/cli_steps.ts
@@ -74,21 +74,22 @@ Then("it should appear as if both tests were skipped", function () {
   );
 });
 
+const ranTestExpr = (spec: string) =>
+  new RegExp("Running:\\s+" + rescape(spec));
+
+Then("it should appear to have ran spec {string}", function (spec) {
+  assert.match(this.lastRun.stdout, ranTestExpr(spec));
+});
+
 Then("it should appear to not have ran spec {string}", function (spec) {
-  assert.doesNotMatch(
-    this.lastRun.stdout,
-    new RegExp("Running:\\s+" + rescape(spec))
-  );
+  assert.doesNotMatch(this.lastRun.stdout, ranTestExpr(spec));
 });
 
 Then(
   "it should appear to have ran spec {string} and {string}",
   function (a, b) {
     for (const spec of [a, b]) {
-      assert.match(
-        this.lastRun.stdout,
-        new RegExp("Running:\\s+" + rescape(spec))
-      );
+      assert.match(this.lastRun.stdout, ranTestExpr(spec));
     }
   }
 );

diff --git a/features/tags/spec_filter.feature b/features/tags/spec_filter.feature
@@ -1,73 +1,106 @@
 Feature: filter spec
 
   Background:
-    Given additional preprocessor configuration
+    Given additional Cypress configuration
+      """
+      {
+        "e2e": {
+          "specPattern": "**/*.{spec.js,feature}"
+        }
+      }
+      """
+    And additional preprocessor configuration
       """
       {
         "filterSpecs": true
       }
       """
+    And a file named "cypress/e2e/foo.feature" with:
+      """
+      @foo
+      Feature: some feature
+        Scenario: first scenario
+          Given a step
+      """
+    And a file named "cypress/e2e/bar.feature" with:
+      """
+      @bar
+      Feature: some other feature
+        Scenario: second scenario
+          Given a step
+      """
+    And a file named "cypress/support/step_definitions/steps.js" with:
+      """
+      const { Given } = require("@badeball/cypress-cucumber-preprocessor");
+      Given("a step", function() {})
+      """
+    And a file named "cypress/e2e/baz.spec.js" with:
+      """
+      it("should work", () => {});
+      """
 
   Rule: it should filter features based on whether they contain a matching scenario
 
     Scenario: 1 / 2 specs matching
-      Given a file named "cypress/e2e/a.feature" with:
-        """
-        @foo
-        Feature: some feature
-          Scenario: first scenario
-            Given a step
-        """
-      And a file named "cypress/e2e/b.feature" with:
-        """
-        @bar
-        Feature: some other feature
-          Scenario: second scenario
-            Given a step
-        """
-      And a file named "cypress/support/step_definitions/steps.js" with:
-        """
-        const { Given } = require("@badeball/cypress-cucumber-preprocessor");
-        Given("a step", function() {})
-        """
       When I run cypress with "--env tags=@foo"
       Then it passes
-      And it should appear to not have ran spec "b.feature"
+      And it should appear to not have ran spec "bar.feature"
+      But it should appear to have ran spec "foo.feature"
+
+    Scenario: 2 / 2 specs matching
+      When I run cypress with "--env tags='@foo or @bar'"
+      Then it passes
+      And it should appear to have ran spec "foo.feature" and "bar.feature"
 
-  Rule: non-feature specs should be filtered as if they have tags equalling the empty set
+  Rule: filterSpecsMixedMode: hide (default) should hide non-feature specs regardless of tag expression
+
+    Scenario: positive tag expression
+      When I run cypress with "--env tags=@foo"
+      Then it passes
+      And it should appear to not have ran spec "baz.spec.js"
+
+    Scenario: negative tag expression
+      When I run cypress with "--env 'tags=not @foo'"
+      Then it passes
+      And it should appear to not have ran spec "baz.spec.js"
+
+  Rule: filterSpecsMixedMode: show should show non-feature specs regardless of tag expression
 
     Background:
-      Given additional Cypress configuration
+      Given additional preprocessor configuration
         """
         {
-          "e2e": {
-            "specPattern": "**/*.{spec.js,feature}"
-          }
+          "filterSpecsMixedMode": "show"
         }
         """
-      And a file named "cypress/e2e/a.feature" with:
-        """
-        @bar
-        Feature: some feature
-          Scenario: first scenario
-            Given a step
-        """
-      And a file named "cypress/support/step_definitions/steps.js" with:
-        """
-        const { Given } = require("@badeball/cypress-cucumber-preprocessor");
-        Given("a step", function() {})
-        """
-      And a file named "cypress/e2e/b.spec.js" with:
+
+    Scenario: positive tag expression
+      When I run cypress with "--env tags=@foo"
+      Then it passes
+      And it should appear to have ran spec "baz.spec.js"
+
+    Scenario: negative tag expression
+      When I run cypress with "--env 'tags=not @foo'"
+      Then it passes
+      And it should appear to have ran spec "baz.spec.js"
+
+
+  Rule: filterSpecsMixedMode: empty-set should filter non-feature specs as if they have tags equalling the empty set
+
+    Background:
+      Given additional preprocessor configuration
         """
-        it("should work", () => {});
+        {
+          "filterSpecsMixedMode": "empty-set"
+        }
         """
 
-    Scenario: logical not
-      When I run cypress with "--env 'tags=not @foo'"
+    Scenario: positive tag expression
+      When I run cypress with "--env tags=@foo"
       Then it passes
-      And it should appear to have ran spec "a.feature" and "b.spec.js"
+      And it should appear to not have ran spec "baz.spec.js"
 
-    Scenario: not logical not
-      When I run cypress with "--env tags=@bar"
+    Scenario: negative tag expression
+      When I run cypress with "--env 'tags=not @foo'"
       Then it passes
-      And it should appear as if only a single test ran
+      And it should appear to have ran spec "baz.spec.js"
diff --git a/lib/add-cucumber-preprocessor-plugin.ts b/lib/add-cucumber-preprocessor-plugin.ts
@@ -45,6 +45,8 @@ import { getTags } from "./helpers/environment";
 
 import { memoize } from "./helpers/memoize";
 
+import { assertNever } from "./helpers/assertions";
+
 const resolve = memoize(origResolve);
 
 export type AddOptions = {
@@ -137,7 +139,16 @@ export async function addCucumberPreprocessorPlugin(
       config as unknown as ICypressConfiguration
     ).filter((testFile) => {
       if (!testFile.endsWith(".feature")) {
-        return node.evaluate([]);
+        switch (preprocessor.filterSpecsMixedMode) {
+          case "hide":
+            return false;
+          case "show":
+            return true;
+          case "empty-set":
+            return node.evaluate([]);
+          default:
+            assertNever(preprocessor.filterSpecsMixedMode);
+        }
       }
 
       const content = fs.readFileSync(testFile).toString("utf-8");

diff --git a/lib/helpers/assertions.ts b/lib/helpers/assertions.ts
@@ -2,6 +2,10 @@ import { createError } from "./error";
 
 import { isString } from "./type-guards";
 
+export function assertNever(value: never): never {
+  throw new Error("Illegal value: " + value);
+}
+
 export function fail(message: string) {
   throw createError(message);
 }