Python: Handle empty lines in Description AST node and add test cases when there is only a comment between the scenario line and the first step (#378)

* Add test cases

* fix approved results

* [Py] Handle case where line_tokens in Description AST node can be an empty list

* Add changelog for regression fix when a feature file only has a comment in the description

---------

Co-authored-by: Jason Allen <jsa34@cam.ac.uk>

Author: gasparnagy

CHANGELOG.md

@@ -9,6 +9,7 @@ This document is formatted according to the principles of [Keep A CHANGELOG](htt
 ## [Unreleased]
 ### Changed
 - [CI] Replace deprecated `::set-output`
+- [Python] Fix a regression when a feature file only has a comment in the description
 
 ## [32.0.0] - 2025-02-17
 ### Changed

python/gherkin/ast_builder.py

@@ -268,15 +268,13 @@ def transform_node(
             return self.get_table_rows(node)
         elif node.rule_type == "Description":
             line_tokens = node.get_tokens("Other")
+            tokens = list(line_tokens)
+
             # Trim trailing empty lines
-            last_non_empty = next(
-                i for i, j in reversed(list(enumerate(line_tokens))) if j.matched_text
-            )
-            description = "\n".join(
-                [token.matched_text for token in line_tokens[: last_non_empty + 1]]
-            )
+            while tokens and not tokens[-1].matched_text:
+                tokens.pop()
 
-            return description
+            return "\n".join(token.matched_text for token in tokens)
         elif node.rule_type == "Rule":
             header = cast(Union[AstNode, None], node.get_single("RuleHeader"))
             if not header:

testdata/good/descriptions_with_comments.feature

@@ -52,3 +52,13 @@ with a comment in the middle
 
     | foo |
     | bar |
+
+  Scenario: scenario with just a comment
+    # comment
+    Given the minimalism
+
+  Scenario: scenario with a comment with new lines around
+
+    # comment
+
+    Given the minimalism  

testdata/good/descriptions_with_comments.feature.ast.ndjson

@@ -1 +1 @@
-{"gherkinDocument":{"comments":[{"location":{"column":1,"line":3},"text":"  # comment"},{"location":{"column":1,"line":5},"text":"  # comment 2"},{"location":{"column":1,"line":9},"text":"  # comment"},{"location":{"column":1,"line":10},"text":"  # comment 2"},{"location":{"column":1,"line":16},"text":"# comment"},{"location":{"column":1,"line":18},"text":"# comment 2"},{"location":{"column":1,"line":24},"text":"  # comment"},{"location":{"column":1,"line":31},"text":"  # comment"},{"location":{"column":1,"line":34},"text":"  # comment"},{"location":{"column":1,"line":39},"text":"# comment"},{"location":{"column":1,"line":41},"text":"# comment 2"},{"location":{"column":1,"line":43},"text":"# comment 3"},{"location":{"column":1,"line":47},"text":"# comment"},{"location":{"column":1,"line":49},"text":"# comment"},{"location":{"column":1,"line":51},"text":"# comment"}],"feature":{"children":[{"scenario":{"description":"  This description\n  has two lines and two comments in the middle and is indented with two spaces","examples":[],"id":"1","keyword":"Scenario","location":{"column":3,"line":7},"name":"two lines","steps":[{"id":"0","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":12},"text":"the minimalism"}],"tags":[]}},{"scenario":{"description":"This is a description without indentation\nand a comment in the middle and at the end","examples":[],"id":"3","keyword":"Scenario","location":{"column":1,"line":14},"name":"without indentation","steps":[{"id":"2","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":20},"text":"the minimalism"}],"tags":[]}},{"scenario":{"description":"  This description\n\n  has an empty line and a comment in the middle","examples":[],"id":"5","keyword":"Scenario","location":{"column":3,"line":22},"name":"empty lines in the middle","steps":[{"id":"4","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":27},"text":"the minimalism"}],"tags":[]}},{"scenario":{"description":"  This description\n  has an empty lines around","examples":[],"id":"7","keyword":"Scenario","location":{"column":3,"line":29},"name":"empty lines around","steps":[{"id":"6","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":36},"text":"the minimalism"}],"tags":[]}},{"scenario":{"description":"This is a scenario outline description with comments\nin the middle and before and at the end","examples":[{"description":"This is an examples description\nwith a comment in the middle","id":"11","keyword":"Examples","location":{"column":3,"line":46},"name":"examples with description","tableBody":[{"cells":[{"location":{"column":7,"line":54},"value":"bar"}],"id":"10","location":{"column":5,"line":54}}],"tableHeader":{"cells":[{"location":{"column":7,"line":53},"value":"foo"}],"id":"9","location":{"column":5,"line":53}},"tags":[]}],"id":"12","keyword":"Scenario Outline","location":{"column":3,"line":38},"name":"scenario outline with a description","steps":[{"id":"8","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":44},"text":"the minimalism"}],"tags":[]}}],"description":"  This is a description\n  with a comment in the middle and at the end","keyword":"Feature","language":"en","location":{"column":1,"line":1},"name":"Descriptions with comments everywhere","tags":[]},"uri":"../testdata/good/descriptions_with_comments.feature"}}
+{"gherkinDocument":{"comments":[{"location":{"column":1,"line":3},"text":"  # comment"},{"location":{"column":1,"line":5},"text":"  # comment 2"},{"location":{"column":1,"line":9},"text":"  # comment"},{"location":{"column":1,"line":10},"text":"  # comment 2"},{"location":{"column":1,"line":16},"text":"# comment"},{"location":{"column":1,"line":18},"text":"# comment 2"},{"location":{"column":1,"line":24},"text":"  # comment"},{"location":{"column":1,"line":31},"text":"  # comment"},{"location":{"column":1,"line":34},"text":"  # comment"},{"location":{"column":1,"line":39},"text":"# comment"},{"location":{"column":1,"line":41},"text":"# comment 2"},{"location":{"column":1,"line":43},"text":"# comment 3"},{"location":{"column":1,"line":47},"text":"# comment"},{"location":{"column":1,"line":49},"text":"# comment"},{"location":{"column":1,"line":51},"text":"# comment"},{"location":{"column":1,"line":57},"text":"    # comment"},{"location":{"column":1,"line":62},"text":"    # comment"}],"feature":{"children":[{"scenario":{"description":"  This description\n  has two lines and two comments in the middle and is indented with two spaces","examples":[],"id":"1","keyword":"Scenario","location":{"column":3,"line":7},"name":"two lines","steps":[{"id":"0","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":12},"text":"the minimalism"}],"tags":[]}},{"scenario":{"description":"This is a description without indentation\nand a comment in the middle and at the end","examples":[],"id":"3","keyword":"Scenario","location":{"column":1,"line":14},"name":"without indentation","steps":[{"id":"2","keyword":"Given ","keywordType":"Context","location":{"column":3,"line":20},"text":"the minimalism"}],"tags":[]}},{"scenario":{"description":"  This description\n\n  has an empty line and a comment in the middle","examples":[],"id":"5","keyword":"Scenario","location":{"column":3,"line":22},"name":"empty lines in the middle","steps":[{"id":"4","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":27},"text":"the minimalism"}],"tags":[]}},{"scenario":{"description":"  This description\n  has an empty lines around","examples":[],"id":"7","keyword":"Scenario","location":{"column":3,"line":29},"name":"empty lines around","steps":[{"id":"6","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":36},"text":"the minimalism"}],"tags":[]}},{"scenario":{"description":"This is a scenario outline description with comments\nin the middle and before and at the end","examples":[{"description":"This is an examples description\nwith a comment in the middle","id":"11","keyword":"Examples","location":{"column":3,"line":46},"name":"examples with description","tableBody":[{"cells":[{"location":{"column":7,"line":54},"value":"bar"}],"id":"10","location":{"column":5,"line":54}}],"tableHeader":{"cells":[{"location":{"column":7,"line":53},"value":"foo"}],"id":"9","location":{"column":5,"line":53}},"tags":[]}],"id":"12","keyword":"Scenario Outline","location":{"column":3,"line":38},"name":"scenario outline with a description","steps":[{"id":"8","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":44},"text":"the minimalism"}],"tags":[]}},{"scenario":{"description":"","examples":[],"id":"14","keyword":"Scenario","location":{"column":3,"line":56},"name":"scenario with just a comment","steps":[{"id":"13","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":58},"text":"the minimalism"}],"tags":[]}},{"scenario":{"description":"","examples":[],"id":"16","keyword":"Scenario","location":{"column":3,"line":60},"name":"scenario with a comment with new lines around","steps":[{"id":"15","keyword":"Given ","keywordType":"Context","location":{"column":5,"line":64},"text":"the minimalism"}],"tags":[]}}],"description":"  This is a description\n  with a comment in the middle and at the end","keyword":"Feature","language":"en","location":{"column":1,"line":1},"name":"Descriptions with comments everywhere","tags":[]},"uri":"../testdata/good/descriptions_with_comments.feature"}}

testdata/good/descriptions_with_comments.feature.pickles.ndjson

@@ -1,5 +1,7 @@
-{"pickle":{"astNodeIds":["1"],"id":"14","language":"en","name":"two lines","steps":[{"astNodeIds":["0"],"id":"13","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}
-{"pickle":{"astNodeIds":["3"],"id":"16","language":"en","name":"without indentation","steps":[{"astNodeIds":["2"],"id":"15","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}
-{"pickle":{"astNodeIds":["5"],"id":"18","language":"en","name":"empty lines in the middle","steps":[{"astNodeIds":["4"],"id":"17","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}
-{"pickle":{"astNodeIds":["7"],"id":"20","language":"en","name":"empty lines around","steps":[{"astNodeIds":["6"],"id":"19","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}
-{"pickle":{"astNodeIds":["12","10"],"id":"22","language":"en","name":"scenario outline with a description","steps":[{"astNodeIds":["8","10"],"id":"21","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}
+{"pickle":{"astNodeIds":["1"],"id":"18","language":"en","name":"two lines","steps":[{"astNodeIds":["0"],"id":"17","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}
+{"pickle":{"astNodeIds":["3"],"id":"20","language":"en","name":"without indentation","steps":[{"astNodeIds":["2"],"id":"19","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}
+{"pickle":{"astNodeIds":["5"],"id":"22","language":"en","name":"empty lines in the middle","steps":[{"astNodeIds":["4"],"id":"21","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}
+{"pickle":{"astNodeIds":["7"],"id":"24","language":"en","name":"empty lines around","steps":[{"astNodeIds":["6"],"id":"23","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}
+{"pickle":{"astNodeIds":["12","10"],"id":"26","language":"en","name":"scenario outline with a description","steps":[{"astNodeIds":["8","10"],"id":"25","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}
+{"pickle":{"astNodeIds":["14"],"id":"28","language":"en","name":"scenario with just a comment","steps":[{"astNodeIds":["13"],"id":"27","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}
+{"pickle":{"astNodeIds":["16"],"id":"30","language":"en","name":"scenario with a comment with new lines around","steps":[{"astNodeIds":["15"],"id":"29","text":"the minimalism","type":"Context"}],"tags":[],"uri":"../testdata/good/descriptions_with_comments.feature"}}

testdata/good/descriptions_with_comments.feature.source.ndjson

@@ -1 +1 @@
-{"source":{"data":"Feature: Descriptions with comments everywhere\n  This is a description\n  # comment\n  with a comment in the middle and at the end\n  # comment 2\n\n  Scenario: two lines\n  This description\n  # comment\n  # comment 2\n  has two lines and two comments in the middle and is indented with two spaces\n    Given the minimalism\n\nScenario: without indentation\nThis is a description without indentation\n# comment\nand a comment in the middle and at the end\n# comment 2\n\n  Given the minimalism\n\n  Scenario: empty lines in the middle\n  This description\n  # comment\n\n  has an empty line and a comment in the middle\n    Given the minimalism\n\n  Scenario: empty lines around\n\n  # comment\n  This description\n  has an empty lines around\n  # comment\n\n    Given the minimalism\n\n  Scenario Outline: scenario outline with a description\n# comment\nThis is a scenario outline description with comments\n# comment 2\nin the middle and before and at the end\n# comment 3\n    Given the minimalism\n\n  Examples: examples with description\n# comment\nThis is an examples description\n# comment\nwith a comment in the middle\n# comment\n\n    | foo |\n    | bar |\n","mediaType":"text/x.cucumber.gherkin+plain","uri":"../testdata/good/descriptions_with_comments.feature"}}
+{"source":{"data":"Feature: Descriptions with comments everywhere\n  This is a description\n  # comment\n  with a comment in the middle and at the end\n  # comment 2\n\n  Scenario: two lines\n  This description\n  # comment\n  # comment 2\n  has two lines and two comments in the middle and is indented with two spaces\n    Given the minimalism\n\nScenario: without indentation\nThis is a description without indentation\n# comment\nand a comment in the middle and at the end\n# comment 2\n\n  Given the minimalism\n\n  Scenario: empty lines in the middle\n  This description\n  # comment\n\n  has an empty line and a comment in the middle\n    Given the minimalism\n\n  Scenario: empty lines around\n\n  # comment\n  This description\n  has an empty lines around\n  # comment\n\n    Given the minimalism\n\n  Scenario Outline: scenario outline with a description\n# comment\nThis is a scenario outline description with comments\n# comment 2\nin the middle and before and at the end\n# comment 3\n    Given the minimalism\n\n  Examples: examples with description\n# comment\nThis is an examples description\n# comment\nwith a comment in the middle\n# comment\n\n    | foo |\n    | bar |\n\n  Scenario: scenario with just a comment\n    # comment\n    Given the minimalism\n\n  Scenario: scenario with a comment with new lines around\n\n    # comment\n\n    Given the minimalism  ","mediaType":"text/x.cucumber.gherkin+plain","uri":"../testdata/good/descriptions_with_comments.feature"}}

testdata/good/descriptions_with_comments.feature.tokens

@@ -52,4 +52,14 @@
 (52:1)Other://
 (53:5)TableRow://7:foo
 (54:5)TableRow://7:bar
+(55:1)Empty://
+(56:3)ScenarioLine:()Scenario/scenario with just a comment/
+(57:1)Comment:/    # comment/
+(58:5)StepLine:(Context)Given /the minimalism/
+(59:1)Empty://
+(60:3)ScenarioLine:()Scenario/scenario with a comment with new lines around/
+(61:1)Empty://
+(62:1)Comment:/    # comment/
+(63:1)Other://
+(64:5)StepLine:(Context)Given /the minimalism/
 EOF