Merge pull request #179 from radish-bdd/feature/error-case-matching

Implement error-case sentence testing

Author: timofurrer

docs/tutorial.rst

@@ -1036,6 +1036,7 @@ sentences defined in a YAML file. We call those files **match configs**. A *matc
 
     - sentence: <SOME STEP SENTENCE>
       should_match: <THE STEP FUNCTION NAME IT SHOULD MATCH>
+      should_not_match: <THE STEP FUNCTION NAME IT SHOULD NOT MATCH>
       with_arguments:
         # argument check if implicit type
         - <ARGUMENT 1 NAME>: <ARGUMENT 1 VALUE>
@@ -1055,10 +1056,13 @@ sentences defined in a YAML file. We call those files **match configs**. A *matc
     It's an example of a sentence which should match
     a certain Step pattern.
 :should_match:
-    **Required**. This is the name of the python Step implementation
+    **Required if should_not_match omitted**. This is the name of the Python Step implementation
     function which you expect the sentence will match with.
+:should_not_match:
+    **Required if should_match omitted**. This is the name of a Python Step implementation
+    function which you expect the sentence will **not** match with.
 :with_arguments:
-    **Optional**. This is a list of arguments which you expect
+    **Optional for should_match**. This is a list of arguments which you expect
     will be passed in the python Step implementation function.
     The arguments can be specified as key-value pairs or as an object
     with a *type* and *value* and a boolean value *cast*.
@@ -1104,6 +1108,13 @@ And a ``step-matches.yml`` file like this:
     - sentence: When I sum them
       should_match: sum_numbers
 
+    - sentence: When I divide them
+      should_not_match: sum_numbers
+
+    - sentence: When I do some weird stuff
+      # if no step is given it shouldn't match any at all
+      should_not_match:
+
     - sentence: Then I expect the result to be 8
       should_match: expect_result
       with_arguments:
@@ -1126,9 +1137,10 @@ For the ``radish-test`` call above we would get the following output:
     Testing sentences from tests/step-matches.yml:
     >> STEP "Given I have the number 5" SHOULD MATCH have_number ✔
     >> STEP "When I sum them" SHOULD MATCH sum_numbers ✔
+    >> STEP "When I divide them" SHOULD NOT MATCH sum_numbers ✔
     >> STEP "Then I expect the result to be 8" SHOULD MATCH expect_result ✔
 
-    3 sentences (3 passed)
+    4 sentences (4 passed)
     Covered 3 of 3 step implementations
 
 In case of success we get the exit code **0** and in case of failure we'd get an exit code which is greater than **0**.

radish/testing/matches.py

@@ -58,7 +58,7 @@ def test_step_matches_configs(match_config_files, basedirs, cover_min_percentage
         failed += failed_sentences
         passed += passed_senteces
 
-        covered_steps = covered_steps.union(x['should_match'] for x in match_config)
+        covered_steps = covered_steps.union(x['should_match'] for x in match_config if 'should_match' in x)
 
         # newline
         sys.stdout.write('\n')
@@ -120,42 +120,66 @@ def test_step_matches(match_config, steps):
         validate_config_item(item)
 
         sentence = item['sentence']
-        expected_step = item['should_match']
 
-        sys.stdout.write('{0} STEP "{1}" SHOULD MATCH {2}    '.format(
-            colorful.yellow('>>'), colorful.cyan(sentence), colorful.cyan(expected_step)))
+        if 'should_match' in item:
+            has_passed = test_step_match(sentence, item['should_match'],
+                                         item.get('with_arguments', None), steps)
+        else:
+            has_passed = test_step_not_match(sentence, item['should_not_match'], steps)
 
-        result = match_step(item['sentence'], steps)
-        if not result:
-            output_failure(None, ['Expected sentence didn\'t match any step implementation'])
+        if has_passed:
+            passed += 1
+        else:
             failed += 1
-            continue
 
-        if expected_step != result.func.__name__:
-            output_failure(result.func, ['Expected sentence matched {0} instead of {1}'.format(result.func.__name__, expected_step)])
-            failed += 1
-            continue
+    return failed, passed
 
 
-        expected_arguments = item.get('with_arguments')
+def test_step_match(sentence, expected_step, expected_arguments, steps):
+    sys.stdout.write('{0} STEP "{1}" SHOULD MATCH {2}    '.format(
+        colorful.yellow('>>'), colorful.cyan(sentence), colorful.cyan(expected_step)))
 
-        if expected_arguments:
-            arguments = merge_step_args(result)
-            expected_arguments = {k: v for expected_arguments in expected_arguments for k, v in expected_arguments.items()}
-            argument_errors = check_step_arguments(expected_arguments, arguments)
-            if argument_errors:
-                output_failure(result.func, argument_errors)
-                failed += 1
-                continue
+    result = match_step(sentence, steps)
+    if not result:
+        output_failure(None, ['Expected sentence didn\'t match any step implementation'])
+        return False
 
-        # check if arguments match
-        print(u(colorful.bold_green(u'✔')))
-        passed += 1
+    if expected_step != result.func.__name__:
+        output_failure(result.func,
+                       ['Expected sentence matched {0} instead of {1}'.format(
+                           result.func.__name__, expected_step)])
+        return False
 
-    return failed, passed
+    if expected_arguments:
+        arguments = merge_step_args(result)
+        expected_arguments = {k: v for expected_arguments in expected_arguments
+                              for k, v in expected_arguments.items()}
+        argument_errors = check_step_arguments(expected_arguments, arguments)
+        if argument_errors:
+            output_failure(result.func, argument_errors)
+            return False
+
+    print(u(colorful.bold_green(u'✔')))
+    return True
+
+
+def test_step_not_match(sentence, expected_not_matching_step, steps):
+    step_to_print = colorful.cyan(expected_not_matching_step) if expected_not_matching_step else 'ANY'
+    sys.stdout.write('{0} STEP "{1}" SHOULD NOT MATCH {2}    '.format(
+        colorful.yellow('>>'), colorful.cyan(sentence), step_to_print))
+
+    result = match_step(sentence, steps)
+    if result:
+        if not expected_not_matching_step or result.func.__name__ == expected_not_matching_step:
+            output_failure(None, ['Expected sentence did match {0} but it shouldn\'t'.format(
+                expected_not_matching_step)])
+            return False
+
+    print(u(colorful.bold_green(u'✔')))
+    return True
 
 
-VALID_CONFIG_ITEMS = {'sentence', 'should_match', 'with_arguments'}
+VALID_CONFIG_ITEMS = {'sentence', 'should_match', 'should_not_match', 'with_arguments'}
 
 
 def validate_config_item(config):
@@ -167,8 +191,8 @@ def validate_config_item(config):
         raise ValueError('The config attributes {0} are invalid. Use only {1}'.format(
             ', '.join(sorted(given_attributes.difference(VALID_CONFIG_ITEMS))), ', '.join(sorted(VALID_CONFIG_ITEMS))))
 
-    if 'sentence' not in config or 'should_match' not in config:
-        raise ValueError('You have to provide a sentence and the function name which should be matched (should_match)')
+    if 'sentence' not in config or ('should_match' not in config and 'should_not_match' not in config):
+        raise ValueError('You have to provide a sentence and the function name which should (not) be matched (should_match, should_not_match)')
 
 
 def output_failure(step_func, errors):

tests/exploratory/basic/tests/radish-matches.yml

@@ -9,6 +9,15 @@
 - sentence: When I sum them
   should_match: sum_numbers
 
+- sentence: When I sum them
+  should_not_match: have_number
+
+- sentence: When I divide them
+  should_not_match:
+
+- sentence: When I divide them
+  should_not_match: sum_numbers
+
 - sentence: Then I expect the result to be 8
   should_match: expect_result
   with_arguments:

tests/unit/testing/test_matches.py

@@ -89,13 +89,13 @@ def test_empty_matches_config(mocker, capsys):
 
 @pytest.mark.parametrize('given_invalid_config, expected_error_msg', [
     ([{'sentence': None}],
-     'You have to provide a sentence and the function name which should be matched (should_match)'),
+     'You have to provide a sentence and the function name which should (not) be matched (should_match, should_not_match)'),
     ([{'should_match': None}],
-     'You have to provide a sentence and the function name which should be matched (should_match)'),
+     'You have to provide a sentence and the function name which should (not) be matched (should_match, should_not_match)'),
     ([{}],
-     'You have to provide a sentence and the function name which should be matched (should_match)'),
+     'You have to provide a sentence and the function name which should (not) be matched (should_match, should_not_match)'),
     ([{'foo': None}],
-     'The config attributes foo are invalid. Use only sentence, should_match, with_arguments')
+     'The config attributes foo are invalid. Use only sentence, should_match, should_not_match, with_arguments')
 ], ids=[
     'test config with missing should_match function attribute',
     'test config with missing sentence attribute',
@@ -183,6 +183,96 @@ def foo(step, foo, bar): pass
     assert actual_returncode == expected_returncode
 
 
+def test_sentence_not_match(capsys):
+    """
+    Test if sentence does not match if that's expected
+    """
+    # given
+    def foo(step, foo, bar): pass
+
+    steps = {re.compile(r'What (.*?) can (.*)'): foo}
+    config = [{
+        'sentence': 'foo', 'should_not_match': 'foo'
+    }]
+    expected_returncode = (0, 1)
+
+    # when
+    actual_returncode = matches.test_step_matches(config, steps)
+    out, _ = capsys.readouterr()
+
+    # then
+    assert actual_returncode == expected_returncode
+
+
+def test_sentence_not_match_anything(capsys):
+    """
+    Test if sentence does not match any steps
+    """
+    # given
+    def foo(step, foo, bar): pass
+
+    steps = {re.compile(r'What (.*?) can (.*)'): foo}
+    config = [{
+        'sentence': 'foo', 'should_not_match': ''
+    }]
+    expected_returncode = (0, 1)
+
+    # when
+    actual_returncode = matches.test_step_matches(config, steps)
+    out, _ = capsys.readouterr()
+
+    # then
+    assert actual_returncode == expected_returncode
+
+
+def test_sentence_not_match_specific_step(capsys):
+    """
+    Test if sentence does not match specific step
+    """
+    # given
+    def foo(step): pass
+
+    def bar(step): pass
+
+    steps = {
+        re.compile(r'foo'): foo,
+        re.compile(r'bar'): bar,
+    }
+    config = [{
+        'sentence': 'foo', 'should_not_match': 'bar'
+    }]
+    expected_returncode = (0, 1)
+
+    # when
+    actual_returncode = matches.test_step_matches(config, steps)
+    out, _ = capsys.readouterr()
+
+    # then
+    assert actual_returncode == expected_returncode
+
+
+def test_sentence_not_match_but_does(capsys):
+    """
+    Test if sentence matched step but shouldn't
+    """
+    # given
+    def foo(): pass
+
+    steps = {'foo': foo}
+    config = [{
+        'sentence': 'foo', 'should_not_match': 'foo'
+    }]
+    expected_returncode = (1, 0)
+
+    # when
+    actual_returncode = matches.test_step_matches(config, steps)
+    out, _ = capsys.readouterr()
+
+    # then
+    assert 'Expected sentence did match foo but it shouldn\'t' in out
+    assert actual_returncode == expected_returncode
+
+
 @pytest.mark.parametrize('given_actual_arguments, expected_messages', [
     (
         {'FOO': None},