From 35aeaf6d43c983fa697f6e9441c7bd2061ab4c33 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Mon, 19 Aug 2024 15:26:22 -0700 Subject: [PATCH] Implement granular protocol tests ignore list. --- .../protocols/protocol-tests-ignore-list.json | 100 ++++++++++++++++++ tests/unit/test_protocols.py | 72 +++++++++++-- 2 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 tests/unit/protocols/protocol-tests-ignore-list.json diff --git a/tests/unit/protocols/protocol-tests-ignore-list.json b/tests/unit/protocols/protocol-tests-ignore-list.json new file mode 100644 index 0000000000..562e73f013 --- /dev/null +++ b/tests/unit/protocols/protocol-tests-ignore-list.json @@ -0,0 +1,100 @@ +{ + "general": { + "input": { + "suites": [ + "Test cases for PutWithContentEncoding operation", + "Test cases for QueryIdempotencyTokenAutoFill operation", + "Test cases for HostWithPathOperation operation" + ] + } + }, + "protocols": { + "ec2" : { + "input" : { + "cases": [ + "Ec2QueryEndpointTraitWithHostLabel", + "Ec2Lists", + "Ec2TimestampsInput" + ] + }, + "output": { + "cases": [ + "Ec2ComplexError" + ] + } + }, + "query" : { + "output": { + "cases": [ + "QueryEmptyInputAndEmptyOutput", + "QueryCustomizedError", + "QueryComplexError", + "QueryNoInputAndNoOutput", + "QueryNoInputAndOutput" + ] + } + }, + "json" : { + "input" : { + "cases": [ + "sends_requests_to_slash", + "includes_x_amz_target_and_content_type" + ] + }, + "output" : { + "cases": [ + "AwsJson11FooErrorUsingXAmznErrorTypeWithUriAndNamespace", + "AwsJson11FooErrorUsingXAmznErrorTypeWithUri" + ] + } + }, + "json_1_0" : { + "output" : { + "cases": [ + "AwsJson10FooErrorUsingXAmznErrorType", + "AwsJson10FooErrorUsingXAmznErrorTypeWithUri", + "AwsJson10FooErrorUsingXAmznErrorTypeWithUriAndNamespace" + ] + } + }, + "rest-json" : { + "input" : { + "cases": [ + "MediaTypeHeaderInputBase64", + "RestJsonHttpChecksumRequired" + ] + }, + "output" : { + "cases": [ + "RestJsonFooErrorUsingXAmznErrorType", + "RestJsonFooErrorUsingXAmznErrorTypeWithUri", + "RestJsonFooErrorUsingXAmznErrorTypeWithUriAndNamespace", + "RestJsonHttpPayloadTraitsWithNoBlobBody", + "RestJsonHttpPayloadWithUnsetUnion", + "RestJsonInputAndOutputWithTimestampHeaders" + ] + } + }, + "rest-xml": { + "input": { + "cases": [ + "BodyWithXmlName", + "RestXmlHttpPayloadWithUnion", + "HttpPayloadWithXmlName", + "HttpPayloadWithXmlNamespace", + "HttpPayloadWithXmlNamespaceAndPrefix", + "XmlAttributesOnPayload" + ] + }, + "output": { + "cases": [ + "InputAndOutputWithTimestampHeaders", + "RestXmlHttpPayloadWithUnsetUnion", + "HttpPayloadTraitsWithNoBlobBody", + "ComplexError", + "BodyWithXmlName" + ] + } + } + } +} diff --git a/tests/unit/test_protocols.py b/tests/unit/test_protocols.py index 501c27b509..4d2ac9caea 100644 --- a/tests/unit/test_protocols.py +++ b/tests/unit/test_protocols.py @@ -98,12 +98,10 @@ 'rest-json': RestJSONParser, 'rest-xml': RestXMLParser, } -PROTOCOL_TEST_BLOCKLIST = ( - # These cases test functionality outside the serializers and parsers. - "Test cases for QueryIdempotencyTokenAutoFill operation", - "Test cases for PutWithContentEncoding operation", - "Test cases for HttpChecksumRequired operation", -) +IGNORE_LIST_FILENAME = "protocol-tests-ignore-list.json" +PROTOCOL_TEST_IGNORE_LIST_PATH = os.path.join(TEST_DIR, IGNORE_LIST_FILENAME) +with open(PROTOCOL_TEST_IGNORE_LIST_PATH) as f: + PROTOCOL_TEST_IGNORE_LIST = json.load(f) class TestType(Enum): @@ -121,7 +119,13 @@ def _compliance_tests(test_type=None): for full_path in _walk_files(): if full_path.endswith('.json'): for model, case, basename in _load_cases(full_path): - if model.get('description') in PROTOCOL_TEST_BLOCKLIST: + protocol = basename.replace('.json', '') + if _should_ignore_test( + protocol, + "input" if inp else "output", + model['description'], + case['id'], + ): continue if 'params' in case and inp: yield model, case, basename @@ -431,7 +435,7 @@ def _assert_requests_equal(actual, expected, protocol_type): expected_body = expected.get('body', '').encode('utf-8') actual_body = actual['body'] # The expected bodies in our consumed protocol tests have extra - # whitespace and newlines that need to handled. We need to normalize + # whitespace and newlines that need to be handled. We need to normalize # the expected and actual response bodies before evaluating equivalence. try: if protocol_type in ['json', 'rest-json']: @@ -481,6 +485,8 @@ def _walk_files(): else: for root, _, filenames in os.walk(TEST_DIR): for filename in filenames: + if filename == IGNORE_LIST_FILENAME: + continue yield os.path.join(root, filename) @@ -525,3 +531,53 @@ def _get_suite_test_id(): "integers." ) return suite_id, test_id + + +def _should_ignore_test(protocol, test_type, suite, case): + """ + Determines if a protocol test should be ignored. + + This function checks the following, in order: + 1. General test suites to ignore across all protocols. + 2. General test cases to ignore across all protocols. + 3. Protocol-specific test suites to ignore + 4. Protocol-specific test cases to ignore + + :type protocol: str + :param protocol: The protocol name as represented by its corresponding + protocol test file name (without the .json extension). + + :type test_type: str + :param test_type: The protocol test type ("input" or "output"). + + :type suite: str + :param suite: The "description" attribute of a protocol test suite. + + :type case: str + :param case: The "id" attribute of a specific protocol test case. + + :return: True if the protocol test should be ignored, False otherwise. + :rtype: bool + """ + general_ignore_list = PROTOCOL_TEST_IGNORE_LIST['general'] + general_suites_to_ignore = general_ignore_list.get(test_type, {}).get( + 'suites', [] + ) + general_cases_to_ignore = general_ignore_list.get(test_type, {}).get( + 'cases', [] + ) + if suite in general_suites_to_ignore or case in general_cases_to_ignore: + return True + + protocol_ignore_list = PROTOCOL_TEST_IGNORE_LIST['protocols'].get( + protocol, {} + ) + protocol_suites_to_ignore = protocol_ignore_list.get(test_type, {}).get( + 'suites', [] + ) + protocol_cases_to_ignore = protocol_ignore_list.get(test_type, {}).get( + 'cases', [] + ) + return ( + suite in protocol_suites_to_ignore or case in protocol_cases_to_ignore + )