feat: Update test harness (copy test files) #1467 (#416)

* feat: Update test harness (copy test files) #1467

Signed-off-by: christian.lutnik <christian.lutnik@dynatrace.com>

* fixup! feat: Update test harness (copy test files) #1467

Signed-off-by: christian.lutnik <christian.lutnik@dynatrace.com>

* fixup! feat: Update test harness (copy test files) #1467

Signed-off-by: christian.lutnik <christian.lutnik@dynatrace.com>

* fixup! feat: Update test harness (copy test files) #1467

Signed-off-by: christian.lutnik <christian.lutnik@dynatrace.com>

* fixup! feat: Update test harness (copy test files) #1467

Signed-off-by: christian.lutnik <christian.lutnik@dynatrace.com>

* fixup! feat: Update test harness (copy test files) #1467

Signed-off-by: christian.lutnik <christian.lutnik@dynatrace.com>

* fixup! feat: Update test harness (copy test files) #1467

Signed-off-by: christian.lutnik <christian.lutnik@dynatrace.com>

* fixup! feat: Update test harness (copy test files) #1467

Signed-off-by: christian.lutnik <christian.lutnik@dynatrace.com>

* fixup! feat: Update test harness (copy test files) #1467

Signed-off-by: christian.lutnik <christian.lutnik@dynatrace.com>

* fixup! feat: Update test harness (copy test files) #1467

Signed-off-by: christian.lutnik <christian.lutnik@dynatrace.com>

---------

Signed-off-by: christian.lutnik <christian.lutnik@dynatrace.com>
Co-authored-by: Anton Grübel <anton.gruebel@gmail.com>

Author: chrfwow

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "test-harness"]
-	path = test-harness
-	url = https://github.com/open-feature/test-harness.git
 [submodule "spec"]
 	path = spec
 	url = https://github.com/open-feature/spec.git

pyproject.toml

@@ -48,8 +48,8 @@ cov = [
     "cov-report",
 ]
 e2e = [
-    "git submodule update --init",
-    "cp test-harness/features/evaluation.feature tests/features/",
+    "git submodule add --force https://github.com/open-feature/spec.git spec",
+    "cp -r spec/specification/assets/gherkin/evaluation.feature tests/features/",
     "behave tests/features/",
     "rm tests/features/*.feature",
 ]

test-harness

@@ -1,5 +1,7 @@
 # flake8: noqa: F811
 
+from time import sleep
+
 from behave import given, then, when
 
 from openfeature.api import get_client, set_provider
@@ -17,15 +19,21 @@
     'the resolved {flag_type} details reason of flag with key "{key}" should be '
     '"{reason}"'
 )
-def step_impl(context, flag_type, key, expected_reason):
+def step_impl_resolved_should_be(context, flag_type, key, expected_reason):
     details: FlagEvaluationDetails = None
     if flag_type == "boolean":
         details = context.boolean_flag_details
         assert expected_reason == details.reason.value
 
 
 @given("a provider is registered with cache disabled")
-def step_impl(context):
+def step_impl_provider_without_cache(context):
+    set_provider(InMemoryProvider(IN_MEMORY_FLAGS))
+    context.client = get_client()
+
+
+@given("a provider is registered")
+def step_impl_provider(context):
     set_provider(InMemoryProvider(IN_MEMORY_FLAGS))
     context.client = get_client()
 
@@ -34,7 +42,7 @@ def step_impl(context):
     'a {flag_type} flag with key "{key}" is evaluated with details and default value '
     '"{default_value}"'
 )
-def step_impl(context, flag_type, key, default_value):
+def step_impl_evaluated_with_details(context, flag_type, key, default_value):
     context.client = get_client()
     if flag_type == "boolean":
         context.boolean_flag_details = context.client.get_boolean_details(
@@ -50,7 +58,9 @@ def step_impl(context, flag_type, key, default_value):
     'a boolean flag with key "{key}" is evaluated with {eval_details} and default '
     'value "{default_value}"'
 )
-def step_impl(context, key, eval_details, default_value):
+def step_impl_bool_evaluated_with_details_and_default(
+    context, key, eval_details, default_value
+):
     client: OpenFeatureClient = context.client
 
     context.boolean_flag_details = client.get_boolean_details(key, default_value)
@@ -60,7 +70,7 @@ def step_impl(context, key, eval_details, default_value):
     'a {flag_type} flag with key "{key}" is evaluated with default value '
     '"{default_value}"'
 )
-def step_impl(context, flag_type, key, default_value):
+def step_impl_evaluated_with_default(context, flag_type, key, default_value):
     client: OpenFeatureClient = context.client
 
     if flag_type == "boolean":
@@ -70,20 +80,20 @@ def step_impl(context, flag_type, key, default_value):
 
 
 @then('the resolved string value should be "{expected_value}"')
-def step_impl(context, expected_value):
+def step_impl_resolved_string_should_be(context, expected_value):
     assert expected_value == context.string_flag_details.value
 
 
 @then('the resolved boolean value should be "{expected_value}"')
-def step_impl(context, expected_value):
+def step_impl_resolved_bool_should_be(context, expected_value):
     assert parse_boolean(expected_value) == context.boolean_flag_details.value
 
 
 @when(
     'an integer flag with key "{key}" is evaluated with details and default value '
     "{default_value:d}"
 )
-def step_impl(context, key, default_value):
+def step_impl_int_evaluated_with_details_and_default(context, key, default_value):
     context.flag_key = key
     context.default_value = default_value
     context.integer_flag_details = context.client.get_integer_details(
@@ -94,7 +104,7 @@ def step_impl(context, key, default_value):
 @when(
     'an integer flag with key "{key}" is evaluated with default value {default_value:d}'
 )
-def step_impl(context, key, default_value):
+def step_impl_int_evaluated_with_default(context, key, default_value):
     context.flag_key = key
     context.default_value = default_value
     context.integer_flag_details = context.client.get_integer_details(
@@ -103,26 +113,26 @@ def step_impl(context, key, default_value):
 
 
 @when('a float flag with key "{key}" is evaluated with default value {default_value:f}')
-def step_impl(context, key, default_value):
+def step_impl_float_evaluated_with_default(context, key, default_value):
     context.flag_key = key
     context.default_value = default_value
     context.float_flag_details = context.client.get_float_details(key, default_value)
 
 
 @when('an object flag with key "{key}" is evaluated with a null default value')
-def step_impl(context, key):
+def step_impl_obj_evaluated_with_default(context, key):
     context.flag_key = key
     context.default_value = None
     context.object_flag_details = context.client.get_object_details(key, None)
 
 
 @then("the resolved integer value should be {expected_value:d}")
-def step_impl(context, expected_value):
+def step_impl_resolved_int_should_be(context, expected_value):
     assert expected_value == context.integer_flag_details.value
 
 
 @then("the resolved float value should be {expected_value:f}")
-def step_impl(context, expected_value):
+def step_impl_resolved_bool_should_be(context, expected_value):
     assert expected_value == context.float_flag_details.value
 
 
@@ -131,7 +141,9 @@ def step_impl(context, expected_value):
     'the resolved boolean details value should be "{expected_value}", the variant '
     'should be "{variant}", and the reason should be "{reason}"'
 )
-def step_impl(context, expected_value, variant, reason):
+def step_impl_resolved_bool_should_be_with_reason(
+    context, expected_value, variant, reason
+):
     assert parse_boolean(expected_value) == context.boolean_flag_details.value
     assert variant == context.boolean_flag_details.variant
     assert reason == context.boolean_flag_details.reason
@@ -141,7 +153,9 @@ def step_impl(context, expected_value, variant, reason):
     'the resolved string details value should be "{expected_value}", the variant '
     'should be "{variant}", and the reason should be "{reason}"'
 )
-def step_impl(context, expected_value, variant, reason):
+def step_impl_resolved_string_should_be_with_reason(
+    context, expected_value, variant, reason
+):
     assert expected_value == context.string_flag_details.value
     assert variant == context.string_flag_details.variant
     assert reason == context.string_flag_details.reason
@@ -151,7 +165,9 @@ def step_impl(context, expected_value, variant, reason):
     'the resolved object value should be contain fields "{field1}", "{field2}", and '
     '"{field3}", with values "{val1}", "{val2}" and {val3}, respectively'
 )
-def step_impl(context, field1, field2, field3, val1, val2, val3):
+def step_impl_resolved_obj_should_contain(
+    context, field1, field2, field3, val1, val2, val3
+):
     value = context.object_flag_details.value
     assert field1 in value
     assert field2 in value
@@ -162,7 +178,7 @@ def step_impl(context, field1, field2, field3, val1, val2, val3):
 
 
 @then('the resolved flag value is "{flag_value}" when the context is empty')
-def step_impl(context, flag_value):
+def step_impl_resolved_is_with_empty_context(context, flag_value):
     context.string_flag_details = context.client.get_boolean_details(
         context.flag_key, context.default_value
     )
@@ -173,13 +189,13 @@ def step_impl(context, flag_value):
     "the reason should indicate an error and the error code should indicate a missing "
     'flag with "{error_code}"'
 )
-def step_impl(context, error_code):
+def step_impl_reason_should_indicate(context, error_code):
     assert context.string_flag_details.reason == Reason.ERROR
     assert context.string_flag_details.error_code == ErrorCode[error_code]
 
 
 @then("the default {flag_type} value should be returned")
-def step_impl(context, flag_type):
+def step_impl_return_default(context, flag_type):
     flag_details = getattr(context, f"{flag_type}_flag_details")
     assert context.default_value == flag_details.value
 
@@ -188,15 +204,15 @@ def step_impl(context, flag_type):
     'a float flag with key "{key}" is evaluated with details and default value '
     "{default_value:f}"
 )
-def step_impl(context, key, default_value):
+def step_impl_float_with_details(context, key, default_value):
     context.float_flag_details = context.client.get_float_details(key, default_value)
 
 
 @then(
     "the resolved float details value should be {expected_value:f}, the variant should "
     'be "{variant}", and the reason should be "{reason}"'
 )
-def step_impl(context, expected_value, variant, reason):
+def step_impl_resolved_float_with_variant(context, expected_value, variant, reason):
     assert expected_value == context.float_flag_details.value
     assert variant == context.float_flag_details.variant
     assert reason == context.float_flag_details.reason
@@ -205,15 +221,15 @@ def step_impl(context, expected_value, variant, reason):
 @when(
     'an object flag with key "{key}" is evaluated with details and a null default value'
 )
-def step_impl(context, key):
+def step_impl_eval_obj(context, key):
     context.object_flag_details = context.client.get_object_details(key, None)
 
 
 @then(
     'the resolved object details value should be contain fields "{field1}", "{field2}",'
     ' and "{field3}", with values "{val1}", "{val2}" and {val3}, respectively'
 )
-def step_impl(context, field1, field2, field3, val1, val2, val3):
+def step_impl_eval_obj_with_fields(context, field1, field2, field3, val1, val2, val3):
     value = context.object_flag_details.value
     assert field1 in value
     assert field2 in value
@@ -224,7 +240,7 @@ def step_impl(context, field1, field2, field3, val1, val2, val3):
 
 
 @then('the variant should be "{variant}", and the reason should be "{reason}"')
-def step_impl(context, variant, reason):
+def step_impl_variant(context, variant, reason):
     assert variant == context.object_flag_details.variant
     assert reason == context.object_flag_details.reason
 
@@ -233,7 +249,7 @@ def step_impl(context, variant, reason):
     'context contains keys "{key1}", "{key2}", "{key3}", "{key4}" with values "{val1}",'
     ' "{val2}", {val3:d}, "{val4}"'
 )
-def step_impl(context, key1, key2, key3, key4, val1, val2, val3, val4):
+def step_impl_context(context, key1, key2, key3, key4, val1, val2, val3, val4):
     context.evaluation_context = EvaluationContext(
         None,
         {
@@ -246,7 +262,7 @@ def step_impl(context, key1, key2, key3, key4, val1, val2, val3, val4):
 
 
 @when('a flag with key "{key}" is evaluated with default value "{default_value}"')
-def step_impl(context, key, default_value):
+def step_impl_flag_with_key_and_default(context, key, default_value):
     context.flag_key = key
     context.default_value = default_value
     context.string_flag_details = context.client.get_string_details(
@@ -255,15 +271,15 @@ def step_impl(context, key, default_value):
 
 
 @then('the resolved string response should be "{expected_value}"')
-def step_impl(context, expected_value):
+def step_impl_reason(context, expected_value):
     assert expected_value == context.string_flag_details.value
 
 
 @when(
     'a non-existent string flag with key "{flag_key}" is evaluated with details and a '
     'default value "{default_value}"'
 )
-def step_impl(context, flag_key, default_value):
+def step_impl_non_existing(context, flag_key, default_value):
     context.flag_key = flag_key
     context.default_value = default_value
     context.string_flag_details = context.client.get_string_details(
@@ -275,7 +291,7 @@ def step_impl(context, flag_key, default_value):
     'a string flag with key "{flag_key}" is evaluated as an integer, with details and a'
     " default value {default_value:d}"
 )
-def step_impl(context, flag_key, default_value):
+def step_impl_string_with_details(context, flag_key, default_value):
     context.flag_key = flag_key
     context.default_value = default_value
     context.integer_flag_details = context.client.get_integer_details(
@@ -287,7 +303,7 @@ def step_impl(context, flag_key, default_value):
     "the reason should indicate an error and the error code should indicate a type "
     'mismatch with "{error_code}"'
 )
-def step_impl(context, error_code):
+def step_impl_type_mismatch(context, error_code):
     assert context.integer_flag_details.reason == Reason.ERROR
     assert context.integer_flag_details.error_code == ErrorCode[error_code]
 
@@ -299,17 +315,17 @@ def step_impl(context, error_code):
     'the flag\'s configuration with key "{key}" is updated to defaultVariant '
     '"{variant}"'
 )
-def step_impl(context, key, variant):
+def step_impl_config_update(context, key, variant):
     raise NotImplementedError("Step definition not implemented yet")
 
 
 @given("sleep for {duration} milliseconds")
-def step_impl(context, duration):
-    raise NotImplementedError("Step definition not implemented yet")
+def step_impl_sleep(context, duration):
+    sleep(float(duration) * 0.001)
 
 
 @then('the resolved string details reason should be "{reason}"')
-def step_impl(context, reason):
+def step_impl_reason_should_be(context, reason):
     raise NotImplementedError("Step definition not implemented yet")