diff --git a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py index d07c674ff2..80d80856a7 100644 --- a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py +++ b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py @@ -51,7 +51,7 @@ def deploy(self): finished_str = "Accepting HTTP Socket connections at" return wait_for_condition( condition=lambda: finished_str in self.get_logs(), - timeout_seconds=5, + timeout_seconds=30, bail_condition=lambda: self.exited, context=None ) diff --git a/behave_framework/src/minifi_test_framework/steps/core_steps.py b/behave_framework/src/minifi_test_framework/steps/core_steps.py index 1e57b56b98..b717082593 100644 --- a/behave_framework/src/minifi_test_framework/steps/core_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/core_steps.py @@ -51,9 +51,41 @@ def step_impl(context: MinifiTestContext): def step_impl(context: MinifiTestContext, directory: str, size: str): size = humanfriendly.parse_size(size) content = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) + dirs = context.get_or_create_default_minifi_container().dirs + if directory in dirs: + dirs[directory].files[str(uuid.uuid4())] = content + return new_dir = Directory(directory) - new_dir.files["input.txt"] = content - context.get_or_create_default_minifi_container().dirs.append(new_dir) + new_dir.files[str(uuid.uuid4())] = content + dirs.append(new_dir) + + +def __add_directory_with_file_to_container(context: MinifiTestContext, directory: str, file_name: str, content: str, container_name: str): + dirs = context.get_or_create_minifi_container(container_name).dirs + new_content = content.replace("\\n", "\n") + if directory in dirs: + dirs[directory].files[file_name] = new_content + return + new_dir = Directory(directory) + new_dir.files[file_name] = new_content + dirs.append(new_dir) + + +@step('a directory at "{directory}" has a file with the content "{content}" in the "{flow_name}" flow') +@step("a directory at '{directory}' has a file with the content '{content}' in the '{flow_name}' flow") +def step_impl(context: MinifiTestContext, directory: str, content: str, flow_name: str): + __add_directory_with_file_to_container(context, directory, str(uuid.uuid4()), content, flow_name) + + +@step('a directory at "{directory}" has a file with the content "{content}"') +@step("a directory at '{directory}' has a file with the content '{content}'") +def step_impl(context: MinifiTestContext, directory: str, content: str): + context.execute_steps(f'given a directory at "{directory}" has a file with the content "{content}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') + + +@step('a directory at "{directory}" has a file "{file_name}" with the content "{content}"') +def step_impl(context: MinifiTestContext, directory: str, file_name: str, content: str): + __add_directory_with_file_to_container(context, directory, file_name, content, DEFAULT_MINIFI_CONTAINER_NAME) @step('a file with filename "{file_name}" and content "{content}" is present in "{path}"') diff --git a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py index a8c07c3be4..08c6bd04dc 100644 --- a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py @@ -19,7 +19,6 @@ import uuid from behave import given, step -from minifi_test_framework.containers.directory import Directory from minifi_test_framework.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext from minifi_test_framework.minifi.connection import Connection from minifi_test_framework.minifi.controller_service import ControllerService @@ -227,29 +226,6 @@ def step_impl(context: MinifiTestContext, parameter_name: str, parameter_value: parameter_context.parameters.append(Parameter(parameter_name, parameter_value, False)) -@step('a directory at "{directory}" has a file with the content "{content}" in the "{flow_name}" flow') -@step("a directory at '{directory}' has a file with the content '{content}' in the '{flow_name}' flow") -def step_impl(context: MinifiTestContext, directory: str, content: str, flow_name: str): - new_content = content.replace("\\n", "\n") - new_dir = Directory(directory) - new_dir.files["input.txt"] = new_content - context.get_or_create_minifi_container(flow_name).dirs.append(new_dir) - - -@step('a directory at "{directory}" has a file with the content "{content}"') -@step("a directory at '{directory}' has a file with the content '{content}'") -def step_impl(context: MinifiTestContext, directory: str, content: str): - context.execute_steps(f'given a directory at "{directory}" has a file with the content "{content}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') - - -@step('a directory at "{directory}" has a file ("{file_name}") with the content "{content}"') -def step_impl(context: MinifiTestContext, directory: str, file_name: str, content: str): - new_content = content.replace("\\n", "\n") - new_dir = Directory(directory) - new_dir.files[file_name] = new_content - context.get_or_create_default_minifi_container().dirs.append(new_dir) - - @given("these processor properties are set in the \"{minifi_container_name}\" flow") def step_impl(context: MinifiTestContext, minifi_container_name: str): for row in context.table: diff --git a/docker/RunBehaveTests.sh b/docker/RunBehaveTests.sh index 02e31d3816..bcfe346400 100755 --- a/docker/RunBehaveTests.sh +++ b/docker/RunBehaveTests.sh @@ -192,22 +192,6 @@ fi echo "${BEHAVE_OPTS[@]}" -exec \ - behavex "${BEHAVE_OPTS[@]}" \ - "${docker_dir}/../extensions/standard-processors/tests/features" \ - "${docker_dir}/../extensions/aws/tests/features" \ - "${docker_dir}/../extensions/azure/tests/features" \ - "${docker_dir}/../extensions/sql/tests/features" \ - "${docker_dir}/../extensions/llamacpp/tests/features" \ - "${docker_dir}/../extensions/opc/tests/features" \ - "${docker_dir}/../extensions/kafka/tests/features" \ - "${docker_dir}/../extensions/couchbase/tests/features" \ - "${docker_dir}/../extensions/elasticsearch/tests/features" \ - "${docker_dir}/../extensions/splunk/tests/features" \ - "${docker_dir}/../extensions/gcp/tests/features" \ - "${docker_dir}/../extensions/grafana-loki/tests/features" \ - "${docker_dir}/../extensions/lua/tests/features/" \ - "${docker_dir}/../extensions/civetweb/tests/features/" \ - "${docker_dir}/../extensions/mqtt/tests/features/" \ - "${docker_dir}/../extensions/prometheus/tests/features/" \ - "${docker_dir}/../extensions/python/tests/features/" +mapfile -t FEATURE_FILES < <(find "${docker_dir}/../extensions" -type f -name '*.feature') + +behavex "${BEHAVE_OPTS[@]}" "${FEATURE_FILES[@]}" diff --git a/extensions/aws/tests/features/steps/kinesis_server_container.py b/extensions/aws/tests/features/containers/kinesis_server_container.py similarity index 100% rename from extensions/aws/tests/features/steps/kinesis_server_container.py rename to extensions/aws/tests/features/containers/kinesis_server_container.py diff --git a/extensions/aws/tests/features/steps/s3_server_container.py b/extensions/aws/tests/features/containers/s3_server_container.py similarity index 99% rename from extensions/aws/tests/features/steps/s3_server_container.py rename to extensions/aws/tests/features/containers/s3_server_container.py index 01db200301..42ac644545 100644 --- a/extensions/aws/tests/features/steps/s3_server_container.py +++ b/extensions/aws/tests/features/containers/s3_server_container.py @@ -31,7 +31,7 @@ def deploy(self): finished_str = "Started S3MockApplication" return wait_for_condition( condition=lambda: finished_str in self.get_logs(), - timeout_seconds=15, + timeout_seconds=60, bail_condition=lambda: self.exited, context=None) diff --git a/extensions/aws/tests/features/kinesis.feature b/extensions/aws/tests/features/kinesis.feature index 6f903cd561..03823ab78c 100644 --- a/extensions/aws/tests/features/kinesis.feature +++ b/extensions/aws/tests/features/kinesis.feature @@ -22,7 +22,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS Kinesis server Scenario: A MiNiFi instance can send data to AWS Kinesis Given a kinesis server is set up in correspondence with the PutKinesisStream And a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content "Schnappi, das kleine Krokodil" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "Schnappi, das kleine Krokodil" And a PutKinesisStream processor And these processor properties are set | processor name | property name | property value | diff --git a/extensions/aws/tests/features/s3.feature b/extensions/aws/tests/features/s3.feature index b16ea1f259..caebe83fca 100644 --- a/extensions/aws/tests/features/s3.feature +++ b/extensions/aws/tests/features/s3.feature @@ -29,9 +29,9 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And the "failure" relationship of the PutS3Object processor is connected to the PutS3Object And PutFile's success relationship is auto-terminated - And an s3 server is set up in correspondence with the PutS3Object + And the s3 server starts up - When both instances start up + When the MiNiFi instance starts up Then a single file with the content "LH_O#L|FD" And a PutFile processor with the "Directory" property set to "/tmp/output" @@ -43,7 +43,7 @@ Feature: Sending data to MQTT streaming platform using PublishMQTT Scenario Outline: If the MQTT broker does not exist, then no flow files are processed Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a PublishMQTT processor set up to communicate with an MQTT broker instance And the "MQTT Version" property of the PublishMQTT processor is set to "" And a PutFile processor with the "Directory" property set to "/tmp/output" @@ -63,7 +63,7 @@ Feature: Sending data to MQTT streaming platform using PublishMQTT Scenario Outline: Verify delivery of message when MQTT broker is unstable Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a PublishMQTT processor set up to communicate with an MQTT broker instance And the "MQTT Version" property of the PublishMQTT processor is set to "" And a PutFile processor with the "Directory" property set to "/tmp/output" @@ -339,7 +339,7 @@ Feature: Sending data to MQTT streaming platform using PublishMQTT # publishing MQTT client Given a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "publisher-client" flow And the scheduling period of the GetFile processor is set to "120 seconds" in the "publisher-client" flow - And a file with the content "test" is present in "/tmp/input" in the "publisher-client" flow + And a directory at "/tmp/input" has a file with the content "test" in the "publisher-client" flow And a PublishMQTT processor set up to communicate with an MQTT broker instance in the "publisher-client" flow And PublishMQTT is EVENT_DRIVEN in the "publisher-client" flow And the "MQTT Version" property of the PublishMQTT processor is set to "" in the "publisher-client" flow @@ -600,7 +600,7 @@ Feature: Sending data to MQTT streaming platform using PublishMQTT And the "Name of Record Tag" property of the XMLRecordSetWriter controller service is set to "record" And the "Name of Root Tag" property of the XMLRecordSetWriter controller service is set to "root" And a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content '[{"string": "test"}, {"int": 42}]' is present in '/tmp/input' + And a directory at '/tmp/input' has a file with the content '[{"string": "test"}, {"int": 42}]' And a PublishMQTT processor set up to communicate with an MQTT broker instance And PublishMQTT is EVENT_DRIVEN And the "MQTT Version" property of the PublishMQTT processor is set to "3.x AUTO" diff --git a/extensions/mqtt/tests/features/steps/steps.py b/extensions/mqtt/tests/features/steps/steps.py index 4bff1b2a17..39a8f4429a 100644 --- a/extensions/mqtt/tests/features/steps/steps.py +++ b/extensions/mqtt/tests/features/steps/steps.py @@ -24,7 +24,7 @@ from minifi_test_framework.minifi.processor import Processor from minifi_test_framework.core.helpers import wait_for_condition -from mqtt_broker_container import MqttBrokerContainer +from containers.mqtt_broker_container import MqttBrokerContainer @given("a {processor_name} processor set up to communicate with an MQTT broker instance in the \"{container_name}\" flow") diff --git a/extensions/opc/tests/features/steps/opc_ua_server_container.py b/extensions/opc/tests/features/containers/opc_ua_server_container.py similarity index 98% rename from extensions/opc/tests/features/steps/opc_ua_server_container.py rename to extensions/opc/tests/features/containers/opc_ua_server_container.py index 4973b90465..2646bb4033 100644 --- a/extensions/opc/tests/features/steps/opc_ua_server_container.py +++ b/extensions/opc/tests/features/containers/opc_ua_server_container.py @@ -28,6 +28,6 @@ def deploy(self): finished_str = "New DiscoveryUrl added: opc.tcp://" return wait_for_condition( condition=lambda: finished_str in self.get_logs(), - timeout_seconds=15, + timeout_seconds=30, bail_condition=lambda: self.exited, context=None) diff --git a/extensions/opc/tests/features/steps/steps.py b/extensions/opc/tests/features/steps/steps.py index f9db97c8ac..d12ed0338e 100644 --- a/extensions/opc/tests/features/steps/steps.py +++ b/extensions/opc/tests/features/steps/steps.py @@ -22,7 +22,7 @@ from minifi_test_framework.steps import flow_building_steps # noqa: F401 from minifi_test_framework.core.minifi_test_context import MinifiTestContext from minifi_test_framework.core.helpers import wait_for_condition -from opc_ua_server_container import OPCUAServerContainer +from containers.opc_ua_server_container import OPCUAServerContainer @step("an OPC UA server is set up") diff --git a/extensions/prometheus/tests/features/steps/prometheus_container.py b/extensions/prometheus/tests/features/containers/prometheus_container.py similarity index 100% rename from extensions/prometheus/tests/features/steps/prometheus_container.py rename to extensions/prometheus/tests/features/containers/prometheus_container.py diff --git a/extensions/prometheus/tests/features/prometheus.feature b/extensions/prometheus/tests/features/prometheus.feature index 0457db5f59..5083295299 100644 --- a/extensions/prometheus/tests/features/prometheus.feature +++ b/extensions/prometheus/tests/features/prometheus.feature @@ -18,7 +18,7 @@ Feature: MiNiFi can publish metrics to Prometheus server Scenario: Published metrics are scraped by Prometheus server Given a GetFile processor with the name "GetFile1" and the "Input Directory" property set to "/tmp/input" - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a PutFile processor with the "Directory" property set to "/tmp/output" And PutFile is EVENT_DRIVEN And the "success" relationship of the GetFile1 processor is connected to the PutFile @@ -37,7 +37,7 @@ Feature: MiNiFi can publish metrics to Prometheus server Scenario: Published metrics are scraped by Prometheus server through SSL connection Given a GetFile processor with the name "GetFile1" and the "Input Directory" property set to "/tmp/input" - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a PutFile processor with the "Directory" property set to "/tmp/output" And PutFile is EVENT_DRIVEN And the "success" relationship of the GetFile1 processor is connected to the PutFile @@ -58,7 +58,7 @@ Feature: MiNiFi can publish metrics to Prometheus server And a GetFile processor with the name "GetFile2" and the "Input Directory" property set to "/tmp/input" And the "Keep Source File" property of the GetFile1 processor is set to "true" And the "Keep Source File" property of the GetFile2 processor is set to "true" - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a PutFile processor with the "Directory" property set to "/tmp/output" And the "success" relationship of the GetFile1 processor is connected to the PutFile And the "success" relationship of the GetFile2 processor is connected to the PutFile diff --git a/extensions/prometheus/tests/features/steps/steps.py b/extensions/prometheus/tests/features/steps/steps.py index e44165c3d0..954ec93ae4 100644 --- a/extensions/prometheus/tests/features/steps/steps.py +++ b/extensions/prometheus/tests/features/steps/steps.py @@ -20,7 +20,7 @@ from minifi_test_framework.steps import flow_building_steps # noqa: F401 from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext -from prometheus_container import PrometheusContainer +from containers.prometheus_container import PrometheusContainer @step('a Prometheus server is set up') diff --git a/extensions/python/tests/features/python.feature b/extensions/python/tests/features/python.feature index be10ee232b..8f4b3daad3 100644 --- a/extensions/python/tests/features/python.feature +++ b/extensions/python/tests/features/python.feature @@ -84,7 +84,7 @@ Feature: MiNiFi can use python processors in its flows @USE_NIFI_PYTHON_PROCESSORS_WITH_LANGCHAIN Scenario Outline: MiNiFi C++ can use native NiFi python processors Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with filename "test_file.log" and content "test_data" is present in "/tmp/input" + And a directory at "/tmp/input" has a file "test_file.log" with the content "test_data" And a org.apache.nifi.minifi.processors.nifi_python_processors.ParseDocument processor with the name "ParseDocument" And ParseDocument is EVENT_DRIVEN And a org.apache.nifi.minifi.processors.nifi_python_processors.ChunkDocument processor with the name "ChunkDocument" @@ -131,10 +131,10 @@ Feature: MiNiFi can use python processors in its flows Scenario: MiNiFi C++ can use custom relationships in NiFi native python processors Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with filename "test_file.log" and content "test_data_one" is present in "/tmp/input" - And a file with filename "test_file2.log" and content "test_data_two" is present in "/tmp/input" - And a file with filename "test_file3.log" and content "test_data_three" is present in "/tmp/input" - And a file with filename "test_file4.log" and content "test_data_four" is present in "/tmp/input" + And a directory at "/tmp/input" has a file "test_file.log" with the content "test_data_one" + And a directory at "/tmp/input" has a file "test_file2.log" with the content "test_data_two" + And a directory at "/tmp/input" has a file "test_file3.log" with the content "test_data_three" + And a directory at "/tmp/input" has a file "test_file4.log" with the content "test_data_four" And a org.apache.nifi.minifi.processors.nifi_python_processors.RotatingForwarder processor with the name "RotatingForwarder" And RotatingForwarder is EVENT_DRIVEN And a PutFile processor with the "Directory" property set to "/tmp/output" @@ -252,8 +252,8 @@ Feature: MiNiFi can use python processors in its flows Scenario: MiNiFi C++ supports RecordTransform native python processors Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content '{"group": "group1", "name": "John"}\n{"group": "group1", "name": "Jane"}\n{"group": "group2", "name": "Kyle"}\n{"name": "Zoe"}' is present in '/tmp/input' - And a file with the content '{"group": "group1", "name": "Steve"}\n{}' is present in '/tmp/input' + And a directory at '/tmp/input' has a file with the content '{"group": "group1", "name": "John"}\n{"group": "group1", "name": "Jane"}\n{"group": "group2", "name": "Kyle"}\n{"name": "Zoe"}' + And a directory at '/tmp/input' has a file with the content '{"group": "group1", "name": "Steve"}\n{}' And a org.apache.nifi.minifi.processors.nifi_python_processors.SetRecordField processor with the name "SetRecordField" And SetRecordField is EVENT_DRIVEN And the "Record Reader" property of the SetRecordField processor is set to "JsonTreeReader" diff --git a/extensions/splunk/tests/features/steps/splunk_container.py b/extensions/splunk/tests/features/containers/splunk_container.py similarity index 100% rename from extensions/splunk/tests/features/steps/splunk_container.py rename to extensions/splunk/tests/features/containers/splunk_container.py diff --git a/extensions/splunk/tests/features/splunk.feature b/extensions/splunk/tests/features/splunk.feature index 8f929e3156..81a5fe4ab9 100644 --- a/extensions/splunk/tests/features/splunk.feature +++ b/extensions/splunk/tests/features/splunk.feature @@ -20,7 +20,7 @@ Feature: Sending data to Splunk HEC using PutSplunkHTTP Scenario: A MiNiFi instance transfers data to a Splunk HEC Given a Splunk HEC is set up and running And a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content "foobar" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "foobar" And a PutSplunkHTTP processor And PutSplunkHTTP is EVENT_DRIVEN And a QuerySplunkIndexingStatus processor @@ -51,7 +51,7 @@ Feature: Sending data to Splunk HEC using PutSplunkHTTP Scenario: A MiNiFi instance transfers data to a Splunk HEC with SSL enabled Given a Splunk HEC is set up and running And a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content "foobar" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "foobar" And a PutSplunkHTTP processor And PutSplunkHTTP is EVENT_DRIVEN And a QuerySplunkIndexingStatus processor diff --git a/extensions/splunk/tests/features/steps/steps.py b/extensions/splunk/tests/features/steps/steps.py index 16a8d66e91..dda17dabe8 100644 --- a/extensions/splunk/tests/features/steps/steps.py +++ b/extensions/splunk/tests/features/steps/steps.py @@ -20,7 +20,7 @@ from minifi_test_framework.steps import flow_building_steps # noqa: F401 from minifi_test_framework.core.helpers import log_due_to_failure from minifi_test_framework.core.minifi_test_context import MinifiTestContext -from splunk_container import SplunkContainer +from containers.splunk_container import SplunkContainer @step("a Splunk HEC is set up and running") diff --git a/extensions/sql/tests/features/steps/postgress_server_container.py b/extensions/sql/tests/features/containers/postgress_server_container.py similarity index 99% rename from extensions/sql/tests/features/steps/postgress_server_container.py rename to extensions/sql/tests/features/containers/postgress_server_container.py index 96e0eb7ebf..c95f808ef6 100644 --- a/extensions/sql/tests/features/steps/postgress_server_container.py +++ b/extensions/sql/tests/features/containers/postgress_server_container.py @@ -53,7 +53,7 @@ def deploy(self) -> bool: finished_str = "database system is ready to accept connections" return wait_for_condition( condition=lambda: finished_str in self.get_logs(), - timeout_seconds=5, + timeout_seconds=60, bail_condition=lambda: self.exited, context=None) diff --git a/extensions/sql/tests/features/steps/steps.py b/extensions/sql/tests/features/steps/steps.py index db2f555e41..31645138f9 100644 --- a/extensions/sql/tests/features/steps/steps.py +++ b/extensions/sql/tests/features/steps/steps.py @@ -25,7 +25,7 @@ from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.minifi.controller_service import ControllerService from minifi_test_framework.core.minifi_test_context import MinifiTestContext -from postgress_server_container import PostgresContainer +from containers.postgress_server_container import PostgresContainer @given("an ODBCService is setup up for {processor_name} with the name \"{service_name}\"") diff --git a/extensions/standard-processors/tests/features/attributes_to_json.feature b/extensions/standard-processors/tests/features/attributes_to_json.feature index 337847ebb3..7f68c0b63d 100644 --- a/extensions/standard-processors/tests/features/attributes_to_json.feature +++ b/extensions/standard-processors/tests/features/attributes_to_json.feature @@ -18,7 +18,7 @@ Feature: Writing attribute data using AttributesToJSON processor Scenario: Write selected attribute data to file Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a directory at "/tmp/input" has a file ("test_file.log") with the content "test_data" + And a directory at "/tmp/input" has a file "test_file.log" with the content "test_data" And a AttributesToJSON processor with the "Attributes List" property set to "filename,invalid" And the "Destination" property of the AttributesToJSON processor is set to "flowfile-content" And the "Null Value" property of the AttributesToJSON processor is set to "true" diff --git a/extensions/standard-processors/tests/features/steps/diag_slave_container.py b/extensions/standard-processors/tests/features/containers/diag_slave_container.py similarity index 98% rename from extensions/standard-processors/tests/features/steps/diag_slave_container.py rename to extensions/standard-processors/tests/features/containers/diag_slave_container.py index a168c73258..e4827e0f64 100644 --- a/extensions/standard-processors/tests/features/steps/diag_slave_container.py +++ b/extensions/standard-processors/tests/features/containers/diag_slave_container.py @@ -45,7 +45,7 @@ def deploy(self): finished_str = "Server started up successfully." return wait_for_condition( condition=lambda: finished_str in self.get_logs(), - timeout_seconds=5, + timeout_seconds=30, bail_condition=lambda: self.exited, context=None ) diff --git a/extensions/standard-processors/tests/features/steps/syslog_container.py b/extensions/standard-processors/tests/features/containers/syslog_container.py similarity index 100% rename from extensions/standard-processors/tests/features/steps/syslog_container.py rename to extensions/standard-processors/tests/features/containers/syslog_container.py diff --git a/extensions/standard-processors/tests/features/steps/tcp_client_container.py b/extensions/standard-processors/tests/features/containers/tcp_client_container.py similarity index 98% rename from extensions/standard-processors/tests/features/steps/tcp_client_container.py rename to extensions/standard-processors/tests/features/containers/tcp_client_container.py index d3f02d16a4..ddffc7d4fd 100644 --- a/extensions/standard-processors/tests/features/steps/tcp_client_container.py +++ b/extensions/standard-processors/tests/features/containers/tcp_client_container.py @@ -34,7 +34,7 @@ def deploy(self): finished_str = "TCP client container started" return wait_for_condition( condition=lambda: finished_str in self.get_logs(), - timeout_seconds=5, + timeout_seconds=30, bail_condition=lambda: self.exited, context=None ) diff --git a/extensions/standard-processors/tests/features/core_functionality.feature b/extensions/standard-processors/tests/features/core_functionality.feature index 4b5f6b38bf..bab09863ff 100644 --- a/extensions/standard-processors/tests/features/core_functionality.feature +++ b/extensions/standard-processors/tests/features/core_functionality.feature @@ -77,7 +77,7 @@ Feature: Core flow functionalities And a non-sensitive parameter in the flow config called 'FILE_INPUT_PATH' with the value '/tmp/input' in the parameter context 'my-context' And a non-sensitive parameter in the flow config called 'FILE_OUTPUT_UPPER_PATH_ATTR' with the value 'upper_out_path_attr' in the parameter context 'my-context' And a GetFile processor with the "Input Directory" property set to "#{FILE_INPUT_PATH}" - And a file with filename "test_file_name" and content "test content" is present in "/tmp/input" + And a directory at "/tmp/input" has a file "test_file_name" with the content "test content" And a UpdateAttribute processor with the "expr-lang-filename" property set to "#{FILENAME}" And the "is-upper-correct" property of the UpdateAttribute processor is set to "${#{FILENAME_IN_EXPRESSION}:toUpper():equals('TEST_FILE_NAME')}" And the "upper_out_path_attr" property of the UpdateAttribute processor is set to "/TMP/OUTPUT" diff --git a/extensions/standard-processors/tests/features/evaluate_json_path.feature b/extensions/standard-processors/tests/features/evaluate_json_path.feature index d2c503363a..c87f91c424 100644 --- a/extensions/standard-processors/tests/features/evaluate_json_path.feature +++ b/extensions/standard-processors/tests/features/evaluate_json_path.feature @@ -18,7 +18,7 @@ Feature: Writing JSON path query result to attribute or flow file using Evaluate Scenario: Write query result to flow file Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with filename "test_file.json" and content "{"books": [{"title": "The Great Gatsby", "author": "F. Scott Fitzgerald"}, {"title": "1984", "author": "George Orwell"}]}" is present in "/tmp/input" + And a directory at "/tmp/input" has a file "test_file.json" with the content "{"books": [{"title": "The Great Gatsby", "author": "F. Scott Fitzgerald"}, {"title": "1984", "author": "George Orwell"}]}" And a EvaluateJsonPath processor with the "Destination" property set to "flowfile-content" And the "JsonPath" property of the EvaluateJsonPath processor is set to "$.books[*].title" And a PutFile processor with the "Directory" property set to "/tmp/output" @@ -32,7 +32,7 @@ Feature: Writing JSON path query result to attribute or flow file using Evaluate Scenario: Write query result to attributes Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with filename "test_file.json" and content "{"title": "1984", "author": null}" is present in "/tmp/input" + And a directory at "/tmp/input" has a file "test_file.json" with the content "{"title": "1984", "author": null}" And a EvaluateJsonPath processor with the "Destination" property set to "flowfile-attribute" And the "Null Value Representation" property of the EvaluateJsonPath processor is set to "the string 'null'" And the "Path Not Found Behavior" property of the EvaluateJsonPath processor is set to "skip" diff --git a/extensions/standard-processors/tests/features/file_system_operations.feature b/extensions/standard-processors/tests/features/file_system_operations.feature index d7341ddb06..5bec6a6f44 100644 --- a/extensions/standard-processors/tests/features/file_system_operations.feature +++ b/extensions/standard-processors/tests/features/file_system_operations.feature @@ -21,7 +21,7 @@ Feature: File system operations are handled by the GetFile, PutFile, ListFile an Scenario: Get and put operations run in a simple flow Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a PutFile processor with the "Directory" property set to "/tmp/output" And PutFile is EVENT_DRIVEN And the "success" relationship of the GetFile processor is connected to the PutFile @@ -56,7 +56,7 @@ Feature: File system operations are handled by the GetFile, PutFile, ListFile an And PutFile_3's success relationship is auto-terminated And PutFile_3's failure relationship is auto-terminated - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" When the MiNiFi instance starts up Then a single file with the content "test" is placed in the "/tmp/output" directory in less than 10 seconds diff --git a/extensions/standard-processors/tests/features/hashcontent.feature b/extensions/standard-processors/tests/features/hashcontent.feature index 8e24bf62b5..096ba183f8 100644 --- a/extensions/standard-processors/tests/features/hashcontent.feature +++ b/extensions/standard-processors/tests/features/hashcontent.feature @@ -21,7 +21,7 @@ Feature: Hash value is added to Flowfiles by HashContent processor Scenario Outline: HashContent adds hash attribute to flowfiles Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "" And a HashContent processor with the "Hash Attribute" property set to "hash" And the "Hash Algorithm" property of the HashContent processor is set to "" And HashContent is EVENT_DRIVEN @@ -34,10 +34,10 @@ Feature: Hash value is added to Flowfiles by HashContent processor Then the Minifi logs contain the following message: "key:hash value:" in less than 10 seconds Examples: - | content | hash_algorithm | hash_value | - | "apple" | MD5 | 1F3870BE274F6C49B3E31A0C6728957F | - | "test" | SHA1 | A94A8FE5CCB19BA61C4C0873D391E987982FBBD3 | - | "coffee" | SHA256 | 37290D74AC4D186E3A8E5785D259D2EC04FAC91AE28092E7620EC8BC99E830AA | + | content | hash_algorithm | hash_value | + | apple | MD5 | 1F3870BE274F6C49B3E31A0C6728957F | + | test | SHA1 | A94A8FE5CCB19BA61C4C0873D391E987982FBBD3 | + | coffee | SHA256 | 37290D74AC4D186E3A8E5785D259D2EC04FAC91AE28092E7620EC8BC99E830AA | Scenario: HashContent fails for an empty file if 'fail on empty' property is set to true Given a GetFile processor with the "Input Directory" property set to "/tmp/input" @@ -57,7 +57,7 @@ Feature: Hash value is added to Flowfiles by HashContent processor Scenario: HashContent can use MD5 in FIPS mode Given OpenSSL FIPS mode is enabled in MiNiFi And a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content "apple" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "apple" And a HashContent processor with the "Hash Attribute" property set to "hash" And the "Hash Algorithm" property of the HashContent processor is set to "MD5" And HashContent is EVENT_DRIVEN diff --git a/extensions/standard-processors/tests/features/minifi_c2_server.feature b/extensions/standard-processors/tests/features/minifi_c2_server.feature index 29d11295ab..26680eedaf 100644 --- a/extensions/standard-processors/tests/features/minifi_c2_server.feature +++ b/extensions/standard-processors/tests/features/minifi_c2_server.feature @@ -19,7 +19,7 @@ Feature: MiNiFi can communicate with Apache NiFi MiNiFi C2 server Scenario: MiNiFi flow config is updated from MiNiFi C2 server Given a GetFile processor with the name "GetFile1" and the "Input Directory" property set to "/tmp/non-existent" And C2 is enabled in MiNiFi - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a MiNiFi C2 server is set up When all instances start up Then the MiNiFi C2 server logs contain the following message: "acknowledged with a state of FULLY_APPLIED(DONE)" in less than 30 seconds @@ -29,14 +29,14 @@ Feature: MiNiFi can communicate with Apache NiFi MiNiFi C2 server Scenario: MiNiFi can get flow config from C2 server through flow url when it is not available at start Given flow configuration path is set up in flow url property And C2 is enabled in MiNiFi - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a MiNiFi C2 server is started When the MiNiFi instance starts up Then the MiNiFi C2 server logs contain the following message: "acknowledged with a state of FULLY_APPLIED(DONE)" in less than 30 seconds And a single file with the content "test" is placed in the "/tmp/output" directory in less than 10 seconds Scenario: MiNiFi flow config is updated from MiNiFi C2 server through SSL with SSL properties - Given a file with the content "test" is present in "/tmp/input" + Given a directory at "/tmp/input" has a file with the content "test" And a GenerateFlowFile processor And ssl properties are set up for MiNiFi C2 server And a MiNiFi C2 server is set up with SSL @@ -49,7 +49,7 @@ Feature: MiNiFi can communicate with Apache NiFi MiNiFi C2 server And parameter context name is set to 'my-context' And a non-sensitive parameter in the flow config called 'INPUT_DIR' with the value '/tmp/non-existent' in the parameter context 'my-context' And C2 is enabled in MiNiFi - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a MiNiFi C2 server is set up When all instances start up Then the MiNiFi C2 server logs contain the following message: "acknowledged with a state of FULLY_APPLIED(DONE)" in less than 30 seconds diff --git a/extensions/standard-processors/tests/features/minifi_controller.feature b/extensions/standard-processors/tests/features/minifi_controller.feature index b73f4b9d6e..a5eba8bcfd 100644 --- a/extensions/standard-processors/tests/features/minifi_controller.feature +++ b/extensions/standard-processors/tests/features/minifi_controller.feature @@ -19,7 +19,7 @@ Feature: MiNiFi Controller functionalities Scenario: Flow config can be updated through MiNiFi controller Given a GenerateFlowFile processor - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And controller socket properties are set up When all instances start up And MiNiFi config is updated through MiNiFi controller @@ -28,7 +28,7 @@ Feature: MiNiFi Controller functionalities Scenario: A component can be stopped Given a GenerateFlowFile processor - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And controller socket properties are set up When all instances start up And the GenerateFlowFile component is stopped through MiNiFi controller @@ -37,7 +37,7 @@ Feature: MiNiFi Controller functionalities Scenario: If FlowController is stopped all other components are stopped Given a GenerateFlowFile processor - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And controller socket properties are set up When all instances start up And the FlowController component is stopped through MiNiFi controller @@ -46,7 +46,7 @@ Feature: MiNiFi Controller functionalities Scenario: FlowController can be stopped and restarted Given a GenerateFlowFile processor - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And controller socket properties are set up When all instances start up And the FlowController component is stopped through MiNiFi controller @@ -67,14 +67,14 @@ Feature: MiNiFi Controller functionalities Scenario: Manifest can be retrieved Given a GenerateFlowFile processor - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And controller socket properties are set up When all instances start up Then manifest can be retrieved through MiNiFi controller Scenario: Debug bundle can be retrieved Given a GenerateFlowFile processor - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And controller socket properties are set up When all instances start up Then debug bundle can be retrieved through MiNiFi controller diff --git a/extensions/standard-processors/tests/features/s2s.feature b/extensions/standard-processors/tests/features/s2s.feature index 4e56b4affa..b12a5cfc28 100644 --- a/extensions/standard-processors/tests/features/s2s.feature +++ b/extensions/standard-processors/tests/features/s2s.feature @@ -21,7 +21,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol Scenario: A MiNiFi instance produces and transfers data to a NiFi instance via s2s Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a RemoteProcessGroup node with name "RemoteProcessGroup" is opened on "http://nifi-${scenario_id}:8080/nifi" And an input port with name "to_nifi" is created on the RemoteProcessGroup named "RemoteProcessGroup" And the "success" relationship of the GetFile processor is connected to the to_nifi @@ -41,7 +41,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol Scenario: A MiNiFi instance produces and transfers a large data file to a NiFi instance via s2s Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content "this is a very long file we want to send by site-to-site" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "this is a very long file we want to send by site-to-site" And a RemoteProcessGroup node with name "RemoteProcessGroup" is opened on "http://nifi-${scenario_id}:8080/nifi" And an input port with name "to_nifi" is created on the RemoteProcessGroup named "RemoteProcessGroup" And the "success" relationship of the GetFile processor is connected to the to_nifi @@ -104,7 +104,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol Given an ssl context service is set up And a GetFile processor with the "Input Directory" property set to "/tmp/input" And the "Keep Source File" property of the GetFile processor is set to "true" - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a RemoteProcessGroup node with name "RemoteProcessGroup" is opened on "https://nifi-${scenario_id}:8443/nifi" And an input port with name "to_nifi" is created on the RemoteProcessGroup named "RemoteProcessGroup" And the "SSL Context Service" property of the "to_nifi" port in the "RemoteProcessGroup" remote process group is set to "SSLContextService" @@ -126,7 +126,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol Scenario: A MiNiFi instance produces and transfers data to a NiFi instance via s2s using SSL config defined in minifi.properties Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And the "Keep Source File" property of the GetFile processor is set to "true" - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a RemoteProcessGroup node with name "RemoteProcessGroup" is opened on "https://nifi-${scenario_id}:8443/nifi" And an input port with name "to_nifi" is created on the RemoteProcessGroup named "RemoteProcessGroup" And the "success" relationship of the GetFile processor is connected to the to_nifi @@ -147,7 +147,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol Scenario: A MiNiFi instance produces and transfers data to a NiFi instance via s2s using HTTP protocol Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a RemoteProcessGroup node with name "RemoteProcessGroup" is opened on "http://nifi-${scenario_id}:8080/nifi" with transport protocol set to "HTTP" And an input port with name "to_nifi" is created on the RemoteProcessGroup named "RemoteProcessGroup" And the "success" relationship of the GetFile processor is connected to the to_nifi @@ -174,7 +174,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol And PutFile's success relationship is auto-terminated And a NiFi container is set up - And a file with the content "test" is present in "/tmp/input" in the "nifi" flow + And a directory at "/tmp/input" has a file with the content "test" in the "nifi" flow And a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "nifi" flow And a NiFi flow is sending data to an output port named "to-minifi-in-nifi" with the id of the port named "from_nifi" from the RemoteProcessGroup named "RemoteProcessGroup" And in the "nifi" flow the "success" relationship of the GetFile processor is connected to the to-minifi-in-nifi @@ -194,7 +194,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol And PutFile's success relationship is auto-terminated And a NiFi container is set up - And a file with the content "test" is present in "/tmp/input" in the "nifi" flow + And a directory at "/tmp/input" has a file with the content "test" in the "nifi" flow And a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "nifi" flow And a NiFi flow is sending data to an output port named "to-minifi-in-nifi" with the id of the port named "from_nifi" from the RemoteProcessGroup named "RemoteProcessGroup" And in the "nifi" flow the "success" relationship of the GetFile processor is connected to the to-minifi-in-nifi @@ -215,7 +215,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol And PutFile's success relationship is auto-terminated And a NiFi container is set up with SSL enabled - And a file with the content "test" is present in "/tmp/input" in the "nifi" flow + And a directory at "/tmp/input" has a file with the content "test" in the "nifi" flow And a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "nifi" flow And a NiFi flow is sending data to an output port named "to-minifi-in-nifi" with the id of the port named "from_nifi" from the RemoteProcessGroup named "RemoteProcessGroup" And in the "nifi" flow the "success" relationship of the GetFile processor is connected to the to-minifi-in-nifi @@ -236,7 +236,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol And PutFile's success relationship is auto-terminated And a NiFi container is set up with SSL enabled - And a file with the content "test" is present in "/tmp/input" in the "nifi" flow + And a directory at "/tmp/input" has a file with the content "test" in the "nifi" flow And a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "nifi" flow And a NiFi flow is sending data to an output port named "to-minifi-in-nifi" with the id of the port named "from_nifi" from the RemoteProcessGroup named "RemoteProcessGroup" And in the "nifi" flow the "success" relationship of the GetFile processor is connected to the to-minifi-in-nifi @@ -249,7 +249,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol Scenario: A MiNiFi instance produces and transfers data to a NiFi instance via s2s using compression Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with the content "test" is present in "/tmp/input" + And a directory at "/tmp/input" has a file with the content "test" And a RemoteProcessGroup node with name "RemoteProcessGroup" is opened on "http://nifi-${scenario_id}:8080/nifi" And an input port using compression with name "to_nifi" is created on the RemoteProcessGroup named "RemoteProcessGroup" And the "success" relationship of the GetFile processor is connected to the to_nifi @@ -276,7 +276,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol And PutFile's success relationship is auto-terminated And a NiFi container is set up - And a file with the content "test" is present in "/tmp/input" in the "nifi" flow + And a directory at "/tmp/input" has a file with the content "test" in the "nifi" flow And a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "nifi" flow And a NiFi flow is sending data to an output port named "to-minifi-in-nifi" with the id of the port named "from_nifi" from the RemoteProcessGroup named "RemoteProcessGroup" And in the "nifi" flow the "success" relationship of the GetFile processor is connected to the to-minifi-in-nifi @@ -296,7 +296,7 @@ Feature: Sending data from MiNiFi-C++ to NiFi using S2S protocol And PutFile's success relationship is auto-terminated And a NiFi container is set up - And a file with the content "test" is present in "/tmp/input" in the "nifi" flow + And a directory at "/tmp/input" has a file with the content "test" in the "nifi" flow And a GetFile processor with the "Input Directory" property set to "/tmp/input" in the "nifi" flow And a NiFi flow is sending data to an output port named "to-minifi-in-nifi" with the id of the port named "from_nifi" from the RemoteProcessGroup named "RemoteProcessGroup" And in the "nifi" flow the "success" relationship of the GetFile processor is connected to the to-minifi-in-nifi diff --git a/extensions/standard-processors/tests/features/split_json.feature b/extensions/standard-processors/tests/features/split_json.feature index 98ea1fd72b..887d3832af 100644 --- a/extensions/standard-processors/tests/features/split_json.feature +++ b/extensions/standard-processors/tests/features/split_json.feature @@ -18,7 +18,7 @@ Feature: Splitting JSON content using SplitJson processor Scenario: Split multiple query results to separate flow files Given a GetFile processor with the "Input Directory" property set to "/tmp/input" - And a file with filename "test_file.json" and content "{"company": {"departments": [{"name": "Engineering", "employees": ["Alice", "Bob"]}, {"name": "Marketing", "employees": "Dave"}, {"name": "Sales", "employees": null}]}}" is present in "/tmp/input" + And a directory at "/tmp/input" has a file "test_file.json" with the content "{"company": {"departments": [{"name": "Engineering", "employees": ["Alice", "Bob"]}, {"name": "Marketing", "employees": "Dave"}, {"name": "Sales", "employees": null}]}}" And a SplitJson processor with the "JsonPath Expression" property set to "$.company.departments[*].employees" And the "Null Value Representation" property of the SplitJson processor is set to "the string 'null'" And SplitJson is EVENT_DRIVEN diff --git a/extensions/standard-processors/tests/features/steps/steps.py b/extensions/standard-processors/tests/features/steps/steps.py index 85158cb9e4..68729c7424 100644 --- a/extensions/standard-processors/tests/features/steps/steps.py +++ b/extensions/standard-processors/tests/features/steps/steps.py @@ -22,9 +22,9 @@ from minifi_test_framework.steps import core_steps # noqa: F401 from minifi_test_framework.steps import flow_building_steps # noqa: F401 from minifi_test_framework.core.minifi_test_context import MinifiTestContext -from syslog_container import SyslogContainer -from diag_slave_container import DiagSlave -from tcp_client_container import TcpClientContainer +from containers.syslog_container import SyslogContainer +from containers.diag_slave_container import DiagSlave +from containers.tcp_client_container import TcpClientContainer @step("a Syslog client with TCP protocol is setup to send logs to minifi")