diff --git a/CONTROLLERS.md b/CONTROLLERS.md
index 587470fdde..b097e21a3b 100644
--- a/CONTROLLERS.md
+++ b/CONTROLLERS.md
@@ -27,6 +27,7 @@ limitations under the License.
- [NetworkPrioritizerService](#NetworkPrioritizerService)
- [ODBCService](#ODBCService)
- [PersistentMapStateStorage](#PersistentMapStateStorage)
+- [ProxyConfigurationService](#ProxyConfigurationService)
- [RocksDbStateStorage](#RocksDbStateStorage)
- [SmbConnectionControllerService](#SmbConnectionControllerService)
- [SSLContextService](#SSLContextService)
@@ -244,6 +245,25 @@ In the list below, the names of required properties appear in bold. Any other pr
| **File** | | | Path to a file to store state |
+## ProxyConfigurationService
+
+### Description
+
+Provides a set of configurations for various MiNiFi C++ components to use a proxy server. Currently these properties can only be used for HTTP proxy configuration, no other protocols are supported at this time.
+
+### Properties
+
+In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language.
+
+| Name | Default Value | Allowable Values | Description |
+|-----------------------|---------------|------------------|--------------------------------------------------------------------------------------------|
+| **Proxy Server Host** | | | Proxy server hostname or ip-address. |
+| Proxy Server Port | | | Proxy server port number. |
+| Proxy User Name | | | The name of the proxy client for user authentication. |
+| Proxy User Password | | | The password of the proxy client for user authentication.
**Sensitive Property: true** |
+| Proxy Type | HTTP | HTTP
HTTPS | Proxy type. |
+
+
## RocksDbStateStorage
### Description
diff --git a/PROCESSORS.md b/PROCESSORS.md
index 7851a06afa..521e5aa05c 100644
--- a/PROCESSORS.md
+++ b/PROCESSORS.md
@@ -553,6 +553,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** |
| Storage Account Name | | | The storage account name.
**Supports Expression Language: true** |
| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** |
@@ -585,6 +586,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|-----------------------------------|---------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** |
| Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** |
| File Name | | | The filename in Azure Storage. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** |
@@ -645,7 +647,6 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Access Key | | | AWS account access key
**Supports Expression Language: true** |
| Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** |
| Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey |
@@ -657,7 +658,9 @@ In the list below, the names of required properties appear in bold. Any other pr
| Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** |
| Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** |
| Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. |
| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. |
+| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** |
| Version | | | The Version of the Object to delete
**Supports Expression Language: true** |
@@ -817,6 +820,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** |
| Storage Account Name | | | The storage account name.
**Supports Expression Language: true** |
| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** |
@@ -850,6 +854,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|-----------------------------------|---------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** |
| Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** |
| File Name | | | The filename in Azure Storage. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** |
@@ -1017,7 +1022,6 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Access Key | | | AWS account access key
**Supports Expression Language: true** |
| Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** |
| Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey |
@@ -1029,7 +1033,9 @@ In the list below, the names of required properties appear in bold. Any other pr
| Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** |
| Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** |
| Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. |
| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. |
+| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** |
| Version | | | The Version of the Object to download
**Supports Expression Language: true** |
| **Requester Pays** | false | true
false | If true, indicates that the requester consents to pay any charges associated with retrieving objects from the S3 bucket. This sets the 'x-amz-request-payer' header to 'requester'. |
@@ -1384,6 +1390,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** |
| Storage Account Name | | | The storage account name.
**Supports Expression Language: true** |
| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** |
@@ -1415,6 +1422,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|-----------------------------------|---------------|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** |
| Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** |
| **Recurse Subdirectories** | true | true
false | Indicates whether to list files from subdirectories of the directory |
@@ -1678,7 +1686,6 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Access Key | | | AWS account access key
**Supports Expression Language: true** |
| Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** |
| Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey |
@@ -1690,7 +1697,9 @@ In the list below, the names of required properties appear in bold. Any other pr
| Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** |
| Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** |
| Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. |
| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. |
+| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Delimiter | | | The string used to delimit directories within the bucket. Please consult the AWS documentation for the correct use of this field. |
| Prefix | | | The prefix used to filter the object list. In most cases, it should end with a forward slash ('/'). |
| **Use Versions** | false | true
false | Specifies whether to use S3 versions, if applicable. If false, only the latest version of each object will be returned. |
@@ -2313,6 +2322,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** |
| Storage Account Name | | | The storage account name.
**Supports Expression Language: true** |
| Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** |
@@ -2345,6 +2355,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|-----------------------------------|---------------|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. |
| **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** |
| Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** |
| File Name | | | The filename in Azure Storage. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** |
@@ -2583,7 +2594,6 @@ In the list below, the names of required properties appear in bold. Any other pr
| Name | Default Value | Allowable Values | Description |
|----------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Access Key | | | AWS account access key
**Supports Expression Language: true** |
| Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** |
| Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey |
@@ -2595,7 +2605,9 @@ In the list below, the names of required properties appear in bold. Any other pr
| Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** |
| Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** |
| Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** |
+| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. |
| **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. |
+| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** |
| Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** |
| Content Type | application/octet-stream | | Sets the Content-Type HTTP header indicating the type of content stored in the associated object. The value of this header is a standard MIME type. If no content type is provided the default content type "application/octet-stream" will be used.
**Supports Expression Language: true** |
| **Storage Class** | Standard | Standard
ReducedRedundancy
StandardIA
OnezoneIA
IntelligentTiering
Glacier
DeepArchive
Outposts
GlacierIR
Snow
ExpressOneZone | AWS S3 Storage Class |
diff --git a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
index d07c674ff2..6412022a4f 100644
--- a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
+++ b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py
@@ -17,26 +17,34 @@
from textwrap import dedent
+from OpenSSL import crypto
from minifi_test_framework.containers.container import Container
from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder
from minifi_test_framework.core.helpers import wait_for_condition
from minifi_test_framework.core.minifi_test_context import MinifiTestContext
+from minifi_test_framework.core.ssl_utils import make_server_cert
+from minifi_test_framework.containers.file import File
class HttpProxy(Container):
def __init__(self, test_context: MinifiTestContext):
dockerfile = dedent("""\
- FROM {base_image}
- RUN apt -y update && apt install -y apache2-utils
+ FROM ubuntu:24.04
+ RUN apt -y update && apt install -y squid-openssl apache2-utils
RUN htpasswd -b -c /etc/squid/.squid_users {proxy_username} {proxy_password}
RUN echo 'auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/.squid_users' > /etc/squid/squid.conf && \
echo 'auth_param basic realm proxy' >> /etc/squid/squid.conf && \
echo 'acl authenticated proxy_auth REQUIRED' >> /etc/squid/squid.conf && \
+ echo 'acl SSL_ports port 443' >> /etc/squid/squid.conf && \
+ echo 'acl SSL_ports port 3002' >> /etc/squid/squid.conf && \
+ echo 'acl Safe_ports port 80' >> /etc/squid/squid.conf && \
echo 'http_access allow authenticated' >> /etc/squid/squid.conf && \
echo 'http_port {proxy_port}' >> /etc/squid/squid.conf && \
- echo 'max_filedescriptors 1024' >> /etc/squid/squid.conf
- """.format(base_image='ubuntu/squid:5.2-22.04_beta', proxy_username='admin', proxy_password='test101',
- proxy_port='3128'))
+ echo 'max_filedescriptors 1024' >> /etc/squid/squid.conf && \
+ echo 'https_port {proxy_ssl_port} tls-cert=/etc/squid/certs/squid-cert.pem tls-key=/etc/squid/certs/squid-key.pem' >> /etc/squid/squid.conf
+ ENTRYPOINT ["/bin/sh", "-c", "squid -N -f /etc/squid/squid.conf & tail -F --pid=$! /var/log/squid/cache.log /var/log/squid/access.log"]
+ """.format(proxy_username='admin', proxy_password='test101',
+ proxy_port='3128', proxy_ssl_port='3129'))
builder = DockerImageBuilder(
image_tag="minifi-http-proxy:latest",
@@ -45,6 +53,10 @@ def __init__(self, test_context: MinifiTestContext):
builder.build()
super().__init__("minifi-http-proxy:latest", f"http-proxy-{test_context.scenario_id}", test_context.network)
+ squid_cert, squid_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key)
+
+ self.files.append(File("/etc/squid/certs/squid-cert.pem", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=squid_cert), permissions=0o666))
+ self.files.append(File("/etc/squid/certs/squid-key.pem", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=squid_key), permissions=0o666))
def deploy(self):
super().deploy()
diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_container.py
index be72155e91..c1cdfd4805 100644
--- a/behave_framework/src/minifi_test_framework/containers/minifi_container.py
+++ b/behave_framework/src/minifi_test_framework/containers/minifi_container.py
@@ -39,6 +39,8 @@ def __init__(self, container_name: str, test_context: MinifiTestContext):
minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key)
self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)))
self.files.append(File("/tmp/resources/root_ca.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)))
+ self.files.append(File("/etc/ssl/certs/ca-certificates.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert)))
+
self.files.append(File("/tmp/resources/minifi_client.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_client_cert)))
self.files.append(File("/tmp/resources/minifi_client.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_client_key)))
diff --git a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py
index 474dd5a0af..c1c9ac5f7e 100644
--- a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py
+++ b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py
@@ -239,6 +239,25 @@ def step_impl(context: MinifiTestContext):
processor.add_property(row["property name"], row["property value"])
+@given("a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the \"{container_name}\" flow")
+def step_impl(context: MinifiTestContext, proxy_type: str, container_name: str):
+ controller_service = ControllerService(class_name="ProxyConfigurationService", service_name="ProxyConfigurationService")
+ controller_service.add_property("Proxy Server Host", f"http-proxy-{context.scenario_id}")
+ controller_service.add_property("Proxy User Name", "admin")
+ controller_service.add_property("Proxy User Password", "test101")
+ controller_service.add_property("Proxy Type", proxy_type)
+ if proxy_type.lower() == "http":
+ controller_service.add_property("Proxy Server Port", "3128")
+ else:
+ controller_service.add_property("Proxy Server Port", "3129")
+ context.get_or_create_minifi_container(container_name).flow_definition.controller_services.append(controller_service)
+
+
+@given("a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration")
+def step_impl(context: MinifiTestContext, proxy_type: str):
+ context.execute_steps(f"given a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the \"{DEFAULT_MINIFI_CONTAINER_NAME}\" flow")
+
+
@step("the processors are connected up as described here")
def step_impl(context: MinifiTestContext):
for row in context.table:
diff --git a/extensions/aws/utils/ProxyOptions.h b/core-framework/include/controllers/ProxyConfiguration.h
similarity index 67%
rename from extensions/aws/utils/ProxyOptions.h
rename to core-framework/include/controllers/ProxyConfiguration.h
index 8227c93b06..0090cf45d0 100644
--- a/extensions/aws/utils/ProxyOptions.h
+++ b/core-framework/include/controllers/ProxyConfiguration.h
@@ -1,5 +1,4 @@
/**
- *
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -18,16 +17,18 @@
#pragma once
#include
-#include
+#include
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
-namespace org::apache::nifi::minifi::aws {
+namespace org::apache::nifi::minifi::controllers {
-struct ProxyOptions {
- std::string host;
- uint32_t port = 0;
- std::string username;
- std::string password;
+struct ProxyConfiguration {
+ ProxyType proxy_type;
+ std::string proxy_host;
+ std::optional proxy_port;
+ std::optional proxy_user;
+ std::optional proxy_password;
};
-} // namespace org::apache::nifi::minifi::aws
+} // namespace org::apache::nifi::minifi::controllers
diff --git a/docker/test/integration/features/steps/steps.py b/docker/test/integration/features/steps/steps.py
index 606d5d76dc..91c93dd5b2 100644
--- a/docker/test/integration/features/steps/steps.py
+++ b/docker/test/integration/features/steps/steps.py
@@ -25,6 +25,7 @@
from minifi.controllers.XMLReader import XMLReader
from minifi.controllers.XMLRecordSetWriter import XMLRecordSetWriter
from minifi.controllers.XMLReader import XMLReader
+from minifi.controllers.ProxyConfigurationService import ProxyConfigurationService
from behave import given, then, when
from behave.model_describe import ModelDescriptor
@@ -403,6 +404,18 @@ def step_impl(context):
context.test.acquire_container(context=context, name="http-proxy", engine="http-proxy")
+@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{container_name}\" flow")
+def step_impl(context, container_name):
+ proxy_service = ProxyConfigurationService("ProxyConfigurationService", host=f"http-proxy-{context.feature_id}", port=3128, username="admin", password="test101")
+ container = context.test.acquire_container(context=context, name=container_name)
+ container.add_controller(proxy_service)
+
+
+@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration")
+def step_impl(context):
+ context.execute_steps("given a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{container_name}\" flow".format(container_name="minifi-cpp-flow"))
+
+
# TLS
@given("an ssl context service is set up for {processor_name}")
@given("an ssl context service with a manual CA cert file is set up for {processor_name}")
diff --git a/docker/test/integration/minifi/controllers/ProxyConfigurationService.py b/docker/test/integration/minifi/controllers/ProxyConfigurationService.py
new file mode 100644
index 0000000000..d0e3f457ec
--- /dev/null
+++ b/docker/test/integration/minifi/controllers/ProxyConfigurationService.py
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from ..core.ControllerService import ControllerService
+
+
+class ProxyConfigurationService(ControllerService):
+ def __init__(self, name, host, port=None, username=None, password=None):
+ super(ProxyConfigurationService, self).__init__(name=name)
+
+ self.service_class = 'ProxyConfigurationService'
+
+ self.properties['Proxy Server Host'] = host
+
+ if port is not None:
+ self.properties['Proxy Server Port'] = port
+
+ if username is not None:
+ self.properties['Proxy User Name'] = username
+
+ if password is not None:
+ self.properties['Proxy User Password'] = password
diff --git a/extensions/aws/processors/AwsProcessor.cpp b/extensions/aws/processors/AwsProcessor.cpp
index 6c0a1b19e8..6ff18913be 100644
--- a/extensions/aws/processors/AwsProcessor.cpp
+++ b/extensions/aws/processors/AwsProcessor.cpp
@@ -66,15 +66,25 @@ std::optional AwsProcessor::getAWSCredentials(
return aws_credentials_provider.getAWSCredentials();
}
-aws::ProxyOptions AwsProcessor::getProxy(core::ProcessContext& context, const core::FlowFile* const flow_file) {
- aws::ProxyOptions proxy;
-
- proxy.host = minifi::utils::parseOptionalProperty(context, ProxyHost, flow_file).value_or("");
- proxy.port = gsl::narrow(minifi::utils::parseOptionalU64Property(context, ProxyPort, flow_file).value_or(0));
- proxy.username = minifi::utils::parseOptionalProperty(context, ProxyUsername, flow_file).value_or("");
- proxy.password = minifi::utils::parseOptionalProperty(context, ProxyPassword, flow_file).value_or("");
+minifi::controllers::ProxyConfiguration AwsProcessor::getProxy(core::ProcessContext& context, const core::FlowFile* const flow_file) {
+ minifi::controllers::ProxyConfiguration proxy;
+
+ auto proxy_controller_service = minifi::utils::parseOptionalControllerService(context, ProxyConfigurationService, getUUID());
+ if (proxy_controller_service) {
+ proxy.proxy_type = proxy_controller_service->getProxyType();
+ proxy.proxy_host = proxy_controller_service->getHost();
+ proxy.proxy_port = proxy_controller_service->getPort().value_or(0);
+ proxy.proxy_user = proxy_controller_service->getUsername().value_or("");
+ proxy.proxy_password = proxy_controller_service->getPassword().value_or("");
+ } else {
+ proxy.proxy_type = minifi::utils::parseOptionalEnumProperty(context, ProxyType).value_or(minifi::controllers::ProxyType::HTTP);
+ proxy.proxy_host = minifi::utils::parseOptionalProperty(context, ProxyHost, flow_file).value_or("");
+ proxy.proxy_port = gsl::narrow(minifi::utils::parseOptionalU64Property(context, ProxyPort, flow_file).value_or(0));
+ proxy.proxy_user = minifi::utils::parseOptionalProperty(context, ProxyUsername, flow_file).value_or("");
+ proxy.proxy_password = minifi::utils::parseOptionalProperty(context, ProxyPassword, flow_file).value_or("");
+ }
- if (!proxy.host.empty()) {
+ if (!proxy.proxy_host.empty()) {
logger_->log_info("Proxy for AwsProcessor was set.");
}
return proxy;
diff --git a/extensions/aws/processors/AwsProcessor.h b/extensions/aws/processors/AwsProcessor.h
index 3637fdc343..0a929bab95 100644
--- a/extensions/aws/processors/AwsProcessor.h
+++ b/extensions/aws/processors/AwsProcessor.h
@@ -28,11 +28,12 @@
#include "aws/core/auth/AWSCredentialsProvider.h"
#include "AWSCredentialsProvider.h"
-#include "utils/ProxyOptions.h"
#include "minifi-cpp/core/PropertyDefinition.h"
#include "core/PropertyDefinitionBuilder.h"
#include "minifi-cpp/core/PropertyValidator.h"
#include "core/ProcessorImpl.h"
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
+#include "controllers/ProxyConfiguration.h"
namespace org::apache::nifi::minifi::aws::processors {
@@ -90,7 +91,7 @@ inline constexpr auto REGIONS = std::array{
struct CommonProperties {
Aws::Auth::AWSCredentials credentials;
- aws::ProxyOptions proxy;
+ minifi::controllers::ProxyConfiguration proxy;
std::string endpoint_override_url;
};
@@ -131,6 +132,11 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s
.withValidator(core::StandardPropertyValidators::NON_BLANK_VALIDATOR)
.supportsExpressionLanguage(true)
.build();
+ EXTENSIONAPI static constexpr auto ProxyType = core::PropertyDefinitionBuilder()>::createProperty("Proxy Type")
+ .withDescription("Proxy type")
+ .withDefaultValue(magic_enum::enum_name(minifi::controllers::ProxyType::HTTP))
+ .withAllowedValues(magic_enum::enum_names())
+ .build();
EXTENSIONAPI static constexpr auto ProxyHost = core::PropertyDefinitionBuilder<>::createProperty("Proxy Host")
.withDescription("Proxy host name or IP")
.supportsExpressionLanguage(true)
@@ -148,6 +154,11 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s
.supportsExpressionLanguage(true)
.isSensitive(true)
.build();
+ EXTENSIONAPI static constexpr auto ProxyConfigurationService = core::PropertyDefinitionBuilder<>::createProperty("Proxy Configuration Service")
+ .withDescription("Specifies the Proxy Configuration Controller Service to proxy network requests. When used, "
+ "this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties.")
+ .withAllowedTypes()
+ .build();
EXTENSIONAPI static constexpr auto UseDefaultCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Default Credentials")
.withDescription("If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc.")
.withValidator(core::StandardPropertyValidators::BOOLEAN_VALIDATOR)
@@ -162,10 +173,12 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s
Region,
CommunicationsTimeout,
EndpointOverrideURL,
+ ProxyType,
ProxyHost,
ProxyPort,
ProxyUsername,
ProxyPassword,
+ ProxyConfigurationService,
UseDefaultCredentials
});
@@ -177,7 +190,7 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s
protected:
std::optional getAWSCredentialsFromControllerService(core::ProcessContext& context) const;
std::optional getAWSCredentials(core::ProcessContext& context, const core::FlowFile* flow_file);
- aws::ProxyOptions getProxy(core::ProcessContext& context, const core::FlowFile* const flow_file);
+ minifi::controllers::ProxyConfiguration getProxy(core::ProcessContext& context, const core::FlowFile* const flow_file);
std::optional getCommonELSupportedProperties(core::ProcessContext& context, const core::FlowFile* flow_file);
std::optional client_config_;
diff --git a/extensions/aws/s3/S3Wrapper.h b/extensions/aws/s3/S3Wrapper.h
index ab92e8b4ed..9059b1fec1 100644
--- a/extensions/aws/s3/S3Wrapper.h
+++ b/extensions/aws/s3/S3Wrapper.h
@@ -46,7 +46,7 @@
#include "utils/OptionalUtils.h"
#include "utils/StringUtils.h"
#include "minifi-cpp/utils/gsl.h"
-#include "utils/ProxyOptions.h"
+#include "controllers/ProxyConfiguration.h"
namespace org::apache::nifi::minifi::aws::s3 {
@@ -126,11 +126,12 @@ struct RequestParameters {
Aws::Auth::AWSCredentials credentials;
Aws::Client::ClientConfiguration client_config;
- void setClientConfig(const aws::ProxyOptions& proxy, const std::string& endpoint_override_url) {
- client_config.proxyHost = proxy.host;
- client_config.proxyPort = proxy.port;
- client_config.proxyUserName = proxy.username;
- client_config.proxyPassword = proxy.password;
+ void setClientConfig(const minifi::controllers::ProxyConfiguration& proxy, const std::string& endpoint_override_url) {
+ client_config.proxyHost = proxy.proxy_host;
+ client_config.proxyPort = proxy.proxy_port.value_or(0);
+ client_config.proxyUserName = proxy.proxy_user.value_or("");
+ client_config.proxyPassword = proxy.proxy_password.value_or("");
+ client_config.proxyScheme = proxy.proxy_type == minifi::controllers::ProxyType::HTTPS ? Aws::Http::Scheme::HTTPS : Aws::Http::Scheme::HTTP;
client_config.endpointOverride = endpoint_override_url;
}
};
diff --git a/extensions/aws/tests/DeleteS3ObjectTests.cpp b/extensions/aws/tests/DeleteS3ObjectTests.cpp
index bf06e060a2..de39899f9f 100644
--- a/extensions/aws/tests/DeleteS3ObjectTests.cpp
+++ b/extensions/aws/tests/DeleteS3ObjectTests.cpp
@@ -84,7 +84,12 @@ TEST_CASE_METHOD(DeleteS3ObjectTestsFixture, "Non blank validator tests") {
TEST_CASE_METHOD(DeleteS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]") {
setRequiredProperties();
- setProxy();
+ SECTION("Use proxy configuration service") {
+ setProxy(true);
+ }
+ SECTION("Use processor properties") {
+ setProxy(false);
+ }
test_controller.runSession(plan, true);
checkProxySettings();
}
diff --git a/extensions/aws/tests/FetchS3ObjectTests.cpp b/extensions/aws/tests/FetchS3ObjectTests.cpp
index 5920c13c57..856ccf68f0 100644
--- a/extensions/aws/tests/FetchS3ObjectTests.cpp
+++ b/extensions/aws/tests/FetchS3ObjectTests.cpp
@@ -101,7 +101,12 @@ TEST_CASE_METHOD(FetchS3ObjectTestsFixture, "Non blank validator tests") {
TEST_CASE_METHOD(FetchS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]") {
setRequiredProperties();
- setProxy();
+ SECTION("Use proxy configuration service") {
+ setProxy(true);
+ }
+ SECTION("Use processor properties") {
+ setProxy(false);
+ }
test_controller.runSession(plan, true);
checkProxySettings();
}
diff --git a/extensions/aws/tests/ListS3Tests.cpp b/extensions/aws/tests/ListS3Tests.cpp
index 424e83fd53..752b329449 100644
--- a/extensions/aws/tests/ListS3Tests.cpp
+++ b/extensions/aws/tests/ListS3Tests.cpp
@@ -74,7 +74,12 @@ TEST_CASE_METHOD(ListS3TestsFixture, "Non blank validator tests") {
TEST_CASE_METHOD(ListS3TestsFixture, "Test proxy setting", "[awsS3Proxy]") {
setRequiredProperties();
- setProxy();
+ SECTION("Use proxy configuration service") {
+ setProxy(true);
+ }
+ SECTION("Use processor properties") {
+ setProxy(false);
+ }
test_controller.runSession(plan, true);
checkProxySettings();
}
diff --git a/extensions/aws/tests/PutS3ObjectTests.cpp b/extensions/aws/tests/PutS3ObjectTests.cpp
index a78b0cdd07..75bbba02e7 100644
--- a/extensions/aws/tests/PutS3ObjectTests.cpp
+++ b/extensions/aws/tests/PutS3ObjectTests.cpp
@@ -200,7 +200,12 @@ TEST_CASE_METHOD(PutS3ObjectTestsFixture, "Test multiple user metadata", "[awsS3
TEST_CASE_METHOD(PutS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]") {
setRequiredProperties();
- setProxy();
+ SECTION("Use proxy configuration service") {
+ setProxy(true);
+ }
+ SECTION("Use processor properties") {
+ setProxy(false);
+ }
test_controller.runSession(plan);
checkProxySettings();
}
diff --git a/extensions/aws/tests/S3TestsFixture.h b/extensions/aws/tests/S3TestsFixture.h
index 360e2722f8..77b9f79c38 100644
--- a/extensions/aws/tests/S3TestsFixture.h
+++ b/extensions/aws/tests/S3TestsFixture.h
@@ -96,7 +96,7 @@ class S3TestsFixture {
virtual void setAccesKeyCredentialsInProcessor() = 0;
virtual void setBucket() = 0;
- virtual void setProxy() = 0;
+ virtual void setProxy(bool use_controller_service) = 0;
void setRequiredProperties() {
setAccesKeyCredentialsInProcessor();
@@ -104,6 +104,7 @@ class S3TestsFixture {
}
void checkProxySettings() {
+ REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyScheme == Aws::Http::Scheme::HTTPS);
REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyHost == "host");
REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyPort == 1234);
REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyUserName == "username");
@@ -136,7 +137,8 @@ class FlowProcessorS3TestsFixture : public S3TestsFixture {
auto mock_s3_request_sender = std::make_unique();
this->mock_s3_request_sender_ptr = mock_s3_request_sender.get();
auto uuid = utils::IdGenerator::getIdGenerator()->generate();
- auto impl = std::unique_ptr(new T(core::ProcessorMetadata{.uuid = uuid, .name = "S3Processor", .logger = core::logging::LoggerFactory::getLogger(uuid)}, std::move(mock_s3_request_sender)));
+ auto impl = std::unique_ptr(new T(core::ProcessorMetadata{ // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks)
+ .uuid = uuid, .name = "S3Processor", .logger = core::logging::LoggerFactory::getLogger(uuid)}, std::move(mock_s3_request_sender)));
auto s3_processor_unique_ptr = std::make_unique("S3Processor", uuid, std::move(impl));
this->s3_processor = s3_processor_unique_ptr.get();
@@ -178,15 +180,26 @@ class FlowProcessorS3TestsFixture : public S3TestsFixture {
this->plan->setProperty(this->s3_processor, "Bucket", "${test.bucket}");
}
- void setProxy() override {
- this->plan->setDynamicProperty(update_attribute, "test.proxyHost", "host");
- this->plan->setProperty(this->s3_processor, "Proxy Host", "${test.proxyHost}");
- this->plan->setDynamicProperty(update_attribute, "test.proxyPort", "1234");
- this->plan->setProperty(this->s3_processor, "Proxy Port", "${test.proxyPort}");
- this->plan->setDynamicProperty(update_attribute, "test.proxyUsername", "username");
- this->plan->setProperty(this->s3_processor, "Proxy Username", "${test.proxyUsername}");
- this->plan->setDynamicProperty(update_attribute, "test.proxyPassword", "password");
- this->plan->setProperty(this->s3_processor, "Proxy Password", "${test.proxyPassword}");
+ void setProxy(bool use_controller_service) override {
+ if (use_controller_service) {
+ auto proxy_configuration_service = this->plan->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ this->plan->setProperty(proxy_configuration_service, "Proxy Type", "HTTPS");
+ this->plan->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ this->plan->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ this->plan->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ this->plan->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ this->plan->setProperty(this->s3_processor, "Proxy Configuration Service", "ProxyConfigurationService");
+ } else {
+ this->plan->setProperty(this->s3_processor, "Proxy Type", "HTTPS");
+ this->plan->setDynamicProperty(update_attribute, "test.proxyHost", "host");
+ this->plan->setProperty(this->s3_processor, "Proxy Host", "${test.proxyHost}");
+ this->plan->setDynamicProperty(update_attribute, "test.proxyPort", "1234");
+ this->plan->setProperty(this->s3_processor, "Proxy Port", "${test.proxyPort}");
+ this->plan->setDynamicProperty(update_attribute, "test.proxyUsername", "username");
+ this->plan->setProperty(this->s3_processor, "Proxy Username", "${test.proxyUsername}");
+ this->plan->setDynamicProperty(update_attribute, "test.proxyPassword", "password");
+ this->plan->setProperty(this->s3_processor, "Proxy Password", "${test.proxyPassword}");
+ }
}
protected:
@@ -224,10 +237,21 @@ class FlowProducerS3TestsFixture : public S3TestsFixture {
this->plan->setProperty(this->s3_processor, "Bucket", this->S3_BUCKET);
}
- void setProxy() override {
- this->plan->setProperty(this->s3_processor, "Proxy Host", "host");
- this->plan->setProperty(this->s3_processor, "Proxy Port", "1234");
- this->plan->setProperty(this->s3_processor, "Proxy Username", "username");
- this->plan->setProperty(this->s3_processor, "Proxy Password", "password");
+ void setProxy(bool use_controller_service) override {
+ if (use_controller_service) {
+ auto proxy_configuration_service = this->plan->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ this->plan->setProperty(proxy_configuration_service, "Proxy Type", "HTTPS");
+ this->plan->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ this->plan->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ this->plan->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ this->plan->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ this->plan->setProperty(this->s3_processor, "Proxy Configuration Service", "ProxyConfigurationService");
+ } else {
+ this->plan->setProperty(this->s3_processor, "Proxy Type", "HTTPS");
+ this->plan->setProperty(this->s3_processor, "Proxy Host", "host");
+ this->plan->setProperty(this->s3_processor, "Proxy Port", "1234");
+ this->plan->setProperty(this->s3_processor, "Proxy Username", "username");
+ this->plan->setProperty(this->s3_processor, "Proxy Password", "password");
+ }
}
};
diff --git a/extensions/aws/tests/features/s3.feature b/extensions/aws/tests/features/s3.feature
index b16ea1f259..f49e39456f 100644
--- a/extensions/aws/tests/features/s3.feature
+++ b/extensions/aws/tests/features/s3.feature
@@ -38,16 +38,17 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
And the object content type on the s3 server is "application/octet-stream" and the object metadata matches use metadata
And the Minifi logs contain the following message: "in a single upload" in less than 10 seconds
- Scenario: A MiNiFi instance transfers encoded data through a http proxy to s3
+ Scenario Outline: A MiNiFi instance transfers encoded data through a http proxy to s3
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
And a directory at "/tmp/input" has a file with the content "LH_O#L|FD |
| PutS3Object | Proxy Username | admin |
| PutS3Object | Proxy Password | test101 |
+ | PutS3Object | Proxy Type | |
And a PutFile processor with the "Directory" property set to "/tmp/output"
And the "success" relationship of the GetFile processor is connected to the PutS3Object
And the "success" relationship of the PutS3Object processor is connected to the PutFile
@@ -61,7 +62,36 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
Then a single file with the content "LH_O#L|FD proxy configuration
+
+ And a s3 server is set up in correspondence with the PutS3Object
+ And the http proxy server is set up
+ When all instances start up
+
+ Then a single file with the content "LH_O#L|FD |
| DeleteS3Object | Proxy Username | admin |
| DeleteS3Object | Proxy Password | test101 |
+ | DeleteS3Object | Proxy Type | |
And the processors are connected up as described here
| source name | relationship name | destination name |
| GetFile | success | PutS3Object |
@@ -126,7 +158,41 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
Then a single file with the content "LH_O#L|FD proxy configuration
+
+ And a s3 server is set up in correspondence with the PutS3Object
+ And the http proxy server is set up
+
+ When all instances start up
+
+ Then a single file with the content "LH_O#L|FD |
| FetchS3Object | Proxy Username | admin |
| FetchS3Object | Proxy Password | test101 |
+ | FetchS3Object | Proxy Type | |
And a PutFile processor with the "Directory" property set to "/tmp/output"
And the processors are connected up as described here
| source name | relationship name | destination name |
@@ -177,7 +244,41 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
When all instances start up
Then a single file with the content "test" is placed in the "/tmp/output" directory in less than 20 seconds
- And no errors were generated on the http-proxy regarding "http://s3-server-s3-6:9090/test_bucket/test_object_key"
+ And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket/test_object_key"
+
+ Examples: Proxy Type
+ | proxy type | proxy port |
+ | HTTP | 3128 |
+ | HTTPS | 3129 |
+
+ Scenario Outline: A MiNiFi instance can download s3 bucket objects via a http-proxy using proxy configuration service
+ Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
+ And a file with the content "test" is present in "/tmp/input"
+ And a PutS3Object processor set up to communicate with an s3 server
+ And the "success" relationship of the GetFile processor is connected to the PutS3Object
+
+ Given a GenerateFlowFile processor with the "File Size" property set to "1 kB"
+ And a FetchS3Object processor set up to communicate with the same s3 server
+ And the "Proxy Configuration Service" property of the FetchS3Object processor is set to "ProxyConfigurationService"
+ And a PutFile processor with the "Directory" property set to "/tmp/output"
+ And the processors are connected up as described here
+ | source name | relationship name | destination name |
+ | GenerateFlowFile | success | FetchS3Object |
+ | FetchS3Object | success | PutFile |
+ And a ProxyConfigurationService controller service is set up with proxy configuration
+
+ And a s3 server is set up in correspondence with the PutS3Object
+ And the http proxy server is set up
+
+ When all instances start up
+
+ Then a single file with the content "test" is placed in the "/tmp/output" directory in less than 60 seconds
+ And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket/test_object_key"
+
+ Examples: Proxy Type
+ | proxy type |
+ | HTTP |
+ | HTTPS |
Scenario: A MiNiFi instance can list an S3 bucket directly
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
@@ -199,7 +300,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
Then 2 files are placed in the "/tmp/output" directory in less than 20 seconds
- Scenario: A MiNiFi instance can list an S3 bucket objects via a http-proxy
+ Scenario Outline: A MiNiFi instance can list an S3 bucket objects via a http-proxy
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
And a directory at "/tmp/input" has a file with the content "test"
And a PutS3Object processor set up to communicate with an s3 server
@@ -209,9 +310,10 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
And these processor properties are set
| processor name | property name | property value |
| ListS3 | Proxy Host | http-proxy-${scenario_id} |
- | ListS3 | Proxy Port | 3128 |
+ | ListS3 | Proxy Port | |
| ListS3 | Proxy Username | admin |
| ListS3 | Proxy Password | test101 |
+ | ListS3 | Proxy Type | |
And a PutFile processor with the "Directory" property set to "/tmp/output"
And the "success" relationship of the ListS3 processor is connected to the PutFile
@@ -221,7 +323,37 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
When all instances start up
Then 1 file is placed in the "/tmp/output" directory in less than 20 seconds
- And no errors were generated on the http-proxy regarding "http://s3-server-s3-8:9090/test_bucket"
+ And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket"
+
+ Examples: Proxy Type
+ | proxy port | proxy type |
+ | 3128 | HTTP |
+ | 3129 | HTTPS |
+
+ Scenario Outline: A MiNiFi instance can list an S3 bucket objects via a http-proxy using proxy configuration service
+ Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
+ And a file with the content "test" is present in "/tmp/input"
+ And a PutS3Object processor set up to communicate with an s3 server
+ And the "success" relationship of the GetFile processor is connected to the PutS3Object
+
+ Given a ListS3 processor set up to communicate with the same s3 server
+ And the "Proxy Configuration Service" property of the ListS3 processor is set to "ProxyConfigurationService"
+ And a PutFile processor with the "Directory" property set to "/tmp/output"
+ And the "success" relationship of the ListS3 processor is connected to the PutFile
+ And a ProxyConfigurationService controller service is set up with proxy configuration
+
+ And a s3 server is set up in correspondence with the PutS3Object
+ And the http proxy server is set up
+
+ When all instances start up
+
+ Then 1 file is placed in the "/tmp/output" directory in less than 120 seconds
+ And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket"
+
+ Examples: Proxy Type
+ | proxy type |
+ | HTTP |
+ | HTTPS |
Scenario: A MiNiFi instance transfers data in multiple parts to s3
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
@@ -240,18 +372,19 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
And the object on the s3 server is present and matches the original hash
And the Minifi logs contain the following message: "passes the multipart threshold, uploading it in multiple parts" in less than 10 seconds
- Scenario: A MiNiFi instance can use multipart upload through http proxy to s3
+ Scenario Outline: A MiNiFi instance can use multipart upload through http proxy to s3
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
And there is a 6MB file at the "/tmp/input" directory and we keep track of the hash of that
And a PutS3Object processor set up to communicate with an s3 server
And the "Multipart Threshold" property of the PutS3Object processor is set to "5 MB"
And the "Multipart Part Size" property of the PutS3Object processor is set to "5 MB"
And these processor properties are set
- | processor name | property name | property value |
+ | processor name | property name | property value |
| PutS3Object | Proxy Host | http-proxy-${scenario_id} |
- | PutS3Object | Proxy Port | 3128 |
- | PutS3Object | Proxy Username | admin |
- | PutS3Object | Proxy Password | test101 |
+ | PutS3Object | Proxy Port | |
+ | PutS3Object | Proxy Username | admin |
+ | PutS3Object | Proxy Password | test101 |
+ | PutS3Object | Proxy Type | |
And the "success" relationship of the GetFile processor is connected to the PutS3Object
And a PutFile processor with the "Directory" property set to "/tmp/output"
And the "success" relationship of the PutS3Object processor is connected to the PutFile
@@ -263,4 +396,9 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
Then 1 file is placed in the "/tmp/output" directory in less than 20 seconds
And the object on the s3 server is present and matches the original hash
And the Minifi logs contain the following message: "passes the multipart threshold, uploading it in multiple parts" in less than 10 seconds
- And no errors were generated on the http-proxy regarding "http://s3-server-s3-10:9090/test_bucket/test_object_key"
+ And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket/test_object_key"
+
+ Examples: Proxy Type
+ | proxy port | proxy type |
+ | 3128 | HTTP |
+ | 3129 | HTTPS |
diff --git a/extensions/aws/tests/features/steps/s3_server_container.py b/extensions/aws/tests/features/steps/s3_server_container.py
index 01db200301..42ac644545 100644
--- a/extensions/aws/tests/features/steps/s3_server_container.py
+++ b/extensions/aws/tests/features/steps/s3_server_container.py
@@ -31,7 +31,7 @@ def deploy(self):
finished_str = "Started S3MockApplication"
return wait_for_condition(
condition=lambda: finished_str in self.get_logs(),
- timeout_seconds=15,
+ timeout_seconds=60,
bail_condition=lambda: self.exited,
context=None)
diff --git a/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp b/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp
index 25355e7044..3bbdaf600e 100644
--- a/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp
+++ b/extensions/azure/processors/AzureBlobStorageProcessorBase.cpp
@@ -25,7 +25,8 @@
namespace org::apache::nifi::minifi::azure::processors {
-void AzureBlobStorageProcessorBase::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) {
+void AzureBlobStorageProcessorBase::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) {
+ AzureStorageProcessorBase::onSchedule(context, session_factory);
if (!context.hasNonEmptyProperty(ContainerName.name)) {
throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Container Name property missing or invalid");
}
@@ -105,6 +106,8 @@ bool AzureBlobStorageProcessorBase::setCommonStorageParameters(
return false;
}
+ params.proxy_configuration = proxy_configuration_;
+
return true;
}
diff --git a/extensions/azure/processors/AzureDataLakeStorageProcessorBase.cpp b/extensions/azure/processors/AzureDataLakeStorageProcessorBase.cpp
index fde8462202..ec404e1dc4 100644
--- a/extensions/azure/processors/AzureDataLakeStorageProcessorBase.cpp
+++ b/extensions/azure/processors/AzureDataLakeStorageProcessorBase.cpp
@@ -25,7 +25,8 @@
namespace org::apache::nifi::minifi::azure::processors {
-void AzureDataLakeStorageProcessorBase::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) {
+void AzureDataLakeStorageProcessorBase::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) {
+ AzureStorageProcessorBase::onSchedule(context, session_factory);
std::optional credentials;
std::tie(std::ignore, credentials) = getCredentialsFromControllerService(context);
if (!credentials) {
@@ -51,6 +52,7 @@ bool AzureDataLakeStorageProcessorBase::setCommonParameters(storage::AzureDataLa
}
params.directory_name = context.getProperty(DirectoryName, flow_file).value_or("");
+ params.proxy_configuration = proxy_configuration_;
return true;
}
diff --git a/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h b/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h
index dc2f55f3e2..88de0b1446 100644
--- a/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h
+++ b/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h
@@ -56,7 +56,7 @@ class AzureDataLakeStorageProcessorBase : public AzureStorageProcessorBase {
~AzureDataLakeStorageProcessorBase() override = default;
- void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& sessionFactory) override;
+ void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override;
protected:
explicit AzureDataLakeStorageProcessorBase(core::ProcessorMetadata metadata, std::unique_ptr data_lake_storage_client)
diff --git a/extensions/azure/processors/AzureStorageProcessorBase.cpp b/extensions/azure/processors/AzureStorageProcessorBase.cpp
index 7795a4786b..f4f1d8d4cc 100644
--- a/extensions/azure/processors/AzureStorageProcessorBase.cpp
+++ b/extensions/azure/processors/AzureStorageProcessorBase.cpp
@@ -25,9 +25,24 @@
#include "minifi-cpp/core/ProcessContext.h"
#include "controllerservices/AzureStorageCredentialsService.h"
+#include "utils/ProcessorConfigUtils.h"
namespace org::apache::nifi::minifi::azure::processors {
+void AzureStorageProcessorBase::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) {
+ auto proxy_controller_service = minifi::utils::parseOptionalControllerService(context, ProxyConfigurationService, getUUID());
+ if (proxy_controller_service) {
+ logger_->log_debug("Proxy configuration is set for Azure Storage processor");
+ proxy_configuration_ = minifi::controllers::ProxyConfiguration{
+ .proxy_type = proxy_controller_service->getProxyType(),
+ .proxy_host = proxy_controller_service->getHost(),
+ .proxy_port = proxy_controller_service->getPort(),
+ .proxy_user = proxy_controller_service->getUsername(),
+ .proxy_password = proxy_controller_service->getPassword()
+ };
+ }
+}
+
std::tuple> AzureStorageProcessorBase::getCredentialsFromControllerService(
core::ProcessContext &context) const {
std::string service_name = context.getProperty(AzureStorageCredentialsService).value_or("");
diff --git a/extensions/azure/processors/AzureStorageProcessorBase.h b/extensions/azure/processors/AzureStorageProcessorBase.h
index 8f98f72a0c..ca86dc58f1 100644
--- a/extensions/azure/processors/AzureStorageProcessorBase.h
+++ b/extensions/azure/processors/AzureStorageProcessorBase.h
@@ -32,6 +32,8 @@
#include "core/ProcessorImpl.h"
#include "minifi-cpp/core/logging/Logger.h"
#include "storage/AzureStorageCredentials.h"
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
+#include "controllers/ProxyConfiguration.h"
namespace org::apache::nifi::minifi::azure::processors {
@@ -40,10 +42,16 @@ class AzureStorageProcessorBase : public core::ProcessorImpl {
EXTENSIONAPI static constexpr auto AzureStorageCredentialsService = core::PropertyDefinitionBuilder<>::createProperty("Azure Storage Credentials Service")
.withDescription("Name of the Azure Storage Credentials Service used to retrieve the connection string from.")
.build();
- EXTENSIONAPI static constexpr auto Properties = std::to_array({AzureStorageCredentialsService});
+ EXTENSIONAPI static constexpr auto ProxyConfigurationService = core::PropertyDefinitionBuilder<>::createProperty("Proxy Configuration Service")
+ .withDescription("Specifies the Proxy Configuration Controller Service to proxy network requests.")
+ .withAllowedTypes()
+ .build();
+ EXTENSIONAPI static constexpr auto Properties = std::to_array({AzureStorageCredentialsService, ProxyConfigurationService});
using ProcessorImpl::ProcessorImpl;
+ void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override;
+
protected:
enum class GetCredentialsFromControllerResult {
OK,
@@ -52,6 +60,9 @@ class AzureStorageProcessorBase : public core::ProcessorImpl {
};
std::tuple> getCredentialsFromControllerService(core::ProcessContext &context) const;
+
+ protected:
+ std::optional proxy_configuration_;
};
} // namespace org::apache::nifi::minifi::azure::processors
diff --git a/extensions/azure/storage/AzureBlobStorageClient.cpp b/extensions/azure/storage/AzureBlobStorageClient.cpp
index 837efa4ac8..c9327ec663 100644
--- a/extensions/azure/storage/AzureBlobStorageClient.cpp
+++ b/extensions/azure/storage/AzureBlobStorageClient.cpp
@@ -28,6 +28,7 @@
#include "utils/AzureSdkLogger.h"
#include "utils/span.h"
#include "io/InputStream.h"
+#include "utils/RegexUtils.h"
namespace org::apache::nifi::minifi::azure::storage {
@@ -56,34 +57,56 @@ AzureBlobStorageClient::AzureBlobStorageClient() {
utils::AzureSdkLogger::initialize();
}
-Azure::Storage::Blobs::BlobContainerClient AzureBlobStorageClient::createClient(const AzureStorageCredentials &credentials, const std::string &container_name) {
+Azure::Storage::Blobs::BlobContainerClient AzureBlobStorageClient::createClient(const AzureStorageCredentials &credentials, const std::string &container_name,
+ const std::optional& proxy_configuration) {
+ Azure::Storage::Blobs::BlobClientOptions client_options;
+
+ if (proxy_configuration) {
+ std::string protocol_prefix;
+ if (!minifi::utils::regexMatch(proxy_configuration->proxy_host, minifi::utils::Regex("^https?:\\/\\/"))) {
+ if (proxy_configuration->proxy_type == controllers::ProxyType::HTTP) {
+ protocol_prefix = "http://";
+ } else if (proxy_configuration->proxy_type == controllers::ProxyType::HTTPS) {
+ protocol_prefix = "https://";
+ }
+ }
+ client_options.Transport.HttpProxy = protocol_prefix + proxy_configuration->proxy_host + (proxy_configuration->proxy_port ? (":" + std::to_string(*proxy_configuration->proxy_port)) : "");
+
+ if (proxy_configuration->proxy_user) {
+ client_options.Transport.ProxyUserName = *proxy_configuration->proxy_user;
+ }
+ if (proxy_configuration->proxy_password) {
+ client_options.Transport.ProxyPassword = *proxy_configuration->proxy_password;
+ }
+ }
+
if (credentials.getCredentialConfigurationStrategy() == CredentialConfigurationStrategyOption::FromProperties) {
- return Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString(credentials.buildConnectionString(), container_name);
+ return Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString(credentials.buildConnectionString(), container_name, client_options);
}
auto storage_client = Azure::Storage::Blobs::BlobServiceClient("https://" + credentials.getStorageAccountName() + ".blob." + credentials.getEndpointSuffix(),
- credentials.createAzureTokenCredential());
+ credentials.createAzureTokenCredential(), client_options);
return storage_client.GetBlobContainerClient(container_name);
}
bool AzureBlobStorageClient::createContainerIfNotExists(const PutAzureBlobStorageParameters& params) {
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
return container_client.CreateIfNotExists().Value.Created;
}
Azure::Storage::Blobs::Models::UploadBlockBlobResult AzureBlobStorageClient::uploadBlob(const PutAzureBlobStorageParameters& params, std::span buffer) {
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
auto blob_client = container_client.GetBlockBlobClient(params.blob_name);
return blob_client.UploadFrom(reinterpret_cast(buffer.data()), buffer.size()).Value;
}
std::string AzureBlobStorageClient::getUrl(const AzureBlobStorageParameters& params) {
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
return container_client.GetUrl();
}
bool AzureBlobStorageClient::deleteBlob(const DeleteAzureBlobStorageParameters& params) {
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
Azure::Storage::Blobs::DeleteBlobOptions delete_options;
if (params.optional_deletion == OptionalDeletion::INCLUDE_SNAPSHOTS) {
delete_options.DeleteSnapshots = Azure::Storage::Blobs::Models::DeleteSnapshotsOption::IncludeSnapshots;
@@ -95,7 +118,7 @@ bool AzureBlobStorageClient::deleteBlob(const DeleteAzureBlobStorageParameters&
}
std::unique_ptr AzureBlobStorageClient::fetchBlob(const FetchAzureBlobStorageParameters& params) {
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
auto blob_client = container_client.GetBlobClient(params.blob_name);
Azure::Storage::Blobs::DownloadBlobOptions options;
if (params.range_start || params.range_length) {
@@ -115,7 +138,7 @@ std::unique_ptr AzureBlobStorageClient::fetchBlob(const FetchAz
std::vector AzureBlobStorageClient::listContainer(const ListAzureBlobStorageParameters& params) {
std::vector result;
- auto container_client = createClient(params.credentials, params.container_name);
+ auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration);
Azure::Storage::Blobs::ListBlobsOptions options;
options.Prefix = params.prefix;
for (auto page_result = container_client.ListBlobs(options); page_result.HasPage(); page_result.MoveToNextPage()) {
diff --git a/extensions/azure/storage/AzureBlobStorageClient.h b/extensions/azure/storage/AzureBlobStorageClient.h
index 2b67c76356..ec3d62eea8 100644
--- a/extensions/azure/storage/AzureBlobStorageClient.h
+++ b/extensions/azure/storage/AzureBlobStorageClient.h
@@ -43,7 +43,8 @@ class AzureBlobStorageClient : public BlobStorageClient {
std::vector listContainer(const ListAzureBlobStorageParameters& params) override;
private:
- static Azure::Storage::Blobs::BlobContainerClient createClient(const AzureStorageCredentials& credentials, const std::string &container_name);
+ static Azure::Storage::Blobs::BlobContainerClient createClient(const AzureStorageCredentials& credentials, const std::string &container_name,
+ const std::optional& proxy_configuration);
std::shared_ptr logger_{core::logging::LoggerFactory::getLogger()};
};
diff --git a/extensions/azure/storage/AzureDataLakeStorageClient.cpp b/extensions/azure/storage/AzureDataLakeStorageClient.cpp
index d7618818ed..1a89c82d19 100644
--- a/extensions/azure/storage/AzureDataLakeStorageClient.cpp
+++ b/extensions/azure/storage/AzureDataLakeStorageClient.cpp
@@ -35,13 +35,29 @@ AzureDataLakeStorageClient::AzureDataLakeStorageClient() {
utils::AzureSdkLogger::initialize();
}
-std::unique_ptr AzureDataLakeStorageClient::createClient(
- const AzureStorageCredentials& credentials, const std::string& file_system_name, std::optional number_of_retries) {
+std::unique_ptr AzureDataLakeStorageClient::createClient(const AzureStorageCredentials& credentials,
+ const std::string& file_system_name, std::optional number_of_retries, const std::optional& proxy_configuration) {
Azure::Storage::Files::DataLake::DataLakeClientOptions options;
if (number_of_retries) {
options.Retry.MaxRetries = gsl::narrow(*number_of_retries);
}
+ if (proxy_configuration) {
+ std::string protocol_prefix;
+ if (proxy_configuration->proxy_type == controllers::ProxyType::HTTP && !minifi::utils::string::startsWith(proxy_configuration->proxy_host, "http://")) {
+ protocol_prefix = "http://";
+ } else if (proxy_configuration->proxy_type == controllers::ProxyType::HTTPS && !minifi::utils::string::startsWith(proxy_configuration->proxy_host, "https://")) {
+ protocol_prefix = "https://";
+ }
+ options.Transport.HttpProxy = protocol_prefix + proxy_configuration->proxy_host + (proxy_configuration->proxy_port ? (":" + std::to_string(*proxy_configuration->proxy_port)) : "");
+ if (proxy_configuration->proxy_user) {
+ options.Transport.ProxyUserName = *proxy_configuration->proxy_user;
+ }
+ if (proxy_configuration->proxy_password) {
+ options.Transport.ProxyPassword = *proxy_configuration->proxy_password;
+ }
+ }
+
if (credentials.getCredentialConfigurationStrategy() == CredentialConfigurationStrategyOption::FromProperties) {
return std::make_unique(
Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString(credentials.buildConnectionString(), file_system_name, options));
@@ -53,7 +69,7 @@ std::unique_ptr Azure
}
Azure::Storage::Files::DataLake::DataLakeDirectoryClient AzureDataLakeStorageClient::getDirectoryClient(const AzureDataLakeStorageParameters& params) {
- auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries);
+ auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries, params.proxy_configuration);
return client->GetDirectoryClient(params.directory_name);
}
@@ -104,7 +120,7 @@ std::unique_ptr AzureDataLakeStorageClient::fetchFile(const Fet
std::vector AzureDataLakeStorageClient::listDirectory(const ListAzureDataLakeStorageParameters& params) {
std::vector result;
if (params.directory_name.empty()) {
- auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries);
+ auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries, params.proxy_configuration);
for (auto page_result = client->ListPaths(params.recurse_subdirectories); page_result.HasPage(); page_result.MoveToNextPage()) {
result.insert(result.end(), page_result.Paths.begin(), page_result.Paths.end());
}
diff --git a/extensions/azure/storage/AzureDataLakeStorageClient.h b/extensions/azure/storage/AzureDataLakeStorageClient.h
index 9c9615869c..b9ab9a27b0 100644
--- a/extensions/azure/storage/AzureDataLakeStorageClient.h
+++ b/extensions/azure/storage/AzureDataLakeStorageClient.h
@@ -93,8 +93,8 @@ class AzureDataLakeStorageClient : public DataLakeStorageClient {
Azure::Storage::Files::DataLake::Models::DownloadFileResult result_;
};
- static std::unique_ptr createClient(
- const AzureStorageCredentials& credentials, const std::string& file_system_name, std::optional number_of_retries);
+ static std::unique_ptr createClient(const AzureStorageCredentials& credentials,
+ const std::string& file_system_name, std::optional number_of_retries, const std::optional& proxy_configuration);
static Azure::Storage::Files::DataLake::DataLakeDirectoryClient getDirectoryClient(const AzureDataLakeStorageParameters& params);
static Azure::Storage::Files::DataLake::DataLakeFileClient getFileClient(const AzureDataLakeStorageFileOperationParameters& params);
diff --git a/extensions/azure/storage/BlobStorageClient.h b/extensions/azure/storage/BlobStorageClient.h
index 007b3b86f3..4b1b0c3684 100644
--- a/extensions/azure/storage/BlobStorageClient.h
+++ b/extensions/azure/storage/BlobStorageClient.h
@@ -30,6 +30,8 @@
#include "minifi-cpp/utils/gsl.h"
#include "utils/Enum.h"
#include "minifi-cpp/io/InputStream.h"
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
+#include "controllers/ProxyConfiguration.h"
namespace org::apache::nifi::minifi::azure::storage {
@@ -62,6 +64,7 @@ namespace org::apache::nifi::minifi::azure::storage {
struct AzureBlobStorageParameters {
AzureStorageCredentials credentials;
std::string container_name;
+ std::optional proxy_configuration;
};
struct AzureBlobStorageBlobOperationParameters : public AzureBlobStorageParameters {
diff --git a/extensions/azure/storage/DataLakeStorageClient.h b/extensions/azure/storage/DataLakeStorageClient.h
index 06445a2e8d..5347f92850 100644
--- a/extensions/azure/storage/DataLakeStorageClient.h
+++ b/extensions/azure/storage/DataLakeStorageClient.h
@@ -31,6 +31,8 @@
#include "azure/storage/files/datalake/datalake_responses.hpp"
#include "utils/Enum.h"
#include "utils/RegexUtils.h"
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
+#include "controllers/ProxyConfiguration.h"
namespace org::apache::nifi::minifi::azure::storage {
@@ -39,6 +41,7 @@ struct AzureDataLakeStorageParameters {
std::string file_system_name;
std::string directory_name;
std::optional number_of_retries;
+ std::optional proxy_configuration;
};
struct AzureDataLakeStorageFileOperationParameters : public AzureDataLakeStorageParameters {
diff --git a/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp b/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp
index b554021bbe..7052b20592 100644
--- a/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp
+++ b/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp
@@ -339,4 +339,30 @@ TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test Azure blob delete wit
REQUIRE(failed_flowfiles[0] == TEST_DATA);
}
+TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test Azure blob delete using proxy", "[azureBlobStorageDelete]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP");
+ plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container");
+ plan_->setProperty(azure_blob_storage_processor_, "Blob", "test.blob");
+ setDefaultCredentials();
+ test_controller_.runSession(plan_, true);
+ auto passed_params = mock_blob_storage_ptr_->getPassedDeleteParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP);
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp
index a241429c25..600e20b52d 100644
--- a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp
+++ b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp
@@ -153,4 +153,29 @@ TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Delete result is false
REQUIRE(failed_flowfiles[0] == TEST_DATA);
}
+TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Test Azure data lake storage delete using proxy", "[azureDataLakeStorageDelete]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP");
+ plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ test_controller_.runSession(plan_, true);
+
+ auto passed_params = mock_data_lake_storage_client_ptr_->getPassedDeleteParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP);
+
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/FetchAzureBlobStorageTests.cpp b/extensions/azure/tests/FetchAzureBlobStorageTests.cpp
index cd57961cbf..57ed393be7 100644
--- a/extensions/azure/tests/FetchAzureBlobStorageTests.cpp
+++ b/extensions/azure/tests/FetchAzureBlobStorageTests.cpp
@@ -329,4 +329,30 @@ TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Fetch full file fails", "[a
REQUIRE(failed_contents[0] == TEST_DATA);
}
+TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Test Azure blob fetch using proxy", "[azureBlobStorageFetch]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP");
+ plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container");
+ plan_->setProperty(azure_blob_storage_processor_, "Blob", "test.blob");
+ setDefaultCredentials();
+ test_controller_.runSession(plan_, true);
+ auto passed_params = mock_blob_storage_ptr_->getPassedFetchParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP);
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp
index c5505f49e8..aea9da732d 100644
--- a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp
+++ b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp
@@ -170,4 +170,28 @@ TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Fetch full file fails",
REQUIRE(failed_contents[0] == TEST_DATA);
}
+TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Test Azure data lake storage fetch using proxy", "[azureDataLakeStorageFetch]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP");
+ plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ test_controller_.runSession(plan_, true);
+
+ auto passed_params = mock_data_lake_storage_client_ptr_->getPassedFetchParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP);
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/ListAzureBlobStorageTests.cpp b/extensions/azure/tests/ListAzureBlobStorageTests.cpp
index 31bdad9098..4b9db71e84 100644
--- a/extensions/azure/tests/ListAzureBlobStorageTests.cpp
+++ b/extensions/azure/tests/ListAzureBlobStorageTests.cpp
@@ -349,4 +349,31 @@ TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "Do not list same files the s
REQUIRE_FALSE(LogTestController::getInstance().contains("key:azure", 0s, 0ms));
}
+TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "List all files through a proxy", "[ListAzureBlobStorage]") {
+ setDefaultCredentials();
+ plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::ContainerName, CONTAINER_NAME);
+ plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::Prefix, PREFIX);
+ plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::ListingStrategy, magic_enum::enum_name(minifi::azure::EntityTracking::none));
+
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP");
+ plan_->setProperty(list_azure_blob_storage_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ test_controller_.runSession(plan_, true);
+ auto passed_params = mock_blob_storage_ptr_->getPassedListParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP);
+}
+
} // namespace
diff --git a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp
index 4f2a19b512..5b635bd35c 100644
--- a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp
+++ b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp
@@ -256,4 +256,27 @@ TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Both SAS Token and Stora
REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception);
}
+TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "List data lake storage files using proxy", "[azureDataLakeStorageParameters]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTPS");
+ plan_->setProperty(list_azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ test_controller_.runSession(plan_, true);
+
+ auto passed_params = mock_data_lake_storage_client_ptr_->getPassedListParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTPS);
+}
+
} // namespace
diff --git a/extensions/azure/tests/PutAzureBlobStorageTests.cpp b/extensions/azure/tests/PutAzureBlobStorageTests.cpp
index a4a0ed19ee..2d564b5eb1 100644
--- a/extensions/azure/tests/PutAzureBlobStorageTests.cpp
+++ b/extensions/azure/tests/PutAzureBlobStorageTests.cpp
@@ -338,4 +338,30 @@ TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test Azure blob upload failur
REQUIRE(failed_flowfiles[0] == TEST_DATA);
}
+TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test Azure blob storage put using proxy", "[azureBlobStorageUpload]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTPS");
+ plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container");
+ plan_->setProperty(azure_blob_storage_processor_, "Blob", "test.blob");
+ setDefaultCredentials();
+ test_controller_.runSession(plan_, true);
+ auto passed_params = mock_blob_storage_ptr_->getPassedPutParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTPS);
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp
index 6938dc841f..b9da3b22b1 100644
--- a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp
+++ b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp
@@ -193,4 +193,28 @@ TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Upload to Azure Data Lake
CHECK(verifyLogLinePresenceInPollTime(1s, "key:azure.directory value:\n"));
}
+TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Test Azure data lake storage upload using proxy", "[azureDataLakeStorageUpload]") {
+ auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host");
+ plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username");
+ plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password");
+ plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP");
+ plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService");
+
+ test_controller_.runSession(plan_, true);
+
+ auto passed_params = mock_data_lake_storage_client_ptr_->getPassedPutParams();
+ REQUIRE(passed_params.proxy_configuration);
+ REQUIRE(passed_params.proxy_configuration->proxy_host == "host");
+ REQUIRE(passed_params.proxy_configuration->proxy_port);
+ REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234);
+ REQUIRE(passed_params.proxy_configuration->proxy_user);
+ REQUIRE(*passed_params.proxy_configuration->proxy_user == "username");
+ REQUIRE(passed_params.proxy_configuration->proxy_password);
+ REQUIRE(*passed_params.proxy_configuration->proxy_password == "password");
+ REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP);
+ CHECK(getFailedFlowFileContents().empty());
+}
+
} // namespace
diff --git a/extensions/azure/tests/features/azure_storage.feature b/extensions/azure/tests/features/azure_storage.feature
index 26fb71dc18..e46db655f5 100644
--- a/extensions/azure/tests/features/azure_storage.feature
+++ b/extensions/azure/tests/features/azure_storage.feature
@@ -108,3 +108,89 @@ Feature: Sending data from MiNiFi-C++ to an Azure storage server
Then the Minifi logs contain the following message: "key:azure.blobname value:test_1" in less than 60 seconds
Then the Minifi logs contain the following message: "key:azure.blobname value:test_2" in less than 60 seconds
And the Minifi logs do not contain the following message: "key:azure.blobname value:other_test" after 0 seconds
+
+ Scenario Outline: A MiNiFi instance can upload data to Azure blob storage through a proxy
+ Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
+ And a file with the content "#test_data$123$#" is present in "/tmp/input"
+ And a PutAzureBlobStorage processor set up to communicate with an Azure blob storage
+ And the "Proxy Configuration Service" property of the PutAzureBlobStorage processor is set to "ProxyConfigurationService"
+ And a PutFile processor with the "Directory" property set to "/tmp/output"
+ And the "success" relationship of the GetFile processor is connected to the PutAzureBlobStorage
+ And the "success" relationship of the PutAzureBlobStorage processor is connected to the PutFile
+ And the "failure" relationship of the PutAzureBlobStorage processor is connected to the PutAzureBlobStorage
+ And a ProxyConfigurationService controller service is set up with proxy configuration
+
+ And an Azure storage server is set up
+ And the http proxy server is set up
+
+ When all instances start up
+
+ Then a single file with the content "#test_data$123$#" is placed in the "/tmp/output" directory in less than 60 seconds
+ And the object on the Azure storage server is "#test_data$123$#"
+ And no errors were generated on the http-proxy regarding "http://azure-storage-server-${scenario_id}:10000/devstoreaccount1/test-container/test-blob"
+
+ Examples: Proxy Type
+ | proxy type |
+ | HTTP |
+ | HTTPS |
+
+ Scenario: A MiNiFi instance can delete blob from Azure blob storage through a proxy
+ Given a GenerateFlowFile processor with the "File Size" property set to "0B"
+ And a DeleteAzureBlobStorage processor set up to communicate with an Azure blob storage
+ And the "Blob" property of the DeleteAzureBlobStorage processor is set to "test"
+ And the "Proxy Configuration Service" property of the DeleteAzureBlobStorage processor is set to "ProxyConfigurationService"
+ And the "success" relationship of the GenerateFlowFile processor is connected to the DeleteAzureBlobStorage
+ And a ProxyConfigurationService controller service is set up with HTTP proxy configuration
+
+ And an Azure storage server is set up
+ And the http proxy server is set up
+
+ When all instances start up
+ And test blob "test" is created on Azure blob storage
+
+ Then the Azure blob storage becomes empty in 30 seconds
+ And no errors were generated on the http-proxy regarding "http://azure-storage-server-${scenario_id}:10000/devstoreaccount1/test-container/test"
+
+ Scenario: A MiNiFi instance can fetch a blob from Azure blob storage through a proxy
+ Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
+ And the "Keep Source File" property of the GetFile processor is set to "true"
+ And a file with the content "dummy" is present in "/tmp/input"
+ And a FetchAzureBlobStorage processor set up to communicate with an Azure blob storage
+ And the "Blob" property of the FetchAzureBlobStorage processor is set to "test"
+ And the "Range Start" property of the FetchAzureBlobStorage processor is set to "6"
+ And the "Range Length" property of the FetchAzureBlobStorage processor is set to "5"
+ And the "Proxy Configuration Service" property of the FetchAzureBlobStorage processor is set to "ProxyConfigurationService"
+ And a PutFile processor with the "Directory" property set to "/tmp/output"
+ And the "success" relationship of the GetFile processor is connected to the FetchAzureBlobStorage
+ And the "success" relationship of the FetchAzureBlobStorage processor is connected to the PutFile
+ And a ProxyConfigurationService controller service is set up with HTTP proxy configuration
+
+ And an Azure storage server is set up
+ And the http proxy server is set up
+
+ When all instances start up
+ And test blob "test" with the content "#test_data$123$#" is created on Azure blob storage
+
+ Then a single file with the content "data$" is placed in the "/tmp/output" directory in less than 60 seconds
+ And no errors were generated on the http-proxy regarding "http://azure-storage-server-${scenario_id}:10000/devstoreaccount1/test-container/test"
+
+ Scenario: A MiNiFi instance can list a container on Azure blob storage through a proxy
+ Given a ListAzureBlobStorage processor set up to communicate with an Azure blob storage
+ And the "Prefix" property of the ListAzureBlobStorage processor is set to "test"
+ And the "Proxy Configuration Service" property of the ListAzureBlobStorage processor is set to "ProxyConfigurationService"
+ And a LogAttribute processor with the "FlowFiles To Log" property set to "0"
+ And the "success" relationship of the ListAzureBlobStorage processor is connected to the LogAttribute
+ And a ProxyConfigurationService controller service is set up with HTTP proxy configuration
+
+ And an Azure storage server is set up
+ And the http proxy server is set up
+
+ When all instances start up
+ And test blob "test_1" with the content "data_1" is created on Azure blob storage
+ And test blob "test_2" with the content "data_2" is created on Azure blob storage
+ And test blob "other_test" with the content "data_3" is created on Azure blob storage
+
+ Then the Minifi logs contain the following message: "key:azure.blobname value:test_1" in less than 60 seconds
+ And the Minifi logs contain the following message: "key:azure.blobname value:test_2" in less than 60 seconds
+ And the Minifi logs do not contain the following message: "key:azure.blobname value:other_test" after 0 seconds
+ And no errors were generated on the http-proxy regarding "http://azure-storage-server-${scenario_id}:10000/devstoreaccount1/test-container"
diff --git a/libminifi/include/controllers/ProxyConfigurationService.h b/libminifi/include/controllers/ProxyConfigurationService.h
new file mode 100644
index 0000000000..e9e474f4ae
--- /dev/null
+++ b/libminifi/include/controllers/ProxyConfigurationService.h
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include
+
+#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h"
+#include "controllers/ProxyConfiguration.h"
+#include "core/controller/ControllerService.h"
+#include "core/PropertyDefinitionBuilder.h"
+#include "minifi-cpp/core/PropertyValidator.h"
+
+namespace org::apache::nifi::minifi::controllers {
+
+class ProxyConfigurationService : public core::controller::ControllerServiceImpl, public ProxyConfigurationServiceInterface {
+ public:
+ explicit ProxyConfigurationService(std::string_view name, const utils::Identifier& uuid = {})
+ : ControllerServiceImpl(name, uuid) {
+ }
+
+ MINIFIAPI static constexpr const char* Description = "Provides a set of configurations for various MiNiFi C++ components to use a proxy server. Currently these properties can only be used for "
+ "HTTP proxy configuration, no other protocols are supported at this time.";
+
+ MINIFIAPI static constexpr auto ProxyServerHost = core::PropertyDefinitionBuilder<>::createProperty("Proxy Server Host")
+ .withDescription("Proxy server hostname or ip-address.")
+ .isRequired(true)
+ .withValidator(core::StandardPropertyValidators::NON_BLANK_VALIDATOR)
+ .build();
+ MINIFIAPI static constexpr auto ProxyServerPort = core::PropertyDefinitionBuilder<>::createProperty("Proxy Server Port")
+ .withDescription("Proxy server port number.")
+ .withValidator(core::StandardPropertyValidators::PORT_VALIDATOR)
+ .build();
+ MINIFIAPI static constexpr auto ProxyUserName = core::PropertyDefinitionBuilder<>::createProperty("Proxy User Name")
+ .withDescription("The name of the proxy client for user authentication.")
+ .build();
+ MINIFIAPI static constexpr auto ProxyUserPassword = core::PropertyDefinitionBuilder<>::createProperty("Proxy User Password")
+ .withDescription("The password of the proxy client for user authentication.")
+ .isSensitive(true)
+ .build();
+ MINIFIAPI static constexpr auto ProxyTypeProperty = core::PropertyDefinitionBuilder()>::createProperty("Proxy Type")
+ .withDescription("Proxy type.")
+ .withDefaultValue(magic_enum::enum_name(ProxyType::HTTP))
+ .withAllowedValues(magic_enum::enum_names())
+ .build();
+ MINIFIAPI static constexpr auto Properties = std::to_array({
+ ProxyServerHost,
+ ProxyServerPort,
+ ProxyUserName,
+ ProxyUserPassword,
+ ProxyTypeProperty
+ });
+
+ MINIFIAPI static constexpr bool SupportsDynamicProperties = false;
+ MINIFIAPI static constexpr auto ImplementsApis = std::array{ ProxyConfigurationServiceInterface::ProvidesApi };
+ ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_CONTROLLER_SERVICES
+
+ void yield() override {
+ }
+
+ bool isRunning() const override {
+ return getState() == core::controller::ControllerServiceState::ENABLED;
+ }
+
+ bool isWorkAvailable() override {
+ return false;
+ }
+
+ void initialize() override;
+ void onEnable() override;
+
+ ProxyType getProxyType() const override {
+ std::lock_guard lock(configuration_mutex_);
+ return proxy_configuration_.proxy_type;
+ }
+
+ std::string getHost() const override {
+ std::lock_guard lock(configuration_mutex_);
+ return proxy_configuration_.proxy_host;
+ }
+
+ std::optional getPort() const override {
+ std::lock_guard lock(configuration_mutex_);
+ return proxy_configuration_.proxy_port;
+ }
+
+ std::optional getUsername() const override {
+ std::lock_guard lock(configuration_mutex_);
+ return proxy_configuration_.proxy_user;
+ }
+
+ std::optional getPassword() const override {
+ std::lock_guard lock(configuration_mutex_);
+ return proxy_configuration_.proxy_password;
+ }
+
+ private:
+ mutable std::mutex configuration_mutex_;
+ ProxyConfiguration proxy_configuration_;
+ std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_);
+};
+
+} // namespace org::apache::nifi::minifi::controllers
diff --git a/libminifi/src/controllers/ProxyConfigurationService.cpp b/libminifi/src/controllers/ProxyConfigurationService.cpp
new file mode 100644
index 0000000000..dd7a420477
--- /dev/null
+++ b/libminifi/src/controllers/ProxyConfigurationService.cpp
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "controllers/ProxyConfigurationService.h"
+
+#include "utils/ParsingUtils.h"
+#include "minifi-cpp/Exception.h"
+#include "core/Resource.h"
+
+namespace org::apache::nifi::minifi::controllers {
+
+void ProxyConfigurationService::initialize() {
+ setSupportedProperties(Properties);
+}
+
+void ProxyConfigurationService::onEnable() {
+ std::lock_guard lock(configuration_mutex_);
+ proxy_configuration_.proxy_type = magic_enum::enum_cast(getProperty(ProxyTypeProperty.name).value_or("HTTP")).value_or(ProxyType::HTTP);
+ proxy_configuration_.proxy_host = getProperty(ProxyServerHost.name).value_or("");
+ if (proxy_configuration_.proxy_host.empty()) {
+ logger_->log_error("Proxy Server Host is required");
+ throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Proxy Server Host is required");
+ }
+ if (auto proxy_port = getProperty(ProxyServerPort.name) | utils::andThen(parsing::parseIntegral)) {
+ proxy_configuration_.proxy_port = *proxy_port;
+ }
+ if (auto proxy_user = getProperty(ProxyUserName.name)) {
+ proxy_configuration_.proxy_user = *proxy_user;
+ }
+ if (auto proxy_password = getProperty(ProxyUserPassword.name)) {
+ proxy_configuration_.proxy_password = *proxy_password;
+ }
+}
+
+REGISTER_RESOURCE(ProxyConfigurationService, ControllerService);
+
+} // namespace org::apache::nifi::minifi::controllers
diff --git a/libminifi/test/unit/ProxyConfigurationServiceTests.cpp b/libminifi/test/unit/ProxyConfigurationServiceTests.cpp
new file mode 100644
index 0000000000..8bc8f593af
--- /dev/null
+++ b/libminifi/test/unit/ProxyConfigurationServiceTests.cpp
@@ -0,0 +1,63 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "unit/TestBase.h"
+#include "unit/Catch.h"
+#include "controllers/ProxyConfigurationService.h"
+
+namespace org::apache::nifi::minifi::test {
+
+struct ProxyConfigurationServiceTestFixture {
+ ProxyConfigurationServiceTestFixture() {
+ LogTestController::getInstance().clear();
+ LogTestController::getInstance().setTrace();
+ }
+
+ TestController test_controller_;
+ std::shared_ptr plan_ = test_controller_.createPlan();
+ std::shared_ptr proxy_configuration_node_ = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService");
+ std::shared_ptr proxy_configuration_service_ =
+ std::dynamic_pointer_cast(proxy_configuration_node_->getControllerServiceImplementation());
+};
+
+TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "ProxyConfigurationService onEnable throws when empty") {
+ REQUIRE_THROWS_WITH(proxy_configuration_service_->onEnable(), "Process Schedule Operation: Proxy Server Host is required");
+}
+
+TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "Only required properties are set in ProxyConfigurationService") {
+ plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyServerHost, "192.168.1.123");
+ REQUIRE_NOTHROW(plan_->finalize());
+ CHECK(proxy_configuration_service_->getHost() == "192.168.1.123");
+ CHECK(proxy_configuration_service_->getPort() == std::nullopt);
+ CHECK(proxy_configuration_service_->getUsername() == std::nullopt);
+ CHECK(proxy_configuration_service_->getPassword() == std::nullopt);
+}
+
+TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "All properties are set in ProxyConfigurationService") {
+ plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyServerHost, "192.168.1.123");
+ plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyServerPort, "8080");
+ plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyUserName, "user");
+ plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyUserPassword, "password");
+ REQUIRE_NOTHROW(plan_->finalize());
+ CHECK(proxy_configuration_service_->getHost() == "192.168.1.123");
+ CHECK(proxy_configuration_service_->getPort() == 8080);
+ CHECK(proxy_configuration_service_->getUsername() == "user");
+ CHECK(proxy_configuration_service_->getPassword() == "password");
+}
+
+} // namespace org::apache::nifi::minifi::test
diff --git a/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h b/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h
new file mode 100644
index 0000000000..f920d86d5f
--- /dev/null
+++ b/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h
@@ -0,0 +1,44 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include "minifi-cpp/core/controller/ControllerService.h"
+#include "minifi-cpp/core/ControllerServiceApiDefinition.h"
+
+namespace org::apache::nifi::minifi::controllers {
+
+enum class ProxyType {
+ HTTP,
+ HTTPS
+};
+
+class ProxyConfigurationServiceInterface : public virtual core::controller::ControllerService {
+ public:
+ static constexpr auto ProvidesApi = core::ControllerServiceApiDefinition {
+ .artifact = "minifi-system",
+ .group = "org.apache.nifi.minifi",
+ .type = "org.apache.nifi.minifi.controllers.ProxyConfigurationServiceInterface",
+ };
+
+ virtual std::string getHost() const = 0;
+ virtual std::optional getPort() const = 0;
+ virtual std::optional getUsername() const = 0;
+ virtual std::optional getPassword() const = 0;
+ virtual ProxyType getProxyType() const = 0;
+};
+
+} // namespace org::apache::nifi::minifi::controllers