From b37b3ff0ba9f2a4010e702c9365e17af0c04e8bb Mon Sep 17 00:00:00 2001 From: Gabor Gyimesi Date: Wed, 1 Oct 2025 16:53:46 +0200 Subject: [PATCH 1/7] MINIFICPP-2644 Add proxy controller service support for AWS and Azure processors --- CONTROLLERS.md | 19 ++++ PROCESSORS.md | 20 ++++- .../test/integration/features/steps/steps.py | 13 +++ .../controllers/ProxyConfigurationService.py | 33 +++++++ extensions/aws/processors/AwsProcessor.cpp | 17 +++- extensions/aws/processors/AwsProcessor.h | 7 ++ extensions/aws/tests/DeleteS3ObjectTests.cpp | 7 +- extensions/aws/tests/FetchS3ObjectTests.cpp | 7 +- extensions/aws/tests/ListS3Tests.cpp | 7 +- extensions/aws/tests/PutS3ObjectTests.cpp | 7 +- extensions/aws/tests/S3TestsFixture.h | 48 ++++++---- extensions/aws/tests/features/s3.feature | 86 ++++++++++++++++++ .../AzureBlobStorageProcessorBase.cpp | 5 +- .../AzureDataLakeStorageProcessorBase.cpp | 4 +- .../AzureDataLakeStorageProcessorBase.h | 2 +- .../processors/AzureStorageProcessorBase.cpp | 9 ++ .../processors/AzureStorageProcessorBase.h | 12 ++- .../azure/storage/AzureBlobStorageClient.cpp | 31 +++++-- .../azure/storage/AzureBlobStorageClient.h | 3 +- .../storage/AzureDataLakeStorageClient.cpp | 18 +++- .../storage/AzureDataLakeStorageClient.h | 4 +- extensions/azure/storage/BlobStorageClient.h | 2 + .../azure/storage/DataLakeStorageClient.h | 2 + .../tests/DeleteAzureBlobStorageTests.cpp | 24 +++++ .../tests/DeleteAzureDataLakeStorageTests.cpp | 22 +++++ .../tests/FetchAzureBlobStorageTests.cpp | 24 +++++ .../tests/FetchAzureDataLakeStorageTests.cpp | 22 +++++ .../azure/tests/ListAzureBlobStorageTests.cpp | 25 ++++++ .../tests/ListAzureDataLakeStorageTests.cpp | 21 +++++ .../azure/tests/PutAzureBlobStorageTests.cpp | 24 +++++ .../tests/PutAzureDataLakeStorageTests.cpp | 22 +++++ .../tests/features/azure_storage.feature | 81 +++++++++++++++++ .../controllers/ProxyConfigurationService.h | 87 +++++++++++++++++++ .../controllers/ProxyConfigurationService.cpp | 49 +++++++++++ .../unit/ProxyConfigurationServiceTests.cpp | 65 ++++++++++++++ .../ProxyConfigurationServiceInterface.h | 42 +++++++++ 36 files changed, 824 insertions(+), 47 deletions(-) create mode 100644 docker/test/integration/minifi/controllers/ProxyConfigurationService.py create mode 100644 libminifi/include/controllers/ProxyConfigurationService.h create mode 100644 libminifi/src/controllers/ProxyConfigurationService.cpp create mode 100644 libminifi/test/unit/ProxyConfigurationServiceTests.cpp create mode 100644 minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h diff --git a/CONTROLLERS.md b/CONTROLLERS.md index 587470fdde..1537fc2231 100644 --- a/CONTROLLERS.md +++ b/CONTROLLERS.md @@ -27,6 +27,7 @@ limitations under the License. - [NetworkPrioritizerService](#NetworkPrioritizerService) - [ODBCService](#ODBCService) - [PersistentMapStateStorage](#PersistentMapStateStorage) +- [ProxyConfigurationService](#ProxyConfigurationService) - [RocksDbStateStorage](#RocksDbStateStorage) - [SmbConnectionControllerService](#SmbConnectionControllerService) - [SSLContextService](#SSLContextService) @@ -244,6 +245,24 @@ In the list below, the names of required properties appear in bold. Any other pr | **File** | | | Path to a file to store state | +## ProxyConfigurationService + +### Description + +Provides a set of configurations for different MiNiFi C++ components to use a proxy server. + +### Properties + +In the list below, the names of required properties appear in bold. Any other properties (not in bold) are considered optional. The table also indicates any default values, and whether a property supports the NiFi Expression Language. + +| Name | Default Value | Allowable Values | Description | +|-----------------------|---------------|------------------|--------------------------------------------------------------------------------------------| +| **Proxy Server Host** | | | Proxy server hostname or ip-address. | +| Proxy Server Port | | | Proxy server port number. | +| Proxy User Name | | | The name of the proxy client for user authentication. | +| Proxy User Password | | | The password of the proxy client for user authentication.
**Sensitive Property: true** | + + ## RocksDbStateStorage ### Description diff --git a/PROCESSORS.md b/PROCESSORS.md index 7851a06afa..521e5aa05c 100644 --- a/PROCESSORS.md +++ b/PROCESSORS.md @@ -553,6 +553,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. | | **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** | | Storage Account Name | | | The storage account name.
**Supports Expression Language: true** | | Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** | @@ -585,6 +586,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |-----------------------------------|---------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. | | **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** | | Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** | | File Name | | | The filename in Azure Storage. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** | @@ -645,7 +647,6 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** | | Access Key | | | AWS account access key
**Supports Expression Language: true** | | Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** | | Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey | @@ -657,7 +658,9 @@ In the list below, the names of required properties appear in bold. Any other pr | Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** | | Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** | | Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. | | **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | +| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** | | Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** | | Version | | | The Version of the Object to delete
**Supports Expression Language: true** | @@ -817,6 +820,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. | | **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** | | Storage Account Name | | | The storage account name.
**Supports Expression Language: true** | | Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** | @@ -850,6 +854,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |-----------------------------------|---------------|------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. | | **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** | | Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** | | File Name | | | The filename in Azure Storage. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** | @@ -1017,7 +1022,6 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** | | Access Key | | | AWS account access key
**Supports Expression Language: true** | | Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** | | Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey | @@ -1029,7 +1033,9 @@ In the list below, the names of required properties appear in bold. Any other pr | Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** | | Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** | | Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. | | **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | +| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** | | Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** | | Version | | | The Version of the Object to download
**Supports Expression Language: true** | | **Requester Pays** | false | true
false | If true, indicates that the requester consents to pay any charges associated with retrieving objects from the S3 bucket. This sets the 'x-amz-request-payer' header to 'requester'. | @@ -1384,6 +1390,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. | | **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** | | Storage Account Name | | | The storage account name.
**Supports Expression Language: true** | | Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** | @@ -1415,6 +1422,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |-----------------------------------|---------------|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. | | **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** | | Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** | | **Recurse Subdirectories** | true | true
false | Indicates whether to list files from subdirectories of the directory | @@ -1678,7 +1686,6 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** | | Access Key | | | AWS account access key
**Supports Expression Language: true** | | Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** | | Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey | @@ -1690,7 +1697,9 @@ In the list below, the names of required properties appear in bold. Any other pr | Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** | | Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** | | Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. | | **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | +| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** | | Delimiter | | | The string used to delimit directories within the bucket. Please consult the AWS documentation for the correct use of this field. | | Prefix | | | The prefix used to filter the object list. In most cases, it should end with a forward slash ('/'). | | **Use Versions** | false | true
false | Specifies whether to use S3 versions, if applicable. If false, only the latest version of each object will be returned. | @@ -2313,6 +2322,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------------------|-----------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. | | **Container Name** | | | Name of the Azure Storage container. In case of PutAzureBlobStorage processor, container can be created if it does not exist.
**Supports Expression Language: true** | | Storage Account Name | | | The storage account name.
**Supports Expression Language: true** | | Storage Account Key | | | The storage account key. This is an admin-like password providing access to every container in this account. It is recommended one uses Shared Access Signature (SAS) token instead for fine-grained control with policies if Credential Configuration Strategy is set to From Properties. If set, SAS Token must be empty.
**Sensitive Property: true**
**Supports Expression Language: true** | @@ -2345,6 +2355,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |-----------------------------------|---------------|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | Azure Storage Credentials Service | | | Name of the Azure Storage Credentials Service used to retrieve the connection string from. | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. | | **Filesystem Name** | | | Name of the Azure Storage File System. It is assumed to be already existing.
**Supports Expression Language: true** | | Directory Name | | | Name of the Azure Storage Directory. The Directory Name cannot contain a leading '/'. If left empty it designates the root directory. The directory will be created if not already existing.
**Supports Expression Language: true** | | File Name | | | The filename in Azure Storage. If left empty the filename attribute will be used by default.
**Supports Expression Language: true** | @@ -2583,7 +2594,6 @@ In the list below, the names of required properties appear in bold. Any other pr | Name | Default Value | Allowable Values | Description | |----------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** | | Access Key | | | AWS account access key
**Supports Expression Language: true** | | Secret Key | | | AWS account secret key
**Sensitive Property: true**
**Supports Expression Language: true** | | Credentials File | | | Path to a file containing AWS access key and secret key in properties file format. Properties used: accessKey and secretKey | @@ -2595,7 +2605,9 @@ In the list below, the names of required properties appear in bold. Any other pr | Proxy Port | | | The port number of the proxy host
**Supports Expression Language: true** | | Proxy Username | | | Username to set when authenticating against proxy
**Supports Expression Language: true** | | Proxy Password | | | Password to set when authenticating against proxy
**Sensitive Property: true**
**Supports Expression Language: true** | +| Proxy Configuration Service | | | Specifies the Proxy Configuration Controller Service to proxy network requests. When used, this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties. | | **Use Default Credentials** | false | true
false | If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc. | +| **Bucket** | | | The S3 bucket
**Supports Expression Language: true** | | Object Key | | | The key of the S3 object. If none is given the filename attribute will be used by default.
**Supports Expression Language: true** | | Content Type | application/octet-stream | | Sets the Content-Type HTTP header indicating the type of content stored in the associated object. The value of this header is a standard MIME type. If no content type is provided the default content type "application/octet-stream" will be used.
**Supports Expression Language: true** | | **Storage Class** | Standard | Standard
ReducedRedundancy
StandardIA
OnezoneIA
IntelligentTiering
Glacier
DeepArchive
Outposts
GlacierIR
Snow
ExpressOneZone | AWS S3 Storage Class | diff --git a/docker/test/integration/features/steps/steps.py b/docker/test/integration/features/steps/steps.py index 606d5d76dc..91c93dd5b2 100644 --- a/docker/test/integration/features/steps/steps.py +++ b/docker/test/integration/features/steps/steps.py @@ -25,6 +25,7 @@ from minifi.controllers.XMLReader import XMLReader from minifi.controllers.XMLRecordSetWriter import XMLRecordSetWriter from minifi.controllers.XMLReader import XMLReader +from minifi.controllers.ProxyConfigurationService import ProxyConfigurationService from behave import given, then, when from behave.model_describe import ModelDescriptor @@ -403,6 +404,18 @@ def step_impl(context): context.test.acquire_container(context=context, name="http-proxy", engine="http-proxy") +@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{container_name}\" flow") +def step_impl(context, container_name): + proxy_service = ProxyConfigurationService("ProxyConfigurationService", host=f"http-proxy-{context.feature_id}", port=3128, username="admin", password="test101") + container = context.test.acquire_container(context=context, name=container_name) + container.add_controller(proxy_service) + + +@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration") +def step_impl(context): + context.execute_steps("given a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{container_name}\" flow".format(container_name="minifi-cpp-flow")) + + # TLS @given("an ssl context service is set up for {processor_name}") @given("an ssl context service with a manual CA cert file is set up for {processor_name}") diff --git a/docker/test/integration/minifi/controllers/ProxyConfigurationService.py b/docker/test/integration/minifi/controllers/ProxyConfigurationService.py new file mode 100644 index 0000000000..d0e3f457ec --- /dev/null +++ b/docker/test/integration/minifi/controllers/ProxyConfigurationService.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ..core.ControllerService import ControllerService + + +class ProxyConfigurationService(ControllerService): + def __init__(self, name, host, port=None, username=None, password=None): + super(ProxyConfigurationService, self).__init__(name=name) + + self.service_class = 'ProxyConfigurationService' + + self.properties['Proxy Server Host'] = host + + if port is not None: + self.properties['Proxy Server Port'] = port + + if username is not None: + self.properties['Proxy User Name'] = username + + if password is not None: + self.properties['Proxy User Password'] = password diff --git a/extensions/aws/processors/AwsProcessor.cpp b/extensions/aws/processors/AwsProcessor.cpp index 6c0a1b19e8..33157f1693 100644 --- a/extensions/aws/processors/AwsProcessor.cpp +++ b/extensions/aws/processors/AwsProcessor.cpp @@ -69,10 +69,19 @@ std::optional AwsProcessor::getAWSCredentials( aws::ProxyOptions AwsProcessor::getProxy(core::ProcessContext& context, const core::FlowFile* const flow_file) { aws::ProxyOptions proxy; - proxy.host = minifi::utils::parseOptionalProperty(context, ProxyHost, flow_file).value_or(""); - proxy.port = gsl::narrow(minifi::utils::parseOptionalU64Property(context, ProxyPort, flow_file).value_or(0)); - proxy.username = minifi::utils::parseOptionalProperty(context, ProxyUsername, flow_file).value_or(""); - proxy.password = minifi::utils::parseOptionalProperty(context, ProxyPassword, flow_file).value_or(""); + auto proxy_controller_service = minifi::utils::parseOptionalControllerService(context, ProxyConfigurationService, getUUID()); + if (proxy_controller_service) { + auto controller_service_proxy = proxy_controller_service->getProxyConfiguration(); + proxy.host = controller_service_proxy.proxy_host; + proxy.port = controller_service_proxy.proxy_port ? *controller_service_proxy.proxy_port : 0; + proxy.username = controller_service_proxy.proxy_user ? *controller_service_proxy.proxy_user : ""; + proxy.password = controller_service_proxy.proxy_password ? *controller_service_proxy.proxy_password : ""; + } else { + proxy.host = minifi::utils::parseOptionalProperty(context, ProxyHost, flow_file).value_or(""); + proxy.port = gsl::narrow(minifi::utils::parseOptionalU64Property(context, ProxyPort, flow_file).value_or(0)); + proxy.username = minifi::utils::parseOptionalProperty(context, ProxyUsername, flow_file).value_or(""); + proxy.password = minifi::utils::parseOptionalProperty(context, ProxyPassword, flow_file).value_or(""); + } if (!proxy.host.empty()) { logger_->log_info("Proxy for AwsProcessor was set."); diff --git a/extensions/aws/processors/AwsProcessor.h b/extensions/aws/processors/AwsProcessor.h index 3637fdc343..548f332795 100644 --- a/extensions/aws/processors/AwsProcessor.h +++ b/extensions/aws/processors/AwsProcessor.h @@ -33,6 +33,7 @@ #include "core/PropertyDefinitionBuilder.h" #include "minifi-cpp/core/PropertyValidator.h" #include "core/ProcessorImpl.h" +#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" namespace org::apache::nifi::minifi::aws::processors { @@ -148,6 +149,11 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s .supportsExpressionLanguage(true) .isSensitive(true) .build(); + EXTENSIONAPI static constexpr auto ProxyConfigurationService = core::PropertyDefinitionBuilder<>::createProperty("Proxy Configuration Service") + .withDescription("Specifies the Proxy Configuration Controller Service to proxy network requests. When used, " + "this will override any values specified for Proxy Host, Proxy Port, Proxy Username, and Proxy Password properties.") + .withAllowedTypes() + .build(); EXTENSIONAPI static constexpr auto UseDefaultCredentials = core::PropertyDefinitionBuilder<>::createProperty("Use Default Credentials") .withDescription("If true, uses the Default Credential chain, including EC2 instance profiles or roles, environment variables, default user credentials, etc.") .withValidator(core::StandardPropertyValidators::BOOLEAN_VALIDATOR) @@ -166,6 +172,7 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s ProxyPort, ProxyUsername, ProxyPassword, + ProxyConfigurationService, UseDefaultCredentials }); diff --git a/extensions/aws/tests/DeleteS3ObjectTests.cpp b/extensions/aws/tests/DeleteS3ObjectTests.cpp index bf06e060a2..de39899f9f 100644 --- a/extensions/aws/tests/DeleteS3ObjectTests.cpp +++ b/extensions/aws/tests/DeleteS3ObjectTests.cpp @@ -84,7 +84,12 @@ TEST_CASE_METHOD(DeleteS3ObjectTestsFixture, "Non blank validator tests") { TEST_CASE_METHOD(DeleteS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]") { setRequiredProperties(); - setProxy(); + SECTION("Use proxy configuration service") { + setProxy(true); + } + SECTION("Use processor properties") { + setProxy(false); + } test_controller.runSession(plan, true); checkProxySettings(); } diff --git a/extensions/aws/tests/FetchS3ObjectTests.cpp b/extensions/aws/tests/FetchS3ObjectTests.cpp index 5920c13c57..856ccf68f0 100644 --- a/extensions/aws/tests/FetchS3ObjectTests.cpp +++ b/extensions/aws/tests/FetchS3ObjectTests.cpp @@ -101,7 +101,12 @@ TEST_CASE_METHOD(FetchS3ObjectTestsFixture, "Non blank validator tests") { TEST_CASE_METHOD(FetchS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]") { setRequiredProperties(); - setProxy(); + SECTION("Use proxy configuration service") { + setProxy(true); + } + SECTION("Use processor properties") { + setProxy(false); + } test_controller.runSession(plan, true); checkProxySettings(); } diff --git a/extensions/aws/tests/ListS3Tests.cpp b/extensions/aws/tests/ListS3Tests.cpp index 424e83fd53..752b329449 100644 --- a/extensions/aws/tests/ListS3Tests.cpp +++ b/extensions/aws/tests/ListS3Tests.cpp @@ -74,7 +74,12 @@ TEST_CASE_METHOD(ListS3TestsFixture, "Non blank validator tests") { TEST_CASE_METHOD(ListS3TestsFixture, "Test proxy setting", "[awsS3Proxy]") { setRequiredProperties(); - setProxy(); + SECTION("Use proxy configuration service") { + setProxy(true); + } + SECTION("Use processor properties") { + setProxy(false); + } test_controller.runSession(plan, true); checkProxySettings(); } diff --git a/extensions/aws/tests/PutS3ObjectTests.cpp b/extensions/aws/tests/PutS3ObjectTests.cpp index a78b0cdd07..75bbba02e7 100644 --- a/extensions/aws/tests/PutS3ObjectTests.cpp +++ b/extensions/aws/tests/PutS3ObjectTests.cpp @@ -200,7 +200,12 @@ TEST_CASE_METHOD(PutS3ObjectTestsFixture, "Test multiple user metadata", "[awsS3 TEST_CASE_METHOD(PutS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]") { setRequiredProperties(); - setProxy(); + SECTION("Use proxy configuration service") { + setProxy(true); + } + SECTION("Use processor properties") { + setProxy(false); + } test_controller.runSession(plan); checkProxySettings(); } diff --git a/extensions/aws/tests/S3TestsFixture.h b/extensions/aws/tests/S3TestsFixture.h index 360e2722f8..24a570649e 100644 --- a/extensions/aws/tests/S3TestsFixture.h +++ b/extensions/aws/tests/S3TestsFixture.h @@ -96,7 +96,7 @@ class S3TestsFixture { virtual void setAccesKeyCredentialsInProcessor() = 0; virtual void setBucket() = 0; - virtual void setProxy() = 0; + virtual void setProxy(bool use_controller_service) = 0; void setRequiredProperties() { setAccesKeyCredentialsInProcessor(); @@ -178,15 +178,24 @@ class FlowProcessorS3TestsFixture : public S3TestsFixture { this->plan->setProperty(this->s3_processor, "Bucket", "${test.bucket}"); } - void setProxy() override { - this->plan->setDynamicProperty(update_attribute, "test.proxyHost", "host"); - this->plan->setProperty(this->s3_processor, "Proxy Host", "${test.proxyHost}"); - this->plan->setDynamicProperty(update_attribute, "test.proxyPort", "1234"); - this->plan->setProperty(this->s3_processor, "Proxy Port", "${test.proxyPort}"); - this->plan->setDynamicProperty(update_attribute, "test.proxyUsername", "username"); - this->plan->setProperty(this->s3_processor, "Proxy Username", "${test.proxyUsername}"); - this->plan->setDynamicProperty(update_attribute, "test.proxyPassword", "password"); - this->plan->setProperty(this->s3_processor, "Proxy Password", "${test.proxyPassword}"); + void setProxy(bool use_controller_service) override { + if (use_controller_service) { + auto proxy_configuration_service = this->plan->addController("ProxyConfigurationService", "ProxyConfigurationService"); + this->plan->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); + this->plan->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); + this->plan->setProperty(proxy_configuration_service, "Proxy User Name", "username"); + this->plan->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + this->plan->setProperty(this->s3_processor, "Proxy Configuration Service", "ProxyConfigurationService"); + } else { + this->plan->setDynamicProperty(update_attribute, "test.proxyHost", "host"); + this->plan->setProperty(this->s3_processor, "Proxy Host", "${test.proxyHost}"); + this->plan->setDynamicProperty(update_attribute, "test.proxyPort", "1234"); + this->plan->setProperty(this->s3_processor, "Proxy Port", "${test.proxyPort}"); + this->plan->setDynamicProperty(update_attribute, "test.proxyUsername", "username"); + this->plan->setProperty(this->s3_processor, "Proxy Username", "${test.proxyUsername}"); + this->plan->setDynamicProperty(update_attribute, "test.proxyPassword", "password"); + this->plan->setProperty(this->s3_processor, "Proxy Password", "${test.proxyPassword}"); + } } protected: @@ -224,10 +233,19 @@ class FlowProducerS3TestsFixture : public S3TestsFixture { this->plan->setProperty(this->s3_processor, "Bucket", this->S3_BUCKET); } - void setProxy() override { - this->plan->setProperty(this->s3_processor, "Proxy Host", "host"); - this->plan->setProperty(this->s3_processor, "Proxy Port", "1234"); - this->plan->setProperty(this->s3_processor, "Proxy Username", "username"); - this->plan->setProperty(this->s3_processor, "Proxy Password", "password"); + void setProxy(bool use_controller_service) override { + if (use_controller_service) { + auto proxy_configuration_service = this->plan->addController("ProxyConfigurationService", "ProxyConfigurationService"); + this->plan->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); + this->plan->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); + this->plan->setProperty(proxy_configuration_service, "Proxy User Name", "username"); + this->plan->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + this->plan->setProperty(this->s3_processor, "Proxy Configuration Service", "ProxyConfigurationService"); + } else { + this->plan->setProperty(this->s3_processor, "Proxy Host", "host"); + this->plan->setProperty(this->s3_processor, "Proxy Port", "1234"); + this->plan->setProperty(this->s3_processor, "Proxy Username", "username"); + this->plan->setProperty(this->s3_processor, "Proxy Password", "password"); + } } }; diff --git a/extensions/aws/tests/features/s3.feature b/extensions/aws/tests/features/s3.feature index b16ea1f259..59ffa9f2fe 100644 --- a/extensions/aws/tests/features/s3.feature +++ b/extensions/aws/tests/features/s3.feature @@ -63,6 +63,25 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And the object content type on the s3 server is "application/octet-stream" and the object metadata matches use metadata And no errors were generated on the http-proxy regarding "http://s3-server-s3-1:9090/test_bucket/test_object_key" + Scenario: A MiNiFi instance transfers encoded data through a http proxy to s3 using proxy configuration service + Given a GetFile processor with the "Input Directory" property set to "/tmp/input" + And a file with the content "LH_O#L|FD credentials; std::tie(std::ignore, credentials) = getCredentialsFromControllerService(context); if (!credentials) { @@ -51,6 +52,7 @@ bool AzureDataLakeStorageProcessorBase::setCommonParameters(storage::AzureDataLa } params.directory_name = context.getProperty(DirectoryName, flow_file).value_or(""); + params.proxy_configuration = proxy_configuration_; return true; } diff --git a/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h b/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h index dc2f55f3e2..88de0b1446 100644 --- a/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h +++ b/extensions/azure/processors/AzureDataLakeStorageProcessorBase.h @@ -56,7 +56,7 @@ class AzureDataLakeStorageProcessorBase : public AzureStorageProcessorBase { ~AzureDataLakeStorageProcessorBase() override = default; - void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& sessionFactory) override; + void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override; protected: explicit AzureDataLakeStorageProcessorBase(core::ProcessorMetadata metadata, std::unique_ptr data_lake_storage_client) diff --git a/extensions/azure/processors/AzureStorageProcessorBase.cpp b/extensions/azure/processors/AzureStorageProcessorBase.cpp index 7795a4786b..f059dbfc36 100644 --- a/extensions/azure/processors/AzureStorageProcessorBase.cpp +++ b/extensions/azure/processors/AzureStorageProcessorBase.cpp @@ -25,9 +25,18 @@ #include "minifi-cpp/core/ProcessContext.h" #include "controllerservices/AzureStorageCredentialsService.h" +#include "utils/ProcessorConfigUtils.h" namespace org::apache::nifi::minifi::azure::processors { +void AzureStorageProcessorBase::onSchedule(core::ProcessContext& context, core::ProcessSessionFactory&) { + auto proxy_controller_service = minifi::utils::parseOptionalControllerService(context, ProxyConfigurationService, getUUID()); + if (proxy_controller_service) { + logger_->log_debug("Proxy configuration is set for Azure Storage processor"); + proxy_configuration_ = proxy_controller_service->getProxyConfiguration(); + } +} + std::tuple> AzureStorageProcessorBase::getCredentialsFromControllerService( core::ProcessContext &context) const { std::string service_name = context.getProperty(AzureStorageCredentialsService).value_or(""); diff --git a/extensions/azure/processors/AzureStorageProcessorBase.h b/extensions/azure/processors/AzureStorageProcessorBase.h index 8f98f72a0c..3e0cefcb2f 100644 --- a/extensions/azure/processors/AzureStorageProcessorBase.h +++ b/extensions/azure/processors/AzureStorageProcessorBase.h @@ -32,6 +32,7 @@ #include "core/ProcessorImpl.h" #include "minifi-cpp/core/logging/Logger.h" #include "storage/AzureStorageCredentials.h" +#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" namespace org::apache::nifi::minifi::azure::processors { @@ -40,10 +41,16 @@ class AzureStorageProcessorBase : public core::ProcessorImpl { EXTENSIONAPI static constexpr auto AzureStorageCredentialsService = core::PropertyDefinitionBuilder<>::createProperty("Azure Storage Credentials Service") .withDescription("Name of the Azure Storage Credentials Service used to retrieve the connection string from.") .build(); - EXTENSIONAPI static constexpr auto Properties = std::to_array({AzureStorageCredentialsService}); + EXTENSIONAPI static constexpr auto ProxyConfigurationService = core::PropertyDefinitionBuilder<>::createProperty("Proxy Configuration Service") + .withDescription("Specifies the Proxy Configuration Controller Service to proxy network requests.") + .withAllowedTypes() + .build(); + EXTENSIONAPI static constexpr auto Properties = std::to_array({AzureStorageCredentialsService, ProxyConfigurationService}); using ProcessorImpl::ProcessorImpl; + void onSchedule(core::ProcessContext& context, core::ProcessSessionFactory& session_factory) override; + protected: enum class GetCredentialsFromControllerResult { OK, @@ -52,6 +59,9 @@ class AzureStorageProcessorBase : public core::ProcessorImpl { }; std::tuple> getCredentialsFromControllerService(core::ProcessContext &context) const; + + protected: + std::optional proxy_configuration_; }; } // namespace org::apache::nifi::minifi::azure::processors diff --git a/extensions/azure/storage/AzureBlobStorageClient.cpp b/extensions/azure/storage/AzureBlobStorageClient.cpp index 837efa4ac8..b4539ff4db 100644 --- a/extensions/azure/storage/AzureBlobStorageClient.cpp +++ b/extensions/azure/storage/AzureBlobStorageClient.cpp @@ -56,34 +56,47 @@ AzureBlobStorageClient::AzureBlobStorageClient() { utils::AzureSdkLogger::initialize(); } -Azure::Storage::Blobs::BlobContainerClient AzureBlobStorageClient::createClient(const AzureStorageCredentials &credentials, const std::string &container_name) { +Azure::Storage::Blobs::BlobContainerClient AzureBlobStorageClient::createClient(const AzureStorageCredentials &credentials, const std::string &container_name, + const std::optional& proxy_configuration) { + Azure::Storage::Blobs::BlobClientOptions client_options; + + if (proxy_configuration) { + client_options.Transport.HttpProxy = proxy_configuration->proxy_host + (proxy_configuration->proxy_port ? (":" + std::to_string(*proxy_configuration->proxy_port)) : ""); + if (proxy_configuration->proxy_user) { + client_options.Transport.ProxyUserName = *proxy_configuration->proxy_user; + } + if (proxy_configuration->proxy_password) { + client_options.Transport.ProxyPassword = *proxy_configuration->proxy_password; + } + } + if (credentials.getCredentialConfigurationStrategy() == CredentialConfigurationStrategyOption::FromProperties) { - return Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString(credentials.buildConnectionString(), container_name); + return Azure::Storage::Blobs::BlobContainerClient::CreateFromConnectionString(credentials.buildConnectionString(), container_name, client_options); } auto storage_client = Azure::Storage::Blobs::BlobServiceClient("https://" + credentials.getStorageAccountName() + ".blob." + credentials.getEndpointSuffix(), - credentials.createAzureTokenCredential()); + credentials.createAzureTokenCredential(), client_options); return storage_client.GetBlobContainerClient(container_name); } bool AzureBlobStorageClient::createContainerIfNotExists(const PutAzureBlobStorageParameters& params) { - auto container_client = createClient(params.credentials, params.container_name); + auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration); return container_client.CreateIfNotExists().Value.Created; } Azure::Storage::Blobs::Models::UploadBlockBlobResult AzureBlobStorageClient::uploadBlob(const PutAzureBlobStorageParameters& params, std::span buffer) { - auto container_client = createClient(params.credentials, params.container_name); + auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration); auto blob_client = container_client.GetBlockBlobClient(params.blob_name); return blob_client.UploadFrom(reinterpret_cast(buffer.data()), buffer.size()).Value; } std::string AzureBlobStorageClient::getUrl(const AzureBlobStorageParameters& params) { - auto container_client = createClient(params.credentials, params.container_name); + auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration); return container_client.GetUrl(); } bool AzureBlobStorageClient::deleteBlob(const DeleteAzureBlobStorageParameters& params) { - auto container_client = createClient(params.credentials, params.container_name); + auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration); Azure::Storage::Blobs::DeleteBlobOptions delete_options; if (params.optional_deletion == OptionalDeletion::INCLUDE_SNAPSHOTS) { delete_options.DeleteSnapshots = Azure::Storage::Blobs::Models::DeleteSnapshotsOption::IncludeSnapshots; @@ -95,7 +108,7 @@ bool AzureBlobStorageClient::deleteBlob(const DeleteAzureBlobStorageParameters& } std::unique_ptr AzureBlobStorageClient::fetchBlob(const FetchAzureBlobStorageParameters& params) { - auto container_client = createClient(params.credentials, params.container_name); + auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration); auto blob_client = container_client.GetBlobClient(params.blob_name); Azure::Storage::Blobs::DownloadBlobOptions options; if (params.range_start || params.range_length) { @@ -115,7 +128,7 @@ std::unique_ptr AzureBlobStorageClient::fetchBlob(const FetchAz std::vector AzureBlobStorageClient::listContainer(const ListAzureBlobStorageParameters& params) { std::vector result; - auto container_client = createClient(params.credentials, params.container_name); + auto container_client = createClient(params.credentials, params.container_name, params.proxy_configuration); Azure::Storage::Blobs::ListBlobsOptions options; options.Prefix = params.prefix; for (auto page_result = container_client.ListBlobs(options); page_result.HasPage(); page_result.MoveToNextPage()) { diff --git a/extensions/azure/storage/AzureBlobStorageClient.h b/extensions/azure/storage/AzureBlobStorageClient.h index 2b67c76356..ec3d62eea8 100644 --- a/extensions/azure/storage/AzureBlobStorageClient.h +++ b/extensions/azure/storage/AzureBlobStorageClient.h @@ -43,7 +43,8 @@ class AzureBlobStorageClient : public BlobStorageClient { std::vector listContainer(const ListAzureBlobStorageParameters& params) override; private: - static Azure::Storage::Blobs::BlobContainerClient createClient(const AzureStorageCredentials& credentials, const std::string &container_name); + static Azure::Storage::Blobs::BlobContainerClient createClient(const AzureStorageCredentials& credentials, const std::string &container_name, + const std::optional& proxy_configuration); std::shared_ptr logger_{core::logging::LoggerFactory::getLogger()}; }; diff --git a/extensions/azure/storage/AzureDataLakeStorageClient.cpp b/extensions/azure/storage/AzureDataLakeStorageClient.cpp index d7618818ed..ace617b06f 100644 --- a/extensions/azure/storage/AzureDataLakeStorageClient.cpp +++ b/extensions/azure/storage/AzureDataLakeStorageClient.cpp @@ -35,13 +35,23 @@ AzureDataLakeStorageClient::AzureDataLakeStorageClient() { utils::AzureSdkLogger::initialize(); } -std::unique_ptr AzureDataLakeStorageClient::createClient( - const AzureStorageCredentials& credentials, const std::string& file_system_name, std::optional number_of_retries) { +std::unique_ptr AzureDataLakeStorageClient::createClient(const AzureStorageCredentials& credentials, + const std::string& file_system_name, std::optional number_of_retries, const std::optional& proxy_configuration) { Azure::Storage::Files::DataLake::DataLakeClientOptions options; if (number_of_retries) { options.Retry.MaxRetries = gsl::narrow(*number_of_retries); } + if (proxy_configuration) { + options.Transport.HttpProxy = proxy_configuration->proxy_host + (proxy_configuration->proxy_port ? (":" + std::to_string(*proxy_configuration->proxy_port)) : ""); + if (proxy_configuration->proxy_user) { + options.Transport.ProxyUserName = *proxy_configuration->proxy_user; + } + if (proxy_configuration->proxy_password) { + options.Transport.ProxyPassword = *proxy_configuration->proxy_password; + } + } + if (credentials.getCredentialConfigurationStrategy() == CredentialConfigurationStrategyOption::FromProperties) { return std::make_unique( Azure::Storage::Files::DataLake::DataLakeFileSystemClient::CreateFromConnectionString(credentials.buildConnectionString(), file_system_name, options)); @@ -53,7 +63,7 @@ std::unique_ptr Azure } Azure::Storage::Files::DataLake::DataLakeDirectoryClient AzureDataLakeStorageClient::getDirectoryClient(const AzureDataLakeStorageParameters& params) { - auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries); + auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries, params.proxy_configuration); return client->GetDirectoryClient(params.directory_name); } @@ -104,7 +114,7 @@ std::unique_ptr AzureDataLakeStorageClient::fetchFile(const Fet std::vector AzureDataLakeStorageClient::listDirectory(const ListAzureDataLakeStorageParameters& params) { std::vector result; if (params.directory_name.empty()) { - auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries); + auto client = createClient(params.credentials, params.file_system_name, params.number_of_retries, params.proxy_configuration); for (auto page_result = client->ListPaths(params.recurse_subdirectories); page_result.HasPage(); page_result.MoveToNextPage()) { result.insert(result.end(), page_result.Paths.begin(), page_result.Paths.end()); } diff --git a/extensions/azure/storage/AzureDataLakeStorageClient.h b/extensions/azure/storage/AzureDataLakeStorageClient.h index 9c9615869c..b9ab9a27b0 100644 --- a/extensions/azure/storage/AzureDataLakeStorageClient.h +++ b/extensions/azure/storage/AzureDataLakeStorageClient.h @@ -93,8 +93,8 @@ class AzureDataLakeStorageClient : public DataLakeStorageClient { Azure::Storage::Files::DataLake::Models::DownloadFileResult result_; }; - static std::unique_ptr createClient( - const AzureStorageCredentials& credentials, const std::string& file_system_name, std::optional number_of_retries); + static std::unique_ptr createClient(const AzureStorageCredentials& credentials, + const std::string& file_system_name, std::optional number_of_retries, const std::optional& proxy_configuration); static Azure::Storage::Files::DataLake::DataLakeDirectoryClient getDirectoryClient(const AzureDataLakeStorageParameters& params); static Azure::Storage::Files::DataLake::DataLakeFileClient getFileClient(const AzureDataLakeStorageFileOperationParameters& params); diff --git a/extensions/azure/storage/BlobStorageClient.h b/extensions/azure/storage/BlobStorageClient.h index 007b3b86f3..a9c82947ba 100644 --- a/extensions/azure/storage/BlobStorageClient.h +++ b/extensions/azure/storage/BlobStorageClient.h @@ -30,6 +30,7 @@ #include "minifi-cpp/utils/gsl.h" #include "utils/Enum.h" #include "minifi-cpp/io/InputStream.h" +#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" namespace org::apache::nifi::minifi::azure::storage { @@ -62,6 +63,7 @@ namespace org::apache::nifi::minifi::azure::storage { struct AzureBlobStorageParameters { AzureStorageCredentials credentials; std::string container_name; + std::optional proxy_configuration; }; struct AzureBlobStorageBlobOperationParameters : public AzureBlobStorageParameters { diff --git a/extensions/azure/storage/DataLakeStorageClient.h b/extensions/azure/storage/DataLakeStorageClient.h index 06445a2e8d..fc94e5d214 100644 --- a/extensions/azure/storage/DataLakeStorageClient.h +++ b/extensions/azure/storage/DataLakeStorageClient.h @@ -31,6 +31,7 @@ #include "azure/storage/files/datalake/datalake_responses.hpp" #include "utils/Enum.h" #include "utils/RegexUtils.h" +#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" namespace org::apache::nifi::minifi::azure::storage { @@ -39,6 +40,7 @@ struct AzureDataLakeStorageParameters { std::string file_system_name; std::string directory_name; std::optional number_of_retries; + std::optional proxy_configuration; }; struct AzureDataLakeStorageFileOperationParameters : public AzureDataLakeStorageParameters { diff --git a/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp b/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp index b554021bbe..cd91a592df 100644 --- a/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp @@ -339,4 +339,28 @@ TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test Azure blob delete wit REQUIRE(failed_flowfiles[0] == TEST_DATA); } +TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test Azure blob delete using proxy", "[azureBlobStorageDelete]") { + auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); + plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); + plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService"); + + plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container"); + plan_->setProperty(azure_blob_storage_processor_, "Blob", "test.blob"); + setDefaultCredentials(); + test_controller_.runSession(plan_, true); + auto passed_params = mock_blob_storage_ptr_->getPassedDeleteParams(); + REQUIRE(passed_params.proxy_configuration); + REQUIRE(passed_params.proxy_configuration->proxy_host == "host"); + REQUIRE(passed_params.proxy_configuration->proxy_port); + REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234); + REQUIRE(passed_params.proxy_configuration->proxy_user); + REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); + REQUIRE(passed_params.proxy_configuration->proxy_password); + REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + CHECK(getFailedFlowFileContents().empty()); +} + } // namespace diff --git a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp index a241429c25..4f2a8e1698 100644 --- a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp @@ -153,4 +153,26 @@ TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Delete result is false REQUIRE(failed_flowfiles[0] == TEST_DATA); } +TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Test Azure data lake storage delete using proxy", "[azureDataLakeStorageDelete]") { + auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); + plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); + plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService"); + + test_controller_.runSession(plan_, true); + + auto passed_params = mock_data_lake_storage_client_ptr_->getPassedDeleteParams(); + REQUIRE(passed_params.proxy_configuration); + REQUIRE(passed_params.proxy_configuration->proxy_host == "host"); + REQUIRE(passed_params.proxy_configuration->proxy_port); + REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234); + REQUIRE(passed_params.proxy_configuration->proxy_user); + REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); + REQUIRE(passed_params.proxy_configuration->proxy_password); + REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + CHECK(getFailedFlowFileContents().empty()); +} + } // namespace diff --git a/extensions/azure/tests/FetchAzureBlobStorageTests.cpp b/extensions/azure/tests/FetchAzureBlobStorageTests.cpp index cd57961cbf..1a145937d6 100644 --- a/extensions/azure/tests/FetchAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/FetchAzureBlobStorageTests.cpp @@ -329,4 +329,28 @@ TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Fetch full file fails", "[a REQUIRE(failed_contents[0] == TEST_DATA); } +TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Test Azure blob fetch using proxy", "[azureBlobStorageFetch]") { + auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); + plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); + plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService"); + + plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container"); + plan_->setProperty(azure_blob_storage_processor_, "Blob", "test.blob"); + setDefaultCredentials(); + test_controller_.runSession(plan_, true); + auto passed_params = mock_blob_storage_ptr_->getPassedFetchParams(); + REQUIRE(passed_params.proxy_configuration); + REQUIRE(passed_params.proxy_configuration->proxy_host == "host"); + REQUIRE(passed_params.proxy_configuration->proxy_port); + REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234); + REQUIRE(passed_params.proxy_configuration->proxy_user); + REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); + REQUIRE(passed_params.proxy_configuration->proxy_password); + REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + CHECK(getFailedFlowFileContents().empty()); +} + } // namespace diff --git a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp index c5505f49e8..7a98f28c84 100644 --- a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp @@ -170,4 +170,26 @@ TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Fetch full file fails", REQUIRE(failed_contents[0] == TEST_DATA); } +TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Test Azure data lake storage fetch using proxy", "[azureDataLakeStorageFetch]") { + auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); + plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); + plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService"); + + test_controller_.runSession(plan_, true); + + auto passed_params = mock_data_lake_storage_client_ptr_->getPassedFetchParams(); + REQUIRE(passed_params.proxy_configuration); + REQUIRE(passed_params.proxy_configuration->proxy_host == "host"); + REQUIRE(passed_params.proxy_configuration->proxy_port); + REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234); + REQUIRE(passed_params.proxy_configuration->proxy_user); + REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); + REQUIRE(passed_params.proxy_configuration->proxy_password); + REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + CHECK(getFailedFlowFileContents().empty()); +} + } // namespace diff --git a/extensions/azure/tests/ListAzureBlobStorageTests.cpp b/extensions/azure/tests/ListAzureBlobStorageTests.cpp index 31bdad9098..59be4b4a97 100644 --- a/extensions/azure/tests/ListAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/ListAzureBlobStorageTests.cpp @@ -349,4 +349,29 @@ TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "Do not list same files the s REQUIRE_FALSE(LogTestController::getInstance().contains("key:azure", 0s, 0ms)); } +TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "List all files through a proxy", "[ListAzureBlobStorage]") { + setDefaultCredentials(); + plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::ContainerName, CONTAINER_NAME); + plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::Prefix, PREFIX); + plan_->setProperty(list_azure_blob_storage_, minifi::azure::processors::ListAzureBlobStorage::ListingStrategy, magic_enum::enum_name(minifi::azure::EntityTracking::none)); + + auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); + plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); + plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(list_azure_blob_storage_, "Proxy Configuration Service", "ProxyConfigurationService"); + + test_controller_.runSession(plan_, true); + auto passed_params = mock_blob_storage_ptr_->getPassedListParams(); + REQUIRE(passed_params.proxy_configuration); + REQUIRE(passed_params.proxy_configuration->proxy_host == "host"); + REQUIRE(passed_params.proxy_configuration->proxy_port); + REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234); + REQUIRE(passed_params.proxy_configuration->proxy_user); + REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); + REQUIRE(passed_params.proxy_configuration->proxy_password); + REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); +} + } // namespace diff --git a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp index 4f2a19b512..caf85449be 100644 --- a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp @@ -256,4 +256,25 @@ TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "Both SAS Token and Stora REQUIRE_THROWS_AS(test_controller_.runSession(plan_, true), minifi::Exception); } +TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "List data lake storage files using proxy", "[azureDataLakeStorageParameters]") { + auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); + plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); + plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(list_azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService"); + + test_controller_.runSession(plan_, true); + + auto passed_params = mock_data_lake_storage_client_ptr_->getPassedListParams(); + REQUIRE(passed_params.proxy_configuration); + REQUIRE(passed_params.proxy_configuration->proxy_host == "host"); + REQUIRE(passed_params.proxy_configuration->proxy_port); + REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234); + REQUIRE(passed_params.proxy_configuration->proxy_user); + REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); + REQUIRE(passed_params.proxy_configuration->proxy_password); + REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); +} + } // namespace diff --git a/extensions/azure/tests/PutAzureBlobStorageTests.cpp b/extensions/azure/tests/PutAzureBlobStorageTests.cpp index a4a0ed19ee..19fa585dcd 100644 --- a/extensions/azure/tests/PutAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/PutAzureBlobStorageTests.cpp @@ -338,4 +338,28 @@ TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test Azure blob upload failur REQUIRE(failed_flowfiles[0] == TEST_DATA); } +TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test Azure blob storage put using proxy", "[azureBlobStorageUpload]") { + auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); + plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); + plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService"); + + plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container"); + plan_->setProperty(azure_blob_storage_processor_, "Blob", "test.blob"); + setDefaultCredentials(); + test_controller_.runSession(plan_, true); + auto passed_params = mock_blob_storage_ptr_->getPassedPutParams(); + REQUIRE(passed_params.proxy_configuration); + REQUIRE(passed_params.proxy_configuration->proxy_host == "host"); + REQUIRE(passed_params.proxy_configuration->proxy_port); + REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234); + REQUIRE(passed_params.proxy_configuration->proxy_user); + REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); + REQUIRE(passed_params.proxy_configuration->proxy_password); + REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + CHECK(getFailedFlowFileContents().empty()); +} + } // namespace diff --git a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp index 6938dc841f..32208afcd1 100644 --- a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp @@ -193,4 +193,26 @@ TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Upload to Azure Data Lake CHECK(verifyLogLinePresenceInPollTime(1s, "key:azure.directory value:\n")); } +TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Test Azure data lake storage upload using proxy", "[azureDataLakeStorageUpload]") { + auto proxy_configuration_service = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); + plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); + plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); + plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService"); + + test_controller_.runSession(plan_, true); + + auto passed_params = mock_data_lake_storage_client_ptr_->getPassedPutParams(); + REQUIRE(passed_params.proxy_configuration); + REQUIRE(passed_params.proxy_configuration->proxy_host == "host"); + REQUIRE(passed_params.proxy_configuration->proxy_port); + REQUIRE(*passed_params.proxy_configuration->proxy_port == 1234); + REQUIRE(passed_params.proxy_configuration->proxy_user); + REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); + REQUIRE(passed_params.proxy_configuration->proxy_password); + REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + CHECK(getFailedFlowFileContents().empty()); +} + } // namespace diff --git a/extensions/azure/tests/features/azure_storage.feature b/extensions/azure/tests/features/azure_storage.feature index 26fb71dc18..55e6af6237 100644 --- a/extensions/azure/tests/features/azure_storage.feature +++ b/extensions/azure/tests/features/azure_storage.feature @@ -108,3 +108,84 @@ Feature: Sending data from MiNiFi-C++ to an Azure storage server Then the Minifi logs contain the following message: "key:azure.blobname value:test_1" in less than 60 seconds Then the Minifi logs contain the following message: "key:azure.blobname value:test_2" in less than 60 seconds And the Minifi logs do not contain the following message: "key:azure.blobname value:other_test" after 0 seconds + + Scenario: A MiNiFi instance can upload data to Azure blob storage through a proxy + Given a GetFile processor with the "Input Directory" property set to "/tmp/input" + And a file with the content "#test_data$123$#" is present in "/tmp/input" + And a PutAzureBlobStorage processor set up to communicate with an Azure blob storage + And the "Proxy Configuration Service" property of the PutAzureBlobStorage processor is set to "ProxyConfigurationService" + And a PutFile processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFile processor is connected to the PutAzureBlobStorage + And the "success" relationship of the PutAzureBlobStorage processor is connected to the PutFile + And the "failure" relationship of the PutAzureBlobStorage processor is connected to the PutAzureBlobStorage + And a ProxyConfigurationService controller service is set up with HTTP proxy configuration + + And an Azure storage server is set up + And the http proxy server is set up + + When all instances start up + + Then a flowfile with the content "test" is placed in the monitored directory in less than 60 seconds + And the object on the Azure storage server is "#test_data$123$#" + And no errors were generated on the http-proxy regarding "http://azure-storage-server-${feature_id}:10000/devstoreaccount1/test-container/test-blob" + + Scenario: A MiNiFi instance can delete blob from Azure blob storage through a proxy + Given a GenerateFlowFile processor with the "File Size" property set to "0B" + And a DeleteAzureBlobStorage processor set up to communicate with an Azure blob storage + And the "Blob" property of the DeleteAzureBlobStorage processor is set to "test" + And the "Proxy Configuration Service" property of the DeleteAzureBlobStorage processor is set to "ProxyConfigurationService" + And the "success" relationship of the GenerateFlowFile processor is connected to the DeleteAzureBlobStorage + And a ProxyConfigurationService controller service is set up with HTTP proxy configuration + + And an Azure storage server is set up + And the http proxy server is set up + + When all instances start up + And test blob "test" is created on Azure blob storage + + Then the Azure blob storage becomes empty in 30 seconds + And no errors were generated on the http-proxy regarding "http://azure-storage-server-${feature_id}:10000/devstoreaccount1/test-container/test" + + Scenario: A MiNiFi instance can fetch a blob from Azure blob storage through a proxy + Given a GetFile processor with the "Input Directory" property set to "/tmp/input" + And the "Keep Source File" property of the GetFile processor is set to "true" + And a file with the content "dummy" is present in "/tmp/input" + And a FetchAzureBlobStorage processor set up to communicate with an Azure blob storage + And the "Blob" property of the FetchAzureBlobStorage processor is set to "test" + And the "Range Start" property of the FetchAzureBlobStorage processor is set to "6" + And the "Range Length" property of the FetchAzureBlobStorage processor is set to "5" + And the "Proxy Configuration Service" property of the FetchAzureBlobStorage processor is set to "ProxyConfigurationService" + And a PutFile processor with the "Directory" property set to "/tmp/output" + And the "success" relationship of the GetFile processor is connected to the FetchAzureBlobStorage + And the "success" relationship of the FetchAzureBlobStorage processor is connected to the PutFile + And a ProxyConfigurationService controller service is set up with HTTP proxy configuration + + And an Azure storage server is set up + And the http proxy server is set up + + When all instances start up + And test blob "test" with the content "#test_data$123$#" is created on Azure blob storage + + Then a flowfile with the content "data$" is placed in the monitored directory in less than 60 seconds + And no errors were generated on the http-proxy regarding "http://azure-storage-server-${feature_id}:10000/devstoreaccount1/test-container/test" + + Scenario: A MiNiFi instance can list a container on Azure blob storage through a proxy + Given a ListAzureBlobStorage processor set up to communicate with an Azure blob storage + And the "Prefix" property of the ListAzureBlobStorage processor is set to "test" + And the "Proxy Configuration Service" property of the ListAzureBlobStorage processor is set to "ProxyConfigurationService" + And a LogAttribute processor with the "FlowFiles To Log" property set to "0" + And the "success" relationship of the ListAzureBlobStorage processor is connected to the LogAttribute + And a ProxyConfigurationService controller service is set up with HTTP proxy configuration + + And an Azure storage server is set up + And the http proxy server is set up + + When all instances start up + And test blob "test_1" with the content "data_1" is created on Azure blob storage + And test blob "test_2" with the content "data_2" is created on Azure blob storage + And test blob "other_test" with the content "data_3" is created on Azure blob storage + + Then the Minifi logs contain the following message: "key:azure.blobname value:test_1" in less than 60 seconds + And the Minifi logs contain the following message: "key:azure.blobname value:test_2" in less than 60 seconds + And the Minifi logs do not contain the following message: "key:azure.blobname value:other_test" after 0 seconds + And no errors were generated on the http-proxy regarding "http://azure-storage-server-${feature_id}:10000/devstoreaccount1/test-container" diff --git a/libminifi/include/controllers/ProxyConfigurationService.h b/libminifi/include/controllers/ProxyConfigurationService.h new file mode 100644 index 0000000000..d8f9644071 --- /dev/null +++ b/libminifi/include/controllers/ProxyConfigurationService.h @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" +#include "core/controller/ControllerService.h" +#include "core/PropertyDefinitionBuilder.h" +#include "minifi-cpp/core/PropertyValidator.h" + +namespace org::apache::nifi::minifi::controllers { + +class ProxyConfigurationService : public core::controller::ControllerServiceImpl, public ProxyConfigurationServiceInterface { + public: + explicit ProxyConfigurationService(std::string_view name, const utils::Identifier& uuid = {}) + : ControllerServiceImpl(name, uuid) { + } + + MINIFIAPI static constexpr const char* Description = "Provides a set of configurations for different MiNiFi C++ components to use a proxy server."; + + MINIFIAPI static constexpr auto ProxyServerHost = core::PropertyDefinitionBuilder<>::createProperty("Proxy Server Host") + .withDescription("Proxy server hostname or ip-address.") + .isRequired(true) + .build(); + MINIFIAPI static constexpr auto ProxyServerPort = core::PropertyDefinitionBuilder<>::createProperty("Proxy Server Port") + .withDescription("Proxy server port number.") + .withValidator(core::StandardPropertyValidators::PORT_VALIDATOR) + .build(); + MINIFIAPI static constexpr auto ProxyUserName = core::PropertyDefinitionBuilder<>::createProperty("Proxy User Name") + .withDescription("The name of the proxy client for user authentication.") + .build(); + MINIFIAPI static constexpr auto ProxyUserPassword = core::PropertyDefinitionBuilder<>::createProperty("Proxy User Password") + .withDescription("The password of the proxy client for user authentication.") + .isSensitive(true) + .build(); + MINIFIAPI static constexpr auto Properties = std::to_array({ + ProxyServerHost, + ProxyServerPort, + ProxyUserName, + ProxyUserPassword + }); + + MINIFIAPI static constexpr bool SupportsDynamicProperties = false; + MINIFIAPI static constexpr auto ImplementsApis = std::array{ ProxyConfigurationServiceInterface::ProvidesApi }; + ADD_COMMON_VIRTUAL_FUNCTIONS_FOR_CONTROLLER_SERVICES + + void yield() override { + } + + bool isRunning() const override { + return getState() == core::controller::ControllerServiceState::ENABLED; + } + + bool isWorkAvailable() override { + return false; + } + + void initialize() override; + void onEnable() override; + + ProxyConfiguration getProxyConfiguration() const override { + std::lock_guard lock(configuration_mutex_); + return proxy_configuration_; + } + + private: + mutable std::mutex configuration_mutex_; + ProxyConfiguration proxy_configuration_; + std::shared_ptr logger_ = core::logging::LoggerFactory::getLogger(uuid_); +}; + +} // namespace org::apache::nifi::minifi::controllers diff --git a/libminifi/src/controllers/ProxyConfigurationService.cpp b/libminifi/src/controllers/ProxyConfigurationService.cpp new file mode 100644 index 0000000000..3fd0df4461 --- /dev/null +++ b/libminifi/src/controllers/ProxyConfigurationService.cpp @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "controllers/ProxyConfigurationService.h" + +#include "utils/ParsingUtils.h" +#include "minifi-cpp/Exception.h" +#include "core/Resource.h" + +namespace org::apache::nifi::minifi::controllers { + +void ProxyConfigurationService::initialize() { + setSupportedProperties(Properties); +} + +void ProxyConfigurationService::onEnable() { + std::lock_guard lock(configuration_mutex_); + proxy_configuration_.proxy_host = getProperty(ProxyServerHost.name).value_or(""); + if (proxy_configuration_.proxy_host.empty()) { + logger_->log_error("Proxy Server Host is required"); + throw minifi::Exception(ExceptionType::PROCESS_SCHEDULE_EXCEPTION, "Proxy Server Host is required"); + } + if (auto proxy_port = getProperty(ProxyServerPort.name) | utils::andThen(parsing::parseIntegral)) { + proxy_configuration_.proxy_port = *proxy_port; + } + if (auto proxy_user = getProperty(ProxyUserName.name)) { + proxy_configuration_.proxy_user = *proxy_user; + } + if (auto proxy_password = getProperty(ProxyUserPassword.name)) { + proxy_configuration_.proxy_password = *proxy_password; + } +} + +REGISTER_RESOURCE(ProxyConfigurationService, ControllerService); + +} // namespace org::apache::nifi::minifi::controllers diff --git a/libminifi/test/unit/ProxyConfigurationServiceTests.cpp b/libminifi/test/unit/ProxyConfigurationServiceTests.cpp new file mode 100644 index 0000000000..c85dbd9529 --- /dev/null +++ b/libminifi/test/unit/ProxyConfigurationServiceTests.cpp @@ -0,0 +1,65 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "unit/TestBase.h" +#include "unit/Catch.h" +#include "controllers/ProxyConfigurationService.h" + +namespace org::apache::nifi::minifi::test { + +struct ProxyConfigurationServiceTestFixture { + ProxyConfigurationServiceTestFixture() { + LogTestController::getInstance().clear(); + LogTestController::getInstance().setTrace(); + } + + TestController test_controller_; + std::shared_ptr plan_ = test_controller_.createPlan(); + std::shared_ptr proxy_configuration_node_ = plan_->addController("ProxyConfigurationService", "ProxyConfigurationService"); + std::shared_ptr proxy_configuration_service_ = + std::dynamic_pointer_cast(proxy_configuration_node_->getControllerServiceImplementation()); +}; + +TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "ProxyConfigurationService onEnable throws when empty") { + REQUIRE_THROWS_WITH(proxy_configuration_service_->onEnable(), "Process Schedule Operation: Proxy Server Host is required"); +} + +TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "Only required properties are set in ProxyConfigurationService") { + plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyServerHost, "192.168.1.123"); + REQUIRE_NOTHROW(plan_->finalize()); + auto proxy = proxy_configuration_service_->getProxyConfiguration(); + CHECK(proxy.proxy_host == "192.168.1.123"); + CHECK(proxy.proxy_port == std::nullopt); + CHECK(proxy.proxy_user == std::nullopt); + CHECK(proxy.proxy_password == std::nullopt); +} + +TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "All properties are set in ProxyConfigurationService") { + plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyServerHost, "192.168.1.123"); + plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyServerPort, "8080"); + plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyUserName, "user"); + plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyUserPassword, "password"); + REQUIRE_NOTHROW(plan_->finalize()); + auto proxy = proxy_configuration_service_->getProxyConfiguration(); + CHECK(proxy.proxy_host == "192.168.1.123"); + CHECK(proxy.proxy_port == 8080); + CHECK(proxy.proxy_user == "user"); + CHECK(proxy.proxy_password == "password"); +} + +} // namespace org::apache::nifi::minifi::test diff --git a/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h b/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h new file mode 100644 index 0000000000..85bf01caa3 --- /dev/null +++ b/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h @@ -0,0 +1,42 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include "minifi-cpp/core/controller/ControllerService.h" +#include "minifi-cpp/core/ControllerServiceApiDefinition.h" + +namespace org::apache::nifi::minifi::controllers { + +struct ProxyConfiguration { + std::string proxy_host; + std::optional proxy_port; + std::optional proxy_user; + std::optional proxy_password; +}; + +class ProxyConfigurationServiceInterface : public virtual core::controller::ControllerService { + public: + static constexpr auto ProvidesApi = core::ControllerServiceApiDefinition { + .artifact = "minifi-system", + .group = "org.apache.nifi.minifi", + .type = "org.apache.nifi.minifi.controllers.ProxyConfigurationServiceInterface", + }; + + virtual ProxyConfiguration getProxyConfiguration() const = 0; +}; + +} // namespace org::apache::nifi::minifi::controllers From 4a602b082890d03e0a7e25bc5309b677dfbaa6aa Mon Sep 17 00:00:00 2001 From: Gabor Gyimesi Date: Mon, 20 Oct 2025 09:18:26 +0200 Subject: [PATCH 2/7] Clang tidy fix --- extensions/aws/tests/S3TestsFixture.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extensions/aws/tests/S3TestsFixture.h b/extensions/aws/tests/S3TestsFixture.h index 24a570649e..68d01aac3c 100644 --- a/extensions/aws/tests/S3TestsFixture.h +++ b/extensions/aws/tests/S3TestsFixture.h @@ -136,7 +136,8 @@ class FlowProcessorS3TestsFixture : public S3TestsFixture { auto mock_s3_request_sender = std::make_unique(); this->mock_s3_request_sender_ptr = mock_s3_request_sender.get(); auto uuid = utils::IdGenerator::getIdGenerator()->generate(); - auto impl = std::unique_ptr(new T(core::ProcessorMetadata{.uuid = uuid, .name = "S3Processor", .logger = core::logging::LoggerFactory::getLogger(uuid)}, std::move(mock_s3_request_sender))); + auto impl = std::unique_ptr(new T(core::ProcessorMetadata{ // NOLINT(clang-analyzer-cplusplus.NewDeleteLeaks) + .uuid = uuid, .name = "S3Processor", .logger = core::logging::LoggerFactory::getLogger(uuid)}, std::move(mock_s3_request_sender))); auto s3_processor_unique_ptr = std::make_unique("S3Processor", uuid, std::move(impl)); this->s3_processor = s3_processor_unique_ptr.get(); From 7c9f1bb1f45259bd88ae0de4805ea1abdc44deaa Mon Sep 17 00:00:00 2001 From: Gabor Gyimesi Date: Thu, 13 Nov 2025 15:42:12 +0100 Subject: [PATCH 3/7] Review update --- CONTROLLERS.md | 2 +- libminifi/include/controllers/ProxyConfigurationService.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CONTROLLERS.md b/CONTROLLERS.md index 1537fc2231..952064b848 100644 --- a/CONTROLLERS.md +++ b/CONTROLLERS.md @@ -249,7 +249,7 @@ In the list below, the names of required properties appear in bold. Any other pr ### Description -Provides a set of configurations for different MiNiFi C++ components to use a proxy server. +Provides a set of configurations for different MiNiFi C++ components to use a proxy server. Currently these properties can only be used for HTTP proxy configuration, not other protocols are supported at this time. ### Properties diff --git a/libminifi/include/controllers/ProxyConfigurationService.h b/libminifi/include/controllers/ProxyConfigurationService.h index d8f9644071..5b13cd822a 100644 --- a/libminifi/include/controllers/ProxyConfigurationService.h +++ b/libminifi/include/controllers/ProxyConfigurationService.h @@ -31,7 +31,8 @@ class ProxyConfigurationService : public core::controller::ControllerServiceImpl : ControllerServiceImpl(name, uuid) { } - MINIFIAPI static constexpr const char* Description = "Provides a set of configurations for different MiNiFi C++ components to use a proxy server."; + MINIFIAPI static constexpr const char* Description = "Provides a set of configurations for different MiNiFi C++ components to use a proxy server. " + "Currently these properties can only be used for HTTP proxy configuration, not other protocols are supported at this time."; MINIFIAPI static constexpr auto ProxyServerHost = core::PropertyDefinitionBuilder<>::createProperty("Proxy Server Host") .withDescription("Proxy server hostname or ip-address.") From accaa4f1a79b3260296bcf66c106524a45ffa883 Mon Sep 17 00:00:00 2001 From: Gabor Gyimesi Date: Thu, 8 Jan 2026 17:13:06 +0100 Subject: [PATCH 4/7] Fix after rebase --- .../steps/flow_building_steps.py | 15 +++++ extensions/aws/tests/features/s3.feature | 58 ++++++++++--------- .../tests/features/azure_storage.feature | 12 ++-- 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py index 474dd5a0af..caa544a838 100644 --- a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py @@ -239,6 +239,21 @@ def step_impl(context: MinifiTestContext): processor.add_property(row["property name"], row["property value"]) +@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{container_name}\" flow") +def step_impl(context: MinifiTestContext, container_name: str): + controller_service = ControllerService(class_name="ProxyConfigurationService", service_name="ProxyConfigurationService") + controller_service.add_property("Proxy Server Host", f"http-proxy-{context.scenario_id}") + controller_service.add_property("Proxy Server Port", "3128") + controller_service.add_property("Proxy User Name", "admin") + controller_service.add_property("Proxy User Password", "test101") + context.get_or_create_minifi_container(container_name).flow_definition.controller_services.append(controller_service) + + +@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration") +def step_impl(context: MinifiTestContext): + context.execute_steps(f"given a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{DEFAULT_MINIFI_CONTAINER_NAME}\" flow") + + @step("the processors are connected up as described here") def step_impl(context: MinifiTestContext): for row in context.table: diff --git a/extensions/aws/tests/features/s3.feature b/extensions/aws/tests/features/s3.feature index 59ffa9f2fe..2fe2832cf8 100644 --- a/extensions/aws/tests/features/s3.feature +++ b/extensions/aws/tests/features/s3.feature @@ -44,7 +44,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And a PutS3Object processor set up to communicate with an s3 server And these processor properties are set | processor name | property name | property value | - | PutS3Object | Proxy Host | http-proxy-${scenario_id} | + | PutS3Object | Proxy Host | http-proxy-${scenario_id} | | PutS3Object | Proxy Port | 3128 | | PutS3Object | Proxy Username | admin | | PutS3Object | Proxy Password | test101 | @@ -61,7 +61,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server Then a single file with the content "LH_O#L|FD Date: Fri, 9 Jan 2026 17:09:51 +0100 Subject: [PATCH 5/7] Review update --- .../include/controllers/ProxyConfiguration.h | 34 +++++++++++++++++++ extensions/aws/processors/AwsProcessor.cpp | 12 ++++--- .../features/steps/s3_server_container.py | 2 +- .../processors/AzureStorageProcessorBase.cpp | 8 ++++- .../processors/AzureStorageProcessorBase.h | 1 + extensions/azure/storage/BlobStorageClient.h | 1 + .../azure/storage/DataLakeStorageClient.h | 1 + .../controllers/ProxyConfigurationService.h | 25 ++++++++++++-- .../controllers/ProxyConfigurationService.cpp | 1 + .../unit/ProxyConfigurationServiceTests.cpp | 18 +++++----- .../ProxyConfigurationServiceInterface.h | 13 +++---- 11 files changed, 91 insertions(+), 25 deletions(-) create mode 100644 core-framework/include/controllers/ProxyConfiguration.h diff --git a/core-framework/include/controllers/ProxyConfiguration.h b/core-framework/include/controllers/ProxyConfiguration.h new file mode 100644 index 0000000000..0090cf45d0 --- /dev/null +++ b/core-framework/include/controllers/ProxyConfiguration.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include + +#include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" + +namespace org::apache::nifi::minifi::controllers { + +struct ProxyConfiguration { + ProxyType proxy_type; + std::string proxy_host; + std::optional proxy_port; + std::optional proxy_user; + std::optional proxy_password; +}; + +} // namespace org::apache::nifi::minifi::controllers diff --git a/extensions/aws/processors/AwsProcessor.cpp b/extensions/aws/processors/AwsProcessor.cpp index 33157f1693..f2b2980f28 100644 --- a/extensions/aws/processors/AwsProcessor.cpp +++ b/extensions/aws/processors/AwsProcessor.cpp @@ -71,11 +71,13 @@ aws::ProxyOptions AwsProcessor::getProxy(core::ProcessContext& context, const co auto proxy_controller_service = minifi::utils::parseOptionalControllerService(context, ProxyConfigurationService, getUUID()); if (proxy_controller_service) { - auto controller_service_proxy = proxy_controller_service->getProxyConfiguration(); - proxy.host = controller_service_proxy.proxy_host; - proxy.port = controller_service_proxy.proxy_port ? *controller_service_proxy.proxy_port : 0; - proxy.username = controller_service_proxy.proxy_user ? *controller_service_proxy.proxy_user : ""; - proxy.password = controller_service_proxy.proxy_password ? *controller_service_proxy.proxy_password : ""; + proxy.host = proxy_controller_service->getHost(); + auto port_opt = proxy_controller_service->getPort(); + proxy.port = port_opt ? *port_opt : 0; + auto username_opt = proxy_controller_service->getUsername(); + proxy.username = username_opt ? *username_opt : ""; + auto password_opt = proxy_controller_service->getPassword(); + proxy.password = password_opt ? *password_opt : ""; } else { proxy.host = minifi::utils::parseOptionalProperty(context, ProxyHost, flow_file).value_or(""); proxy.port = gsl::narrow(minifi::utils::parseOptionalU64Property(context, ProxyPort, flow_file).value_or(0)); diff --git a/extensions/aws/tests/features/steps/s3_server_container.py b/extensions/aws/tests/features/steps/s3_server_container.py index 01db200301..42ac644545 100644 --- a/extensions/aws/tests/features/steps/s3_server_container.py +++ b/extensions/aws/tests/features/steps/s3_server_container.py @@ -31,7 +31,7 @@ def deploy(self): finished_str = "Started S3MockApplication" return wait_for_condition( condition=lambda: finished_str in self.get_logs(), - timeout_seconds=15, + timeout_seconds=60, bail_condition=lambda: self.exited, context=None) diff --git a/extensions/azure/processors/AzureStorageProcessorBase.cpp b/extensions/azure/processors/AzureStorageProcessorBase.cpp index f059dbfc36..32ac26dc60 100644 --- a/extensions/azure/processors/AzureStorageProcessorBase.cpp +++ b/extensions/azure/processors/AzureStorageProcessorBase.cpp @@ -33,7 +33,13 @@ void AzureStorageProcessorBase::onSchedule(core::ProcessContext& context, core:: auto proxy_controller_service = minifi::utils::parseOptionalControllerService(context, ProxyConfigurationService, getUUID()); if (proxy_controller_service) { logger_->log_debug("Proxy configuration is set for Azure Storage processor"); - proxy_configuration_ = proxy_controller_service->getProxyConfiguration(); + proxy_configuration_ = minifi::controllers::ProxyConfiguration{ + .proxy_type = minifi::controllers::ProxyType::HTTP, + .proxy_host = proxy_controller_service->getHost(), + .proxy_port = proxy_controller_service->getPort(), + .proxy_user = proxy_controller_service->getUsername(), + .proxy_password = proxy_controller_service->getPassword() + }; } } diff --git a/extensions/azure/processors/AzureStorageProcessorBase.h b/extensions/azure/processors/AzureStorageProcessorBase.h index 3e0cefcb2f..ca86dc58f1 100644 --- a/extensions/azure/processors/AzureStorageProcessorBase.h +++ b/extensions/azure/processors/AzureStorageProcessorBase.h @@ -33,6 +33,7 @@ #include "minifi-cpp/core/logging/Logger.h" #include "storage/AzureStorageCredentials.h" #include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" +#include "controllers/ProxyConfiguration.h" namespace org::apache::nifi::minifi::azure::processors { diff --git a/extensions/azure/storage/BlobStorageClient.h b/extensions/azure/storage/BlobStorageClient.h index a9c82947ba..4b1b0c3684 100644 --- a/extensions/azure/storage/BlobStorageClient.h +++ b/extensions/azure/storage/BlobStorageClient.h @@ -31,6 +31,7 @@ #include "utils/Enum.h" #include "minifi-cpp/io/InputStream.h" #include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" +#include "controllers/ProxyConfiguration.h" namespace org::apache::nifi::minifi::azure::storage { diff --git a/extensions/azure/storage/DataLakeStorageClient.h b/extensions/azure/storage/DataLakeStorageClient.h index fc94e5d214..5347f92850 100644 --- a/extensions/azure/storage/DataLakeStorageClient.h +++ b/extensions/azure/storage/DataLakeStorageClient.h @@ -32,6 +32,7 @@ #include "utils/Enum.h" #include "utils/RegexUtils.h" #include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" +#include "controllers/ProxyConfiguration.h" namespace org::apache::nifi::minifi::azure::storage { diff --git a/libminifi/include/controllers/ProxyConfigurationService.h b/libminifi/include/controllers/ProxyConfigurationService.h index 5b13cd822a..f4b1468afe 100644 --- a/libminifi/include/controllers/ProxyConfigurationService.h +++ b/libminifi/include/controllers/ProxyConfigurationService.h @@ -19,6 +19,7 @@ #include #include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" +#include "controllers/ProxyConfiguration.h" #include "core/controller/ControllerService.h" #include "core/PropertyDefinitionBuilder.h" #include "minifi-cpp/core/PropertyValidator.h" @@ -74,9 +75,29 @@ class ProxyConfigurationService : public core::controller::ControllerServiceImpl void initialize() override; void onEnable() override; - ProxyConfiguration getProxyConfiguration() const override { + ProxyType getProxyType() const override { std::lock_guard lock(configuration_mutex_); - return proxy_configuration_; + return proxy_configuration_.proxy_type; + } + + std::string getHost() const override { + std::lock_guard lock(configuration_mutex_); + return proxy_configuration_.proxy_host; + } + + std::optional getPort() const override { + std::lock_guard lock(configuration_mutex_); + return proxy_configuration_.proxy_port; + } + + std::optional getUsername() const override { + std::lock_guard lock(configuration_mutex_); + return proxy_configuration_.proxy_user; + } + + std::optional getPassword() const override { + std::lock_guard lock(configuration_mutex_); + return proxy_configuration_.proxy_password; } private: diff --git a/libminifi/src/controllers/ProxyConfigurationService.cpp b/libminifi/src/controllers/ProxyConfigurationService.cpp index 3fd0df4461..69564f3fb4 100644 --- a/libminifi/src/controllers/ProxyConfigurationService.cpp +++ b/libminifi/src/controllers/ProxyConfigurationService.cpp @@ -28,6 +28,7 @@ void ProxyConfigurationService::initialize() { void ProxyConfigurationService::onEnable() { std::lock_guard lock(configuration_mutex_); + proxy_configuration_.proxy_type = ProxyType::HTTP; proxy_configuration_.proxy_host = getProperty(ProxyServerHost.name).value_or(""); if (proxy_configuration_.proxy_host.empty()) { logger_->log_error("Proxy Server Host is required"); diff --git a/libminifi/test/unit/ProxyConfigurationServiceTests.cpp b/libminifi/test/unit/ProxyConfigurationServiceTests.cpp index c85dbd9529..8bc8f593af 100644 --- a/libminifi/test/unit/ProxyConfigurationServiceTests.cpp +++ b/libminifi/test/unit/ProxyConfigurationServiceTests.cpp @@ -42,11 +42,10 @@ TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "ProxyConfigurationServic TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "Only required properties are set in ProxyConfigurationService") { plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyServerHost, "192.168.1.123"); REQUIRE_NOTHROW(plan_->finalize()); - auto proxy = proxy_configuration_service_->getProxyConfiguration(); - CHECK(proxy.proxy_host == "192.168.1.123"); - CHECK(proxy.proxy_port == std::nullopt); - CHECK(proxy.proxy_user == std::nullopt); - CHECK(proxy.proxy_password == std::nullopt); + CHECK(proxy_configuration_service_->getHost() == "192.168.1.123"); + CHECK(proxy_configuration_service_->getPort() == std::nullopt); + CHECK(proxy_configuration_service_->getUsername() == std::nullopt); + CHECK(proxy_configuration_service_->getPassword() == std::nullopt); } TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "All properties are set in ProxyConfigurationService") { @@ -55,11 +54,10 @@ TEST_CASE_METHOD(ProxyConfigurationServiceTestFixture, "All properties are set i plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyUserName, "user"); plan_->setProperty(proxy_configuration_node_, controllers::ProxyConfigurationService::ProxyUserPassword, "password"); REQUIRE_NOTHROW(plan_->finalize()); - auto proxy = proxy_configuration_service_->getProxyConfiguration(); - CHECK(proxy.proxy_host == "192.168.1.123"); - CHECK(proxy.proxy_port == 8080); - CHECK(proxy.proxy_user == "user"); - CHECK(proxy.proxy_password == "password"); + CHECK(proxy_configuration_service_->getHost() == "192.168.1.123"); + CHECK(proxy_configuration_service_->getPort() == 8080); + CHECK(proxy_configuration_service_->getUsername() == "user"); + CHECK(proxy_configuration_service_->getPassword() == "password"); } } // namespace org::apache::nifi::minifi::test diff --git a/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h b/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h index 85bf01caa3..a39d9572a3 100644 --- a/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h +++ b/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h @@ -21,11 +21,8 @@ namespace org::apache::nifi::minifi::controllers { -struct ProxyConfiguration { - std::string proxy_host; - std::optional proxy_port; - std::optional proxy_user; - std::optional proxy_password; +enum class ProxyType { + HTTP }; class ProxyConfigurationServiceInterface : public virtual core::controller::ControllerService { @@ -36,7 +33,11 @@ class ProxyConfigurationServiceInterface : public virtual core::controller::Cont .type = "org.apache.nifi.minifi.controllers.ProxyConfigurationServiceInterface", }; - virtual ProxyConfiguration getProxyConfiguration() const = 0; + virtual std::string getHost() const = 0; + virtual std::optional getPort() const = 0; + virtual std::optional getUsername() const = 0; + virtual std::optional getPassword() const = 0; + virtual ProxyType getProxyType() const = 0; }; } // namespace org::apache::nifi::minifi::controllers From 615d49c0b7d5071392949fd19d66c8853cafc482 Mon Sep 17 00:00:00 2001 From: Gabor Gyimesi Date: Wed, 14 Jan 2026 14:33:49 +0100 Subject: [PATCH 6/7] Add HTTPS proxy support for AWS processors --- CONTROLLERS.md | 1 + .../containers/http_proxy_container.py | 22 +++-- .../containers/minifi_container.py | 2 + .../steps/flow_building_steps.py | 16 ++-- extensions/aws/processors/AwsProcessor.cpp | 24 +++--- extensions/aws/processors/AwsProcessor.h | 12 ++- extensions/aws/s3/S3Wrapper.h | 13 +-- extensions/aws/tests/S3TestsFixture.h | 5 ++ extensions/aws/tests/features/s3.feature | 86 +++++++++++++++---- extensions/aws/utils/ProxyOptions.h | 33 ------- .../processors/AzureStorageProcessorBase.cpp | 2 +- .../azure/storage/AzureBlobStorageClient.cpp | 12 ++- .../storage/AzureDataLakeStorageClient.cpp | 8 +- .../tests/DeleteAzureBlobStorageTests.cpp | 2 + .../tests/DeleteAzureDataLakeStorageTests.cpp | 3 + .../tests/FetchAzureBlobStorageTests.cpp | 2 + .../tests/FetchAzureDataLakeStorageTests.cpp | 2 + .../azure/tests/ListAzureBlobStorageTests.cpp | 2 + .../tests/ListAzureDataLakeStorageTests.cpp | 2 + .../azure/tests/PutAzureBlobStorageTests.cpp | 2 + .../tests/PutAzureDataLakeStorageTests.cpp | 2 + .../tests/features/azure_storage.feature | 9 +- .../controllers/ProxyConfigurationService.h | 8 +- .../controllers/ProxyConfigurationService.cpp | 2 +- .../ProxyConfigurationServiceInterface.h | 3 +- 25 files changed, 185 insertions(+), 90 deletions(-) delete mode 100644 extensions/aws/utils/ProxyOptions.h diff --git a/CONTROLLERS.md b/CONTROLLERS.md index 952064b848..41bf9678e8 100644 --- a/CONTROLLERS.md +++ b/CONTROLLERS.md @@ -261,6 +261,7 @@ In the list below, the names of required properties appear in bold. Any other pr | Proxy Server Port | | | Proxy server port number. | | Proxy User Name | | | The name of the proxy client for user authentication. | | Proxy User Password | | | The password of the proxy client for user authentication.
**Sensitive Property: true** | +| Proxy Type | HTTP | HTTP
HTTPS | Proxy type. | ## RocksDbStateStorage diff --git a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py index d07c674ff2..6412022a4f 100644 --- a/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py +++ b/behave_framework/src/minifi_test_framework/containers/http_proxy_container.py @@ -17,26 +17,34 @@ from textwrap import dedent +from OpenSSL import crypto from minifi_test_framework.containers.container import Container from minifi_test_framework.containers.docker_image_builder import DockerImageBuilder from minifi_test_framework.core.helpers import wait_for_condition from minifi_test_framework.core.minifi_test_context import MinifiTestContext +from minifi_test_framework.core.ssl_utils import make_server_cert +from minifi_test_framework.containers.file import File class HttpProxy(Container): def __init__(self, test_context: MinifiTestContext): dockerfile = dedent("""\ - FROM {base_image} - RUN apt -y update && apt install -y apache2-utils + FROM ubuntu:24.04 + RUN apt -y update && apt install -y squid-openssl apache2-utils RUN htpasswd -b -c /etc/squid/.squid_users {proxy_username} {proxy_password} RUN echo 'auth_param basic program /usr/lib/squid/basic_ncsa_auth /etc/squid/.squid_users' > /etc/squid/squid.conf && \ echo 'auth_param basic realm proxy' >> /etc/squid/squid.conf && \ echo 'acl authenticated proxy_auth REQUIRED' >> /etc/squid/squid.conf && \ + echo 'acl SSL_ports port 443' >> /etc/squid/squid.conf && \ + echo 'acl SSL_ports port 3002' >> /etc/squid/squid.conf && \ + echo 'acl Safe_ports port 80' >> /etc/squid/squid.conf && \ echo 'http_access allow authenticated' >> /etc/squid/squid.conf && \ echo 'http_port {proxy_port}' >> /etc/squid/squid.conf && \ - echo 'max_filedescriptors 1024' >> /etc/squid/squid.conf - """.format(base_image='ubuntu/squid:5.2-22.04_beta', proxy_username='admin', proxy_password='test101', - proxy_port='3128')) + echo 'max_filedescriptors 1024' >> /etc/squid/squid.conf && \ + echo 'https_port {proxy_ssl_port} tls-cert=/etc/squid/certs/squid-cert.pem tls-key=/etc/squid/certs/squid-key.pem' >> /etc/squid/squid.conf + ENTRYPOINT ["/bin/sh", "-c", "squid -N -f /etc/squid/squid.conf & tail -F --pid=$! /var/log/squid/cache.log /var/log/squid/access.log"] + """.format(proxy_username='admin', proxy_password='test101', + proxy_port='3128', proxy_ssl_port='3129')) builder = DockerImageBuilder( image_tag="minifi-http-proxy:latest", @@ -45,6 +53,10 @@ def __init__(self, test_context: MinifiTestContext): builder.build() super().__init__("minifi-http-proxy:latest", f"http-proxy-{test_context.scenario_id}", test_context.network) + squid_cert, squid_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + + self.files.append(File("/etc/squid/certs/squid-cert.pem", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=squid_cert), permissions=0o666)) + self.files.append(File("/etc/squid/certs/squid-key.pem", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=squid_key), permissions=0o666)) def deploy(self): super().deploy() diff --git a/behave_framework/src/minifi_test_framework/containers/minifi_container.py b/behave_framework/src/minifi_test_framework/containers/minifi_container.py index be72155e91..c1cdfd4805 100644 --- a/behave_framework/src/minifi_test_framework/containers/minifi_container.py +++ b/behave_framework/src/minifi_test_framework/containers/minifi_container.py @@ -39,6 +39,8 @@ def __init__(self, container_name: str, test_context: MinifiTestContext): minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) self.files.append(File("/usr/local/share/certs/ca-root-nss.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert))) self.files.append(File("/tmp/resources/root_ca.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert))) + self.files.append(File("/etc/ssl/certs/ca-certificates.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=test_context.root_ca_cert))) + self.files.append(File("/tmp/resources/minifi_client.crt", crypto.dump_certificate(type=crypto.FILETYPE_PEM, cert=minifi_client_cert))) self.files.append(File("/tmp/resources/minifi_client.key", crypto.dump_privatekey(type=crypto.FILETYPE_PEM, pkey=minifi_client_key))) diff --git a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py index caa544a838..c1c9ac5f7e 100644 --- a/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py +++ b/behave_framework/src/minifi_test_framework/steps/flow_building_steps.py @@ -239,19 +239,23 @@ def step_impl(context: MinifiTestContext): processor.add_property(row["property name"], row["property value"]) -@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{container_name}\" flow") -def step_impl(context: MinifiTestContext, container_name: str): +@given("a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the \"{container_name}\" flow") +def step_impl(context: MinifiTestContext, proxy_type: str, container_name: str): controller_service = ControllerService(class_name="ProxyConfigurationService", service_name="ProxyConfigurationService") controller_service.add_property("Proxy Server Host", f"http-proxy-{context.scenario_id}") - controller_service.add_property("Proxy Server Port", "3128") controller_service.add_property("Proxy User Name", "admin") controller_service.add_property("Proxy User Password", "test101") + controller_service.add_property("Proxy Type", proxy_type) + if proxy_type.lower() == "http": + controller_service.add_property("Proxy Server Port", "3128") + else: + controller_service.add_property("Proxy Server Port", "3129") context.get_or_create_minifi_container(container_name).flow_definition.controller_services.append(controller_service) -@given("a ProxyConfigurationService controller service is set up with HTTP proxy configuration") -def step_impl(context: MinifiTestContext): - context.execute_steps(f"given a ProxyConfigurationService controller service is set up with HTTP proxy configuration in the \"{DEFAULT_MINIFI_CONTAINER_NAME}\" flow") +@given("a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration") +def step_impl(context: MinifiTestContext, proxy_type: str): + context.execute_steps(f"given a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the \"{DEFAULT_MINIFI_CONTAINER_NAME}\" flow") @step("the processors are connected up as described here") diff --git a/extensions/aws/processors/AwsProcessor.cpp b/extensions/aws/processors/AwsProcessor.cpp index f2b2980f28..f59c531a15 100644 --- a/extensions/aws/processors/AwsProcessor.cpp +++ b/extensions/aws/processors/AwsProcessor.cpp @@ -66,26 +66,28 @@ std::optional AwsProcessor::getAWSCredentials( return aws_credentials_provider.getAWSCredentials(); } -aws::ProxyOptions AwsProcessor::getProxy(core::ProcessContext& context, const core::FlowFile* const flow_file) { - aws::ProxyOptions proxy; +minifi::controllers::ProxyConfiguration AwsProcessor::getProxy(core::ProcessContext& context, const core::FlowFile* const flow_file) { + minifi::controllers::ProxyConfiguration proxy; auto proxy_controller_service = minifi::utils::parseOptionalControllerService(context, ProxyConfigurationService, getUUID()); if (proxy_controller_service) { - proxy.host = proxy_controller_service->getHost(); + proxy.proxy_type = proxy_controller_service->getProxyType(); + proxy.proxy_host = proxy_controller_service->getHost(); auto port_opt = proxy_controller_service->getPort(); - proxy.port = port_opt ? *port_opt : 0; + proxy.proxy_port = port_opt.value_or(0); auto username_opt = proxy_controller_service->getUsername(); - proxy.username = username_opt ? *username_opt : ""; + proxy.proxy_user = username_opt.value_or(""); auto password_opt = proxy_controller_service->getPassword(); - proxy.password = password_opt ? *password_opt : ""; + proxy.proxy_password = password_opt.value_or(""); } else { - proxy.host = minifi::utils::parseOptionalProperty(context, ProxyHost, flow_file).value_or(""); - proxy.port = gsl::narrow(minifi::utils::parseOptionalU64Property(context, ProxyPort, flow_file).value_or(0)); - proxy.username = minifi::utils::parseOptionalProperty(context, ProxyUsername, flow_file).value_or(""); - proxy.password = minifi::utils::parseOptionalProperty(context, ProxyPassword, flow_file).value_or(""); + proxy.proxy_type = minifi::utils::parseOptionalEnumProperty(context, ProxyType).value_or(minifi::controllers::ProxyType::HTTP); + proxy.proxy_host = minifi::utils::parseOptionalProperty(context, ProxyHost, flow_file).value_or(""); + proxy.proxy_port = gsl::narrow(minifi::utils::parseOptionalU64Property(context, ProxyPort, flow_file).value_or(0)); + proxy.proxy_user = minifi::utils::parseOptionalProperty(context, ProxyUsername, flow_file).value_or(""); + proxy.proxy_password = minifi::utils::parseOptionalProperty(context, ProxyPassword, flow_file).value_or(""); } - if (!proxy.host.empty()) { + if (!proxy.proxy_host.empty()) { logger_->log_info("Proxy for AwsProcessor was set."); } return proxy; diff --git a/extensions/aws/processors/AwsProcessor.h b/extensions/aws/processors/AwsProcessor.h index 548f332795..0a929bab95 100644 --- a/extensions/aws/processors/AwsProcessor.h +++ b/extensions/aws/processors/AwsProcessor.h @@ -28,12 +28,12 @@ #include "aws/core/auth/AWSCredentialsProvider.h" #include "AWSCredentialsProvider.h" -#include "utils/ProxyOptions.h" #include "minifi-cpp/core/PropertyDefinition.h" #include "core/PropertyDefinitionBuilder.h" #include "minifi-cpp/core/PropertyValidator.h" #include "core/ProcessorImpl.h" #include "minifi-cpp/controllers/ProxyConfigurationServiceInterface.h" +#include "controllers/ProxyConfiguration.h" namespace org::apache::nifi::minifi::aws::processors { @@ -91,7 +91,7 @@ inline constexpr auto REGIONS = std::array{ struct CommonProperties { Aws::Auth::AWSCredentials credentials; - aws::ProxyOptions proxy; + minifi::controllers::ProxyConfiguration proxy; std::string endpoint_override_url; }; @@ -132,6 +132,11 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s .withValidator(core::StandardPropertyValidators::NON_BLANK_VALIDATOR) .supportsExpressionLanguage(true) .build(); + EXTENSIONAPI static constexpr auto ProxyType = core::PropertyDefinitionBuilder()>::createProperty("Proxy Type") + .withDescription("Proxy type") + .withDefaultValue(magic_enum::enum_name(minifi::controllers::ProxyType::HTTP)) + .withAllowedValues(magic_enum::enum_names()) + .build(); EXTENSIONAPI static constexpr auto ProxyHost = core::PropertyDefinitionBuilder<>::createProperty("Proxy Host") .withDescription("Proxy host name or IP") .supportsExpressionLanguage(true) @@ -168,6 +173,7 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s Region, CommunicationsTimeout, EndpointOverrideURL, + ProxyType, ProxyHost, ProxyPort, ProxyUsername, @@ -184,7 +190,7 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s protected: std::optional getAWSCredentialsFromControllerService(core::ProcessContext& context) const; std::optional getAWSCredentials(core::ProcessContext& context, const core::FlowFile* flow_file); - aws::ProxyOptions getProxy(core::ProcessContext& context, const core::FlowFile* const flow_file); + minifi::controllers::ProxyConfiguration getProxy(core::ProcessContext& context, const core::FlowFile* const flow_file); std::optional getCommonELSupportedProperties(core::ProcessContext& context, const core::FlowFile* flow_file); std::optional client_config_; diff --git a/extensions/aws/s3/S3Wrapper.h b/extensions/aws/s3/S3Wrapper.h index ab92e8b4ed..9059b1fec1 100644 --- a/extensions/aws/s3/S3Wrapper.h +++ b/extensions/aws/s3/S3Wrapper.h @@ -46,7 +46,7 @@ #include "utils/OptionalUtils.h" #include "utils/StringUtils.h" #include "minifi-cpp/utils/gsl.h" -#include "utils/ProxyOptions.h" +#include "controllers/ProxyConfiguration.h" namespace org::apache::nifi::minifi::aws::s3 { @@ -126,11 +126,12 @@ struct RequestParameters { Aws::Auth::AWSCredentials credentials; Aws::Client::ClientConfiguration client_config; - void setClientConfig(const aws::ProxyOptions& proxy, const std::string& endpoint_override_url) { - client_config.proxyHost = proxy.host; - client_config.proxyPort = proxy.port; - client_config.proxyUserName = proxy.username; - client_config.proxyPassword = proxy.password; + void setClientConfig(const minifi::controllers::ProxyConfiguration& proxy, const std::string& endpoint_override_url) { + client_config.proxyHost = proxy.proxy_host; + client_config.proxyPort = proxy.proxy_port.value_or(0); + client_config.proxyUserName = proxy.proxy_user.value_or(""); + client_config.proxyPassword = proxy.proxy_password.value_or(""); + client_config.proxyScheme = proxy.proxy_type == minifi::controllers::ProxyType::HTTPS ? Aws::Http::Scheme::HTTPS : Aws::Http::Scheme::HTTP; client_config.endpointOverride = endpoint_override_url; } }; diff --git a/extensions/aws/tests/S3TestsFixture.h b/extensions/aws/tests/S3TestsFixture.h index 68d01aac3c..77b9f79c38 100644 --- a/extensions/aws/tests/S3TestsFixture.h +++ b/extensions/aws/tests/S3TestsFixture.h @@ -104,6 +104,7 @@ class S3TestsFixture { } void checkProxySettings() { + REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyScheme == Aws::Http::Scheme::HTTPS); REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyHost == "host"); REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyPort == 1234); REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyUserName == "username"); @@ -182,12 +183,14 @@ class FlowProcessorS3TestsFixture : public S3TestsFixture { void setProxy(bool use_controller_service) override { if (use_controller_service) { auto proxy_configuration_service = this->plan->addController("ProxyConfigurationService", "ProxyConfigurationService"); + this->plan->setProperty(proxy_configuration_service, "Proxy Type", "HTTPS"); this->plan->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); this->plan->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); this->plan->setProperty(proxy_configuration_service, "Proxy User Name", "username"); this->plan->setProperty(proxy_configuration_service, "Proxy User Password", "password"); this->plan->setProperty(this->s3_processor, "Proxy Configuration Service", "ProxyConfigurationService"); } else { + this->plan->setProperty(this->s3_processor, "Proxy Type", "HTTPS"); this->plan->setDynamicProperty(update_attribute, "test.proxyHost", "host"); this->plan->setProperty(this->s3_processor, "Proxy Host", "${test.proxyHost}"); this->plan->setDynamicProperty(update_attribute, "test.proxyPort", "1234"); @@ -237,12 +240,14 @@ class FlowProducerS3TestsFixture : public S3TestsFixture { void setProxy(bool use_controller_service) override { if (use_controller_service) { auto proxy_configuration_service = this->plan->addController("ProxyConfigurationService", "ProxyConfigurationService"); + this->plan->setProperty(proxy_configuration_service, "Proxy Type", "HTTPS"); this->plan->setProperty(proxy_configuration_service, "Proxy Server Host", "host"); this->plan->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); this->plan->setProperty(proxy_configuration_service, "Proxy User Name", "username"); this->plan->setProperty(proxy_configuration_service, "Proxy User Password", "password"); this->plan->setProperty(this->s3_processor, "Proxy Configuration Service", "ProxyConfigurationService"); } else { + this->plan->setProperty(this->s3_processor, "Proxy Type", "HTTPS"); this->plan->setProperty(this->s3_processor, "Proxy Host", "host"); this->plan->setProperty(this->s3_processor, "Proxy Port", "1234"); this->plan->setProperty(this->s3_processor, "Proxy Username", "username"); diff --git a/extensions/aws/tests/features/s3.feature b/extensions/aws/tests/features/s3.feature index 2fe2832cf8..f49e39456f 100644 --- a/extensions/aws/tests/features/s3.feature +++ b/extensions/aws/tests/features/s3.feature @@ -38,16 +38,17 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And the object content type on the s3 server is "application/octet-stream" and the object metadata matches use metadata And the Minifi logs contain the following message: "in a single upload" in less than 10 seconds - Scenario: A MiNiFi instance transfers encoded data through a http proxy to s3 + Scenario Outline: A MiNiFi instance transfers encoded data through a http proxy to s3 Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And a directory at "/tmp/input" has a file with the content "LH_O#L|FD | | PutS3Object | Proxy Username | admin | | PutS3Object | Proxy Password | test101 | + | PutS3Object | Proxy Type | | And a PutFile processor with the "Directory" property set to "/tmp/output" And the "success" relationship of the GetFile processor is connected to the PutS3Object And the "success" relationship of the PutS3Object processor is connected to the PutFile @@ -63,7 +64,12 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And the object content type on the s3 server is "application/octet-stream" and the object metadata matches use metadata And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket/test_object_key" - Scenario: A MiNiFi instance transfers encoded data through a http proxy to s3 using proxy configuration service + Examples: Proxy Type + | proxy type | proxy port | + | HTTP | 3128 | + | HTTPS | 3129 | + + Scenario Outline: A MiNiFi instance transfers encoded data through a http proxy to s3 using proxy configuration service Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And a file with the content "LH_O#L|FD proxy configuration And a s3 server is set up in correspondence with the PutS3Object And the http proxy server is set up @@ -82,6 +88,11 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And the object content type on the s3 server is "application/octet-stream" and the object metadata matches use metadata And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket/test_object_key" + Examples: Proxy Type + | proxy type | + | HTTP | + | HTTPS | + Scenario: A MiNiFi instance can remove s3 bucket objects Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And a directory at "/tmp/input" has a file with the content "LH_O#L|FD | | DeleteS3Object | Proxy Username | admin | | DeleteS3Object | Proxy Password | test101 | + | DeleteS3Object | Proxy Type | | And the processors are connected up as described here | source name | relationship name | destination name | | GetFile | success | PutS3Object | @@ -148,7 +160,12 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And the object bucket on the s3 server is empty in less than 10 seconds And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket/test_object_key" - Scenario: Deletion of a s3 object through a proxy-server succeeds using proxy configuration service + Examples: Proxy Type + | proxy type | proxy port | + | HTTP | 3128 | + | HTTPS | 3129 | + + Scenario Outline: Deletion of a s3 object through a proxy-server succeeds using proxy configuration service Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And the scheduling period of the GetFile processor is set to "60 sec" And a file with the content "LH_O#L|FD proxy configuration And a s3 server is set up in correspondence with the PutS3Object And the http proxy server is set up @@ -172,6 +189,11 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And the object bucket on the s3 server is empty in less than 10 seconds And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket/test_object_key" + Examples: Proxy Type + | proxy type | + | HTTP | + | HTTPS | + Scenario: A MiNiFi instance can download s3 bucket objects directly Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And a directory at "/tmp/input" has a file with the content "test" @@ -195,7 +217,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server Then a single file with the content "test" is placed in the "/tmp/output" directory in less than 20 seconds - Scenario: A MiNiFi instance can download s3 bucket objects via a http-proxy + Scenario Outline: A MiNiFi instance can download s3 bucket objects via a http-proxy Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And a directory at "/tmp/input" has a file with the content "test" And a PutS3Object processor set up to communicate with an s3 server @@ -206,9 +228,10 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And these processor properties are set | processor name | property name | property value | | FetchS3Object | Proxy Host | http-proxy-${scenario_id} | - | FetchS3Object | Proxy Port | 3128 | + | FetchS3Object | Proxy Port | | | FetchS3Object | Proxy Username | admin | | FetchS3Object | Proxy Password | test101 | + | FetchS3Object | Proxy Type | | And a PutFile processor with the "Directory" property set to "/tmp/output" And the processors are connected up as described here | source name | relationship name | destination name | @@ -223,7 +246,12 @@ Feature: Sending data from MiNiFi-C++ to an AWS server Then a single file with the content "test" is placed in the "/tmp/output" directory in less than 20 seconds And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket/test_object_key" - Scenario: A MiNiFi instance can download s3 bucket objects via a http-proxy using proxy configuration service + Examples: Proxy Type + | proxy type | proxy port | + | HTTP | 3128 | + | HTTPS | 3129 | + + Scenario Outline: A MiNiFi instance can download s3 bucket objects via a http-proxy using proxy configuration service Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And a file with the content "test" is present in "/tmp/input" And a PutS3Object processor set up to communicate with an s3 server @@ -237,7 +265,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server | source name | relationship name | destination name | | GenerateFlowFile | success | FetchS3Object | | FetchS3Object | success | PutFile | - And a ProxyConfigurationService controller service is set up with HTTP proxy configuration + And a ProxyConfigurationService controller service is set up with proxy configuration And a s3 server is set up in correspondence with the PutS3Object And the http proxy server is set up @@ -247,6 +275,11 @@ Feature: Sending data from MiNiFi-C++ to an AWS server Then a single file with the content "test" is placed in the "/tmp/output" directory in less than 60 seconds And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket/test_object_key" + Examples: Proxy Type + | proxy type | + | HTTP | + | HTTPS | + Scenario: A MiNiFi instance can list an S3 bucket directly Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And the "Batch Size" property of the GetFile processor is set to "1" @@ -267,7 +300,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server Then 2 files are placed in the "/tmp/output" directory in less than 20 seconds - Scenario: A MiNiFi instance can list an S3 bucket objects via a http-proxy + Scenario Outline: A MiNiFi instance can list an S3 bucket objects via a http-proxy Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And a directory at "/tmp/input" has a file with the content "test" And a PutS3Object processor set up to communicate with an s3 server @@ -277,9 +310,10 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And these processor properties are set | processor name | property name | property value | | ListS3 | Proxy Host | http-proxy-${scenario_id} | - | ListS3 | Proxy Port | 3128 | + | ListS3 | Proxy Port | | | ListS3 | Proxy Username | admin | | ListS3 | Proxy Password | test101 | + | ListS3 | Proxy Type | | And a PutFile processor with the "Directory" property set to "/tmp/output" And the "success" relationship of the ListS3 processor is connected to the PutFile @@ -291,7 +325,12 @@ Feature: Sending data from MiNiFi-C++ to an AWS server Then 1 file is placed in the "/tmp/output" directory in less than 20 seconds And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket" - Scenario: A MiNiFi instance can list an S3 bucket objects via a http-proxy using proxy configuration service + Examples: Proxy Type + | proxy port | proxy type | + | 3128 | HTTP | + | 3129 | HTTPS | + + Scenario Outline: A MiNiFi instance can list an S3 bucket objects via a http-proxy using proxy configuration service Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And a file with the content "test" is present in "/tmp/input" And a PutS3Object processor set up to communicate with an s3 server @@ -301,7 +340,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And the "Proxy Configuration Service" property of the ListS3 processor is set to "ProxyConfigurationService" And a PutFile processor with the "Directory" property set to "/tmp/output" And the "success" relationship of the ListS3 processor is connected to the PutFile - And a ProxyConfigurationService controller service is set up with HTTP proxy configuration + And a ProxyConfigurationService controller service is set up with proxy configuration And a s3 server is set up in correspondence with the PutS3Object And the http proxy server is set up @@ -311,6 +350,11 @@ Feature: Sending data from MiNiFi-C++ to an AWS server Then 1 file is placed in the "/tmp/output" directory in less than 120 seconds And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket" + Examples: Proxy Type + | proxy type | + | HTTP | + | HTTPS | + Scenario: A MiNiFi instance transfers data in multiple parts to s3 Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And there is a 6MB file at the "/tmp/input" directory and we keep track of the hash of that @@ -328,7 +372,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And the object on the s3 server is present and matches the original hash And the Minifi logs contain the following message: "passes the multipart threshold, uploading it in multiple parts" in less than 10 seconds - Scenario: A MiNiFi instance can use multipart upload through http proxy to s3 + Scenario Outline: A MiNiFi instance can use multipart upload through http proxy to s3 Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And there is a 6MB file at the "/tmp/input" directory and we keep track of the hash of that And a PutS3Object processor set up to communicate with an s3 server @@ -337,9 +381,10 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And these processor properties are set | processor name | property name | property value | | PutS3Object | Proxy Host | http-proxy-${scenario_id} | - | PutS3Object | Proxy Port | 3128 | + | PutS3Object | Proxy Port | | | PutS3Object | Proxy Username | admin | | PutS3Object | Proxy Password | test101 | + | PutS3Object | Proxy Type | | And the "success" relationship of the GetFile processor is connected to the PutS3Object And a PutFile processor with the "Directory" property set to "/tmp/output" And the "success" relationship of the PutS3Object processor is connected to the PutFile @@ -352,3 +397,8 @@ Feature: Sending data from MiNiFi-C++ to an AWS server And the object on the s3 server is present and matches the original hash And the Minifi logs contain the following message: "passes the multipart threshold, uploading it in multiple parts" in less than 10 seconds And no errors were generated on the http-proxy regarding "http://s3-server-${scenario_id}:9090/test_bucket/test_object_key" + + Examples: Proxy Type + | proxy port | proxy type | + | 3128 | HTTP | + | 3129 | HTTPS | diff --git a/extensions/aws/utils/ProxyOptions.h b/extensions/aws/utils/ProxyOptions.h deleted file mode 100644 index 8227c93b06..0000000000 --- a/extensions/aws/utils/ProxyOptions.h +++ /dev/null @@ -1,33 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#pragma once - -#include -#include - - -namespace org::apache::nifi::minifi::aws { - -struct ProxyOptions { - std::string host; - uint32_t port = 0; - std::string username; - std::string password; -}; - -} // namespace org::apache::nifi::minifi::aws diff --git a/extensions/azure/processors/AzureStorageProcessorBase.cpp b/extensions/azure/processors/AzureStorageProcessorBase.cpp index 32ac26dc60..f4f1d8d4cc 100644 --- a/extensions/azure/processors/AzureStorageProcessorBase.cpp +++ b/extensions/azure/processors/AzureStorageProcessorBase.cpp @@ -34,7 +34,7 @@ void AzureStorageProcessorBase::onSchedule(core::ProcessContext& context, core:: if (proxy_controller_service) { logger_->log_debug("Proxy configuration is set for Azure Storage processor"); proxy_configuration_ = minifi::controllers::ProxyConfiguration{ - .proxy_type = minifi::controllers::ProxyType::HTTP, + .proxy_type = proxy_controller_service->getProxyType(), .proxy_host = proxy_controller_service->getHost(), .proxy_port = proxy_controller_service->getPort(), .proxy_user = proxy_controller_service->getUsername(), diff --git a/extensions/azure/storage/AzureBlobStorageClient.cpp b/extensions/azure/storage/AzureBlobStorageClient.cpp index b4539ff4db..c9327ec663 100644 --- a/extensions/azure/storage/AzureBlobStorageClient.cpp +++ b/extensions/azure/storage/AzureBlobStorageClient.cpp @@ -28,6 +28,7 @@ #include "utils/AzureSdkLogger.h" #include "utils/span.h" #include "io/InputStream.h" +#include "utils/RegexUtils.h" namespace org::apache::nifi::minifi::azure::storage { @@ -61,7 +62,16 @@ Azure::Storage::Blobs::BlobContainerClient AzureBlobStorageClient::createClient( Azure::Storage::Blobs::BlobClientOptions client_options; if (proxy_configuration) { - client_options.Transport.HttpProxy = proxy_configuration->proxy_host + (proxy_configuration->proxy_port ? (":" + std::to_string(*proxy_configuration->proxy_port)) : ""); + std::string protocol_prefix; + if (!minifi::utils::regexMatch(proxy_configuration->proxy_host, minifi::utils::Regex("^https?:\\/\\/"))) { + if (proxy_configuration->proxy_type == controllers::ProxyType::HTTP) { + protocol_prefix = "http://"; + } else if (proxy_configuration->proxy_type == controllers::ProxyType::HTTPS) { + protocol_prefix = "https://"; + } + } + client_options.Transport.HttpProxy = protocol_prefix + proxy_configuration->proxy_host + (proxy_configuration->proxy_port ? (":" + std::to_string(*proxy_configuration->proxy_port)) : ""); + if (proxy_configuration->proxy_user) { client_options.Transport.ProxyUserName = *proxy_configuration->proxy_user; } diff --git a/extensions/azure/storage/AzureDataLakeStorageClient.cpp b/extensions/azure/storage/AzureDataLakeStorageClient.cpp index ace617b06f..1a89c82d19 100644 --- a/extensions/azure/storage/AzureDataLakeStorageClient.cpp +++ b/extensions/azure/storage/AzureDataLakeStorageClient.cpp @@ -43,7 +43,13 @@ std::unique_ptr Azure } if (proxy_configuration) { - options.Transport.HttpProxy = proxy_configuration->proxy_host + (proxy_configuration->proxy_port ? (":" + std::to_string(*proxy_configuration->proxy_port)) : ""); + std::string protocol_prefix; + if (proxy_configuration->proxy_type == controllers::ProxyType::HTTP && !minifi::utils::string::startsWith(proxy_configuration->proxy_host, "http://")) { + protocol_prefix = "http://"; + } else if (proxy_configuration->proxy_type == controllers::ProxyType::HTTPS && !minifi::utils::string::startsWith(proxy_configuration->proxy_host, "https://")) { + protocol_prefix = "https://"; + } + options.Transport.HttpProxy = protocol_prefix + proxy_configuration->proxy_host + (proxy_configuration->proxy_port ? (":" + std::to_string(*proxy_configuration->proxy_port)) : ""); if (proxy_configuration->proxy_user) { options.Transport.ProxyUserName = *proxy_configuration->proxy_user; } diff --git a/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp b/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp index cd91a592df..7052b20592 100644 --- a/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/DeleteAzureBlobStorageTests.cpp @@ -345,6 +345,7 @@ TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test Azure blob delete usi plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP"); plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService"); plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container"); @@ -360,6 +361,7 @@ TEST_CASE_METHOD(DeleteAzureBlobStorageTestsFixture, "Test Azure blob delete usi REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); REQUIRE(passed_params.proxy_configuration->proxy_password); REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP); CHECK(getFailedFlowFileContents().empty()); } diff --git a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp index 4f2a8e1698..600e20b52d 100644 --- a/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/DeleteAzureDataLakeStorageTests.cpp @@ -159,6 +159,7 @@ TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Test Azure data lake s plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP"); plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService"); test_controller_.runSession(plan_, true); @@ -172,6 +173,8 @@ TEST_CASE_METHOD(DeleteAzureDataLakeStorageTestsFixture, "Test Azure data lake s REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); REQUIRE(passed_params.proxy_configuration->proxy_password); REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP); + CHECK(getFailedFlowFileContents().empty()); } diff --git a/extensions/azure/tests/FetchAzureBlobStorageTests.cpp b/extensions/azure/tests/FetchAzureBlobStorageTests.cpp index 1a145937d6..57ed393be7 100644 --- a/extensions/azure/tests/FetchAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/FetchAzureBlobStorageTests.cpp @@ -335,6 +335,7 @@ TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Test Azure blob fetch using plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP"); plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService"); plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container"); @@ -350,6 +351,7 @@ TEST_CASE_METHOD(FetchAzureBlobStorageTestsFixture, "Test Azure blob fetch using REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); REQUIRE(passed_params.proxy_configuration->proxy_password); REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP); CHECK(getFailedFlowFileContents().empty()); } diff --git a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp index 7a98f28c84..aea9da732d 100644 --- a/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/FetchAzureDataLakeStorageTests.cpp @@ -176,6 +176,7 @@ TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Test Azure data lake st plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP"); plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService"); test_controller_.runSession(plan_, true); @@ -189,6 +190,7 @@ TEST_CASE_METHOD(FetchAzureDataLakeStorageTestsFixture, "Test Azure data lake st REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); REQUIRE(passed_params.proxy_configuration->proxy_password); REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP); CHECK(getFailedFlowFileContents().empty()); } diff --git a/extensions/azure/tests/ListAzureBlobStorageTests.cpp b/extensions/azure/tests/ListAzureBlobStorageTests.cpp index 59be4b4a97..4b9db71e84 100644 --- a/extensions/azure/tests/ListAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/ListAzureBlobStorageTests.cpp @@ -360,6 +360,7 @@ TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "List all files through a pro plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP"); plan_->setProperty(list_azure_blob_storage_, "Proxy Configuration Service", "ProxyConfigurationService"); test_controller_.runSession(plan_, true); @@ -372,6 +373,7 @@ TEST_CASE_METHOD(ListAzureBlobStorageTestsFixture, "List all files through a pro REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); REQUIRE(passed_params.proxy_configuration->proxy_password); REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP); } } // namespace diff --git a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp index caf85449be..5b635bd35c 100644 --- a/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/ListAzureDataLakeStorageTests.cpp @@ -262,6 +262,7 @@ TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "List data lake storage f plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTPS"); plan_->setProperty(list_azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService"); test_controller_.runSession(plan_, true); @@ -275,6 +276,7 @@ TEST_CASE_METHOD(ListAzureDataLakeStorageTestsFixture, "List data lake storage f REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); REQUIRE(passed_params.proxy_configuration->proxy_password); REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTPS); } } // namespace diff --git a/extensions/azure/tests/PutAzureBlobStorageTests.cpp b/extensions/azure/tests/PutAzureBlobStorageTests.cpp index 19fa585dcd..2d564b5eb1 100644 --- a/extensions/azure/tests/PutAzureBlobStorageTests.cpp +++ b/extensions/azure/tests/PutAzureBlobStorageTests.cpp @@ -344,6 +344,7 @@ TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test Azure blob storage put u plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTPS"); plan_->setProperty(azure_blob_storage_processor_, "Proxy Configuration Service", "ProxyConfigurationService"); plan_->setProperty(azure_blob_storage_processor_, "Container Name", "test.container"); @@ -359,6 +360,7 @@ TEST_CASE_METHOD(PutAzureBlobStorageTestsFixture, "Test Azure blob storage put u REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); REQUIRE(passed_params.proxy_configuration->proxy_password); REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTPS); CHECK(getFailedFlowFileContents().empty()); } diff --git a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp index 32208afcd1..b9da3b22b1 100644 --- a/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp +++ b/extensions/azure/tests/PutAzureDataLakeStorageTests.cpp @@ -199,6 +199,7 @@ TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Test Azure data lake stor plan_->setProperty(proxy_configuration_service, "Proxy Server Port", "1234"); plan_->setProperty(proxy_configuration_service, "Proxy User Name", "username"); plan_->setProperty(proxy_configuration_service, "Proxy User Password", "password"); + plan_->setProperty(proxy_configuration_service, "Proxy Type", "HTTP"); plan_->setProperty(azure_data_lake_storage_, "Proxy Configuration Service", "ProxyConfigurationService"); test_controller_.runSession(plan_, true); @@ -212,6 +213,7 @@ TEST_CASE_METHOD(PutAzureDataLakeStorageTestsFixture, "Test Azure data lake stor REQUIRE(*passed_params.proxy_configuration->proxy_user == "username"); REQUIRE(passed_params.proxy_configuration->proxy_password); REQUIRE(*passed_params.proxy_configuration->proxy_password == "password"); + REQUIRE(passed_params.proxy_configuration->proxy_type == minifi::controllers::ProxyType::HTTP); CHECK(getFailedFlowFileContents().empty()); } diff --git a/extensions/azure/tests/features/azure_storage.feature b/extensions/azure/tests/features/azure_storage.feature index 687e41790a..e46db655f5 100644 --- a/extensions/azure/tests/features/azure_storage.feature +++ b/extensions/azure/tests/features/azure_storage.feature @@ -109,7 +109,7 @@ Feature: Sending data from MiNiFi-C++ to an Azure storage server Then the Minifi logs contain the following message: "key:azure.blobname value:test_2" in less than 60 seconds And the Minifi logs do not contain the following message: "key:azure.blobname value:other_test" after 0 seconds - Scenario: A MiNiFi instance can upload data to Azure blob storage through a proxy + Scenario Outline: A MiNiFi instance can upload data to Azure blob storage through a proxy Given a GetFile processor with the "Input Directory" property set to "/tmp/input" And a file with the content "#test_data$123$#" is present in "/tmp/input" And a PutAzureBlobStorage processor set up to communicate with an Azure blob storage @@ -118,7 +118,7 @@ Feature: Sending data from MiNiFi-C++ to an Azure storage server And the "success" relationship of the GetFile processor is connected to the PutAzureBlobStorage And the "success" relationship of the PutAzureBlobStorage processor is connected to the PutFile And the "failure" relationship of the PutAzureBlobStorage processor is connected to the PutAzureBlobStorage - And a ProxyConfigurationService controller service is set up with HTTP proxy configuration + And a ProxyConfigurationService controller service is set up with proxy configuration And an Azure storage server is set up And the http proxy server is set up @@ -129,6 +129,11 @@ Feature: Sending data from MiNiFi-C++ to an Azure storage server And the object on the Azure storage server is "#test_data$123$#" And no errors were generated on the http-proxy regarding "http://azure-storage-server-${scenario_id}:10000/devstoreaccount1/test-container/test-blob" + Examples: Proxy Type + | proxy type | + | HTTP | + | HTTPS | + Scenario: A MiNiFi instance can delete blob from Azure blob storage through a proxy Given a GenerateFlowFile processor with the "File Size" property set to "0B" And a DeleteAzureBlobStorage processor set up to communicate with an Azure blob storage diff --git a/libminifi/include/controllers/ProxyConfigurationService.h b/libminifi/include/controllers/ProxyConfigurationService.h index f4b1468afe..382a410306 100644 --- a/libminifi/include/controllers/ProxyConfigurationService.h +++ b/libminifi/include/controllers/ProxyConfigurationService.h @@ -50,11 +50,17 @@ class ProxyConfigurationService : public core::controller::ControllerServiceImpl .withDescription("The password of the proxy client for user authentication.") .isSensitive(true) .build(); + MINIFIAPI static constexpr auto ProxyTypeProperty = core::PropertyDefinitionBuilder()>::createProperty("Proxy Type") + .withDescription("Proxy type.") + .withDefaultValue(magic_enum::enum_name(ProxyType::HTTP)) + .withAllowedValues(magic_enum::enum_names()) + .build(); MINIFIAPI static constexpr auto Properties = std::to_array({ ProxyServerHost, ProxyServerPort, ProxyUserName, - ProxyUserPassword + ProxyUserPassword, + ProxyTypeProperty }); MINIFIAPI static constexpr bool SupportsDynamicProperties = false; diff --git a/libminifi/src/controllers/ProxyConfigurationService.cpp b/libminifi/src/controllers/ProxyConfigurationService.cpp index 69564f3fb4..dd7a420477 100644 --- a/libminifi/src/controllers/ProxyConfigurationService.cpp +++ b/libminifi/src/controllers/ProxyConfigurationService.cpp @@ -28,7 +28,7 @@ void ProxyConfigurationService::initialize() { void ProxyConfigurationService::onEnable() { std::lock_guard lock(configuration_mutex_); - proxy_configuration_.proxy_type = ProxyType::HTTP; + proxy_configuration_.proxy_type = magic_enum::enum_cast(getProperty(ProxyTypeProperty.name).value_or("HTTP")).value_or(ProxyType::HTTP); proxy_configuration_.proxy_host = getProperty(ProxyServerHost.name).value_or(""); if (proxy_configuration_.proxy_host.empty()) { logger_->log_error("Proxy Server Host is required"); diff --git a/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h b/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h index a39d9572a3..f920d86d5f 100644 --- a/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h +++ b/minifi-api/include/minifi-cpp/controllers/ProxyConfigurationServiceInterface.h @@ -22,7 +22,8 @@ namespace org::apache::nifi::minifi::controllers { enum class ProxyType { - HTTP + HTTP, + HTTPS }; class ProxyConfigurationServiceInterface : public virtual core::controller::ControllerService { From fa7ded8afdf472df14255ff2cbba4be47eed6271 Mon Sep 17 00:00:00 2001 From: Gabor Gyimesi Date: Wed, 18 Feb 2026 17:19:33 +0100 Subject: [PATCH 7/7] Review update --- CONTROLLERS.md | 2 +- extensions/aws/processors/AwsProcessor.cpp | 9 +++------ .../include/controllers/ProxyConfigurationService.h | 5 +++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/CONTROLLERS.md b/CONTROLLERS.md index 41bf9678e8..b097e21a3b 100644 --- a/CONTROLLERS.md +++ b/CONTROLLERS.md @@ -249,7 +249,7 @@ In the list below, the names of required properties appear in bold. Any other pr ### Description -Provides a set of configurations for different MiNiFi C++ components to use a proxy server. Currently these properties can only be used for HTTP proxy configuration, not other protocols are supported at this time. +Provides a set of configurations for various MiNiFi C++ components to use a proxy server. Currently these properties can only be used for HTTP proxy configuration, no other protocols are supported at this time. ### Properties diff --git a/extensions/aws/processors/AwsProcessor.cpp b/extensions/aws/processors/AwsProcessor.cpp index f59c531a15..6ff18913be 100644 --- a/extensions/aws/processors/AwsProcessor.cpp +++ b/extensions/aws/processors/AwsProcessor.cpp @@ -73,12 +73,9 @@ minifi::controllers::ProxyConfiguration AwsProcessor::getProxy(core::ProcessCont if (proxy_controller_service) { proxy.proxy_type = proxy_controller_service->getProxyType(); proxy.proxy_host = proxy_controller_service->getHost(); - auto port_opt = proxy_controller_service->getPort(); - proxy.proxy_port = port_opt.value_or(0); - auto username_opt = proxy_controller_service->getUsername(); - proxy.proxy_user = username_opt.value_or(""); - auto password_opt = proxy_controller_service->getPassword(); - proxy.proxy_password = password_opt.value_or(""); + proxy.proxy_port = proxy_controller_service->getPort().value_or(0); + proxy.proxy_user = proxy_controller_service->getUsername().value_or(""); + proxy.proxy_password = proxy_controller_service->getPassword().value_or(""); } else { proxy.proxy_type = minifi::utils::parseOptionalEnumProperty(context, ProxyType).value_or(minifi::controllers::ProxyType::HTTP); proxy.proxy_host = minifi::utils::parseOptionalProperty(context, ProxyHost, flow_file).value_or(""); diff --git a/libminifi/include/controllers/ProxyConfigurationService.h b/libminifi/include/controllers/ProxyConfigurationService.h index 382a410306..e9e474f4ae 100644 --- a/libminifi/include/controllers/ProxyConfigurationService.h +++ b/libminifi/include/controllers/ProxyConfigurationService.h @@ -32,12 +32,13 @@ class ProxyConfigurationService : public core::controller::ControllerServiceImpl : ControllerServiceImpl(name, uuid) { } - MINIFIAPI static constexpr const char* Description = "Provides a set of configurations for different MiNiFi C++ components to use a proxy server. " - "Currently these properties can only be used for HTTP proxy configuration, not other protocols are supported at this time."; + MINIFIAPI static constexpr const char* Description = "Provides a set of configurations for various MiNiFi C++ components to use a proxy server. Currently these properties can only be used for " + "HTTP proxy configuration, no other protocols are supported at this time."; MINIFIAPI static constexpr auto ProxyServerHost = core::PropertyDefinitionBuilder<>::createProperty("Proxy Server Host") .withDescription("Proxy server hostname or ip-address.") .isRequired(true) + .withValidator(core::StandardPropertyValidators::NON_BLANK_VALIDATOR) .build(); MINIFIAPI static constexpr auto ProxyServerPort = core::PropertyDefinitionBuilder<>::createProperty("Proxy Server Port") .withDescription("Proxy server port number.")