diff --git a/README.adoc b/README.adoc index 852bc0b93..3649986e3 100644 --- a/README.adoc +++ b/README.adoc @@ -2,114 +2,20 @@ In this repository, you will find a collection of components that can meet various data integration use cases and requirements. -The repository's primary focus is to provide a set of standalone Java functions that can be useful in the end-user applications as-is. - -Besides, this repository builds on the Java functions to generate standalone Spring Cloud Stream applications that can run against Spring Cloud Stream's RabbitMQ or Apache Kafka binder implementations. +This repository builds on top of the https://github.com/spring-cloud/spring-functions-catalog[Spring Functions Catalog] and generates standalone Spring Cloud Stream applications that can run against Spring Cloud Stream's RabbitMQ or Apache Kafka binder implementations. It is also possible to extend the generator to bundle the Java functions with the other supported binder implementations. These applications can run standalone or as part of a data flow, such as the one orchestrated using Spring Cloud Data Flow. === Project Structure -The repository includes two major sections - `Functions` and `Applications`. -The former hosts the various Java functions, and the latter is for generating the standalone Spring Cloud Stream applications and hosting their related components. - The following are the various components of this repository. -* https://github.com/spring-cloud/stream-applications/tree/master/functions[Standalone Java Functions] * https://github.com/spring-cloud/stream-applications/tree/master/applications/stream-applications-core[Common Core Components for Applications] * https://github.com/spring-cloud/stream-applications/tree/master/applications[Spring Cloud Stream Applications] * https://github.com/spring-cloud/stream-applications/tree/master/stream-applications-build[Build Parent] * https://github.com/spring-cloud/stream-applications/tree/master/stream-applications-release-train[Release Train] -=== Reusable Functions - -|=== -| `java.util.Supplier` | `java.util.Function` | `java.util.Consumer` - -|link:functions/supplier/debezium-supplier/README.adoc[Debezium supplier] -|link:functions/function/aggregator-function/README.adoc[Aggregator] -|link:functions/consumer/analytics-consumer/README.adoc[Analytics] - -|link:functions/supplier/file-supplier/README.adoc[File] -|link:functions/function/filter-function/README.adoc[Filter] -|link:functions/consumer/cassandra-consumer/README.adoc[Cassandra] - -|link:functions/supplier/ftp-supplier/README.adoc[FTP] -|link:functions/function/header-enricher-function/README.adoc[Header-Enricher] -|link:functions/consumer/elasticsearch-consumer/README.adoc[Elasticsearch] - -| -|link:functions/function/header-filter-function/README.adoc[Header-Filter] -|link:functions/consumer/file-consumer/README.adoc[File] - -|link:functions/supplier/http-supplier/README.adoc[HTTP] -|link:functions/function/http-request-function/README.adoc[HTTP Request] -|link:functions/consumer/ftp-consumer/README.adoc[FTP] - -|link:functions/supplier/jdbc-supplier/README.adoc[JDBC] -|link:functions/function/image-recognition-function/README.adoc[Image Recognition(Tensorflow)] -| - -|link:functions/supplier/jms-supplier/README.adoc[JMS] -|link:functions/function/object-detection-function/README.adoc[Object Detection(Tensorflow)] -|link:functions/consumer/jdbc-consumer/README.adoc[JDBC] - -|link:functions/supplier/mail-supplier/README.adoc[Mail] -|link:functions/function/semantic-segmentation-function/README.adoc[Semantic Segmentation(Tensorflow)] -|link:functions/consumer/log-consumer/README.adoc[Log] - -|link:functions/supplier/mongodb-supplier/README.adoc[MongoDB] -|link:functions/function/spel-function/README.adoc[SpEL] -|link:functions/consumer/mongodb-consumer/README.adoc[MongoDB] - -|link:functions/supplier/mqtt-supplier/README.adoc[MQTT] - -|link:functions/function/splitter-function/README.adoc[Splitter] -|link:functions/consumer/mqtt-consumer/README.adoc[MQTT] - -|link:functions/supplier/rabbit-supplier/README.adoc[RabbitMQ] -|link:functions/function/task-launch-request-function/README.adoc[Task Launch Request] -|link:functions/consumer/rabbit-consumer/README.adoc[RabbitMQ] - -|link:functions/supplier/s3-supplier/README.adoc[AWS S3] -| -|link:functions/consumer/redis-consumer/README.adoc[Redis] - -|link:functions/supplier/sftp-supplier/README.adoc[SFTP] -| -|link:functions/consumer/rsocket-consumer/README.adoc[RSocket] - -|link:functions/supplier/syslog-supplier/README.adoc[Syslog] -| -|link:functions/consumer/s3-consumer/README.adoc[AWS S3] - -|link:functions/supplier/tcp-supplier/README.adoc[TCP] -| -|link:functions/consumer/sftp-consumer/README.adoc[SFTP] - -|link:functions/supplier/time-supplier/README.adoc[Time] -| -|link:functions/consumer/tcp-consumer/README.adoc[TCP] - -|link:functions/supplier/twitter-supplier/README.adoc[Twitter] -|link:functions/function/twitter-function/README.adoc[Twitter] -|link:functions/consumer/twitter-consumer/README.adoc[Twitter] - -|link:functions/supplier/websocket-supplier/README.adoc[Websocket] -| -|link:functions/consumer/websocket-consumer/README.adoc[Websocket] - -| -| -|link:functions/consumer/wavefront-consumer/README.adoc[Wavefront] - -|link:functions/supplier/xmpp-supplier/README.adoc[XMPP] -| -|link:functions/consumer/xmpp-consumer/README.adoc[XMPP] - -|=== - === Reusable Spring Cloud Stream Applications |=== @@ -139,11 +45,11 @@ The following are the various components of this repository. | |link:applications/source/jms-source/README.adoc[JMS] -|link:applications/processor/image-recognition-processor/README.adoc[Image Recognition(Tensorflow)] +| |link:applications/sink/jdbc-sink/README.adoc[JDBC] |link:applications/source/load-generator-source/README.adoc[Load-Generator] -|link:applications/processor/object-detection-processor/README.adoc[Object Detection(Tensorflow)] +| |link:applications/sink/log-sink/README.adoc[Log] |link:applications/source/mail-source/README.adoc[Mail] @@ -151,7 +57,7 @@ The following are the various components of this repository. |link:applications/sink/mongodb-sink/README.adoc[MongoDB] |link:applications/source/mongodb-source/README.adoc[MongoDB] -|link:applications/processor/semantic-segmentation-processor/README.adoc[Semantic Segmentation(Tensorflow)] +| |link:applications/sink/mqtt-sink/README.adoc[MQTT] |link:applications/source/mqtt-source/README.adoc[MQTT] @@ -180,7 +86,7 @@ The following are the various components of this repository. |link:applications/source/time-source/README.adoc[Time] | -|link:applications/sink/tasklauncher-sink/README.adoc[Task Launcher] +| |link:applications/source/twitter-message-source/README.adoc[Twitter Message] | @@ -253,7 +159,8 @@ You can then build the desired apps. ./build-app.sh . applications/sink/log-sink .... -NOTE: In order to disable metrics by default there needs to be application properties configured like in `default-application.properties`. The `build-app.sh` script will copy default-application.properties into src/main/resources if no application.properties,yml,yaml or json is present. +NOTE: In order to disable metrics by default there needs to be application properties configured like in `default-application.properties`. +The `build-app.sh` script will copy default-application.properties into src/main/resources if no application.properties,yml,yaml or json is present. === Additional Resources diff --git a/applications/processor/aggregator-processor/pom.xml b/applications/processor/aggregator-processor/pom.xml index 925103deb..e98704e84 100644 --- a/applications/processor/aggregator-processor/pom.xml +++ b/applications/processor/aggregator-processor/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - aggregator-function + spring-aggregator-function @@ -35,13 +35,13 @@ aggregator processor ${project.version} - org.springframework.cloud.fn.aggregator.AggregatorFunctionConfiguration.class + AUTOCONFIGURATION jsonBytesToMap|aggregatorFunction org.springframework.cloud.fn - aggregator-function + spring-aggregator-function diff --git a/applications/processor/filter-processor/pom.xml b/applications/processor/filter-processor/pom.xml index 1b2f620c1..53795c9a6 100644 --- a/applications/processor/filter-processor/pom.xml +++ b/applications/processor/filter-processor/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - filter-function + spring-filter-function @@ -43,13 +43,13 @@ filter processor ${project.version} - org.springframework.cloud.fn.filter.FilterFunctionConfiguration.class + AUTOCONFIGURATION byteArrayTextToString|filterFunction org.springframework.cloud.fn - filter-function + spring-filter-function diff --git a/applications/processor/groovy-processor/pom.xml b/applications/processor/groovy-processor/pom.xml index c8c72fea2..dd37ad156 100644 --- a/applications/processor/groovy-processor/pom.xml +++ b/applications/processor/groovy-processor/pom.xml @@ -54,7 +54,7 @@ org.springframework.cloud.fn - payload-converter-function + spring-payload-converter-function org.springframework.boot diff --git a/applications/processor/header-enricher-processor/pom.xml b/applications/processor/header-enricher-processor/pom.xml index 77e34ba33..63b21db10 100644 --- a/applications/processor/header-enricher-processor/pom.xml +++ b/applications/processor/header-enricher-processor/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - header-enricher-function + spring-header-enricher-function org.springframework.integration @@ -48,13 +48,13 @@ header-enricher processor ${project.version} - org.springframework.cloud.fn.header.enricher.HeaderEnricherFunctionConfiguration.class + AUTOCONFIGURATION headerEnricherFunction org.springframework.cloud.fn - header-enricher-function + spring-header-enricher-function diff --git a/applications/processor/header-filter-processor/pom.xml b/applications/processor/header-filter-processor/pom.xml index d36d4cd89..3a5d5e275 100644 --- a/applications/processor/header-filter-processor/pom.xml +++ b/applications/processor/header-filter-processor/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - header-filter-function + spring-header-filter-function org.springframework.integration @@ -53,13 +53,13 @@ header-filter processor ${project.version} - org.springframework.cloud.fn.header.filter.HeaderFilterFunctionConfiguration.class + AUTOCONFIGURATION headerFilterFunction org.springframework.cloud.fn - header-filter-function + spring-header-filter-function diff --git a/applications/processor/http-request-processor/README.adoc b/applications/processor/http-request-processor/README.adoc index 44a8b4779..338e6b70c 100644 --- a/applications/processor/http-request-processor/README.adoc +++ b/applications/processor/http-request-processor/README.adoc @@ -52,7 +52,7 @@ $$body-expression$$:: $$A SpEL expression to derive the request body from the in $$expected-response-type$$:: $$The type used to interpret the response.$$ *($$Class$$, default: `$$$$`)* $$headers-expression$$:: $$A SpEL expression used to derive the http headers map to use.$$ *($$Expression$$, default: `$$$$`)* $$http-method-expression$$:: $$A SpEL expression to derive the request method from the incoming message.$$ *($$Expression$$, default: `$$$$`)* -$$reply-expression$$:: $$A SpEL expression used to compute the final result, applied against the whole http {@link org.springframework.http.ResponseEntity}.$$ *($$Expression$$, default: `$$$$`)* +$$reply-expression$$:: $$A SpEL expression used to compute the final result, applied against the whole http {@link ResponseEntity}.$$ *($$Expression$$, default: `$$$$`)* $$timeout$$:: $$Request timeout in milliseconds.$$ *($$Long$$, default: `$$30000$$`)* $$url-expression$$:: $$A SpEL expression against incoming message to determine the URL to use.$$ *($$Expression$$, default: `$$$$`)* diff --git a/applications/processor/http-request-processor/pom.xml b/applications/processor/http-request-processor/pom.xml index bf8dfc783..9eddc9b9f 100644 --- a/applications/processor/http-request-processor/pom.xml +++ b/applications/processor/http-request-processor/pom.xml @@ -21,7 +21,7 @@ org.springframework.cloud.fn - http-request-function + spring-http-request-function com.squareup.okhttp3 @@ -56,15 +56,14 @@ http-request processor ${project.version} - org.springframework.cloud.fn.http.request.HttpRequestFunctionConfiguration.class - + AUTOCONFIGURATION httpRequestFunction org.springframework.cloud.fn - http-request-function + spring-http-request-function diff --git a/applications/processor/image-recognition-processor/README.adoc b/applications/processor/image-recognition-processor/README.adoc deleted file mode 100644 index 84377609c..000000000 --- a/applications/processor/image-recognition-processor/README.adoc +++ /dev/null @@ -1,56 +0,0 @@ -//tag::ref-doc[] -:image-root: https://raw.githubusercontent.com/spring-cloud-stream-app-starters/tensorflow/master/images - -= Image Recognition Processor - -A processor that uses an https://github.com/tensorflow/models/tree/master/inception[Inception model] to classify -in real-time images into different categories (e.g. labels). - -Model implements a deep https://en.wikipedia.org/wiki/Convolutional_neural_network[Convolutional Neural Network] that can achieve reasonable performance on hard visual recognition tasks -- matching or exceeding human performance in some domains like https://www.tensorflow.org/tutorials/image_recognition[image recognition]. - -The input of the model is an image as binary array. - -The output is a JSON message in this format: - -[source,json] -.... -{ - "labels" : [ - {"giant panda":0.98649305} - ] -} -.... - -Result contains the name of the recognized category (e.g. label) along with the confidence (e.g. confidence) that the image represents this category. - -If the `response-seize` is set to value higher then 1, then the result will include the top `response-seize` probable labels. For example `response-size=3` would return: - -[source,json] -.... -{ - "labels": [ - {"giant panda":0.98649305}, - {"badger":0.010562794}, - {"ice bear":0.001130851} - ] -} -.... - -== Payload - -If the incoming type is `byte[]` and the content type is set to `application/octet-stream` , then the application process the input `byte[]` image into and outputs augmented `byte[]` image payload and json header. - -== Options - -//tag::configuration-properties[] -$$image.recognition.cache-model$$:: $$cache the pre-trained tensorflow model.$$ *($$Boolean$$, default: `$$$$`)* -$$image.recognition.debug-output$$:: $$$$ *($$Boolean$$, default: `$$$$`)* -$$image.recognition.debug-output-path$$:: $$$$ *($$String$$, default: `$$$$`)* -$$image.recognition.model$$:: $$pre-trained tensorflow image recognition model. Note that the model must match the selected model type!$$ *($$String$$, default: `$$$$`)* -$$image.recognition.model-type$$:: $$Supports three different pre-trained tensorflow image recognition models: Inception, MobileNetV1 and MobileNetV2 1. Inception graph uses 'input' as input and 'output' as output. 2. MobileNetV2 pre-trained models: https://github.com/tensorflow/models/tree/master/research/slim/nets/mobilenet#pretrained-models - normalized image size is always square (e.g. H=W) - graph uses 'input' as input and 'MobilenetV2/Predictions/Reshape_1' as output. 3. MobileNetV1 pre-trained models: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md#pre-trained-models - graph uses 'input' as input and 'MobilenetV1/Predictions/Reshape_1' as output.$$ *($$ModelType$$, default: `$$$$`, possible values: `inception`,`mobilenetv1`,`mobilenetv2`)* -$$image.recognition.normalized-image-size$$:: $$Normalized image size.$$ *($$Integer$$, default: `$$$$`)* -$$image.recognition.response-size$$:: $$number of recognized images.$$ *($$Integer$$, default: `$$$$`)* -//end::configuration-properties[] - -//end::ref-doc[] diff --git a/applications/processor/image-recognition-processor/pom.xml b/applications/processor/image-recognition-processor/pom.xml deleted file mode 100644 index ca4581057..000000000 --- a/applications/processor/image-recognition-processor/pom.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - 4.0.0 - image-recognition-processor - image-recognition-processor - Image recognition (tensorflow) processor apps - jar - - - org.springframework.cloud.stream.app - stream-applications-core - 5.0.0-SNAPSHOT - ../../stream-applications-core/pom.xml - - - - - org.springframework.boot - spring-boot-configuration-processor - - - org.springframework.cloud.fn - image-recognition-function - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.0.0 - - false - - - - org.springframework.cloud - spring-cloud-dataflow-apps-docs-plugin - - - org.springframework.cloud - spring-cloud-dataflow-apps-generator-plugin - - - image-recognition - processor - ${project.version} - org.springframework.cloud.stream.app.processor.image.recognition.ImageRecognitionProcessorConfiguration.class - imageRecognitionFunction - - - - - org.springframework.cloud.stream.app - image-recognition-processor - ${project.version} - - - - - - - - - - diff --git a/applications/processor/image-recognition-processor/src/main/java/org/springframework/cloud/stream/app/processor/image/recognition/ImageRecognitionProcessorConfiguration.java b/applications/processor/image-recognition-processor/src/main/java/org/springframework/cloud/stream/app/processor/image/recognition/ImageRecognitionProcessorConfiguration.java deleted file mode 100644 index b6a676435..000000000 --- a/applications/processor/image-recognition-processor/src/main/java/org/springframework/cloud/stream/app/processor/image/recognition/ImageRecognitionProcessorConfiguration.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.app.processor.image.recognition; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.List; -import java.util.function.Function; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.tensorflow.deprecated.JsonMapperFunction; -import org.springframework.cloud.fn.image.recognition.ImageRecognition; -import org.springframework.cloud.fn.image.recognition.ImageRecognitionAugmenter; -import org.springframework.cloud.fn.image.recognition.RecognitionResponse; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; - -/** - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties(ImageRecognitionProcessorProperties.class) -public class ImageRecognitionProcessorConfiguration { - - private static final Log logger = LogFactory.getLog(ImageRecognitionProcessorConfiguration.class); - - /** - * Name of the Message header containing the JSON encoded recognition response. - */ - public static final String RECOGNIZED_OBJECTS_HEADER = "recognized_objects"; - - @Bean - public Function, Message> imageRecognitionFunction(ImageRecognitionProcessorProperties properties) { - - return input -> { - // You can use file:, http: or classpath: to provide the path to the input image. - byte[] inputImage = input.getPayload(); - - try (ImageRecognition imageRecognition = createImageRecognitionFunction(properties)) { - - List recognizedObjects = - ImageRecognition.toRecognitionResponse(imageRecognition.recognizeTopK(inputImage)); - - // Draw the predicted labels on top of the input image. - byte[] augmentedImage = new ImageRecognitionAugmenter().apply(inputImage, recognizedObjects); - - String jsonRecognizedObjects = new JsonMapperFunction().apply(recognizedObjects); - - Message outMessage = MessageBuilder - .withPayload(augmentedImage) - .setHeader(RECOGNIZED_OBJECTS_HEADER, jsonRecognizedObjects) - .build(); - - if (properties.isDebugOutput()) { - try { - logger.info("recognized objects = " + jsonRecognizedObjects); - IOUtils.write(augmentedImage, new FileOutputStream(properties.getDebugOutputPath())); - } - catch (IOException e) { - logger.warn("Cloud not produce debug output", e); - } - } - - return outMessage; - } - }; - } - - private static ImageRecognition createImageRecognitionFunction(ImageRecognitionProcessorProperties properties) { - switch (properties.getModelType()) { - case inception: - return ImageRecognition.inception( - properties.getModel(), - properties.getNormalizedImageSize(), - properties.getResponseSize(), - properties.isCacheModel()); - case mobilenetv1: - return ImageRecognition.mobileNetV1( - properties.getModel(), - properties.getNormalizedImageSize(), - properties.getResponseSize(), - properties.isCacheModel()); - case mobilenetv2: - return ImageRecognition.mobileNetV2( - properties.getModel(), - properties.getNormalizedImageSize(), - properties.getResponseSize(), - properties.isCacheModel()); - default: - throw new RuntimeException("Not supported Model Type: " + properties.getModelType()); - - } - } -} diff --git a/applications/processor/image-recognition-processor/src/main/java/org/springframework/cloud/stream/app/processor/image/recognition/ImageRecognitionProcessorProperties.java b/applications/processor/image-recognition-processor/src/main/java/org/springframework/cloud/stream/app/processor/image/recognition/ImageRecognitionProcessorProperties.java deleted file mode 100644 index e59e3e9e9..000000000 --- a/applications/processor/image-recognition-processor/src/main/java/org/springframework/cloud/stream/app/processor/image/recognition/ImageRecognitionProcessorProperties.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.app.processor.image.recognition; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("image.recognition") -@Validated -public class ImageRecognitionProcessorProperties { - - enum ModelType { - inception, - mobilenetv1, - mobilenetv2 - } - - /** - * Supports three different pre-trained tensorflow image recognition models: Inception, MobileNetV1 and MobileNetV2 - * - * 1. Inception graph uses 'input' as input and 'output' as output. - * 2. MobileNetV2 pre-trained models: https://github.com/tensorflow/models/tree/master/research/slim/nets/mobilenet#pretrained-models - * - normalized image size is always square (e.g. H=W) - * - graph uses 'input' as input and 'MobilenetV2/Predictions/Reshape_1' as output. - * 3. MobileNetV1 pre-trained models: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md#pre-trained-models - * - graph uses 'input' as input and 'MobilenetV1/Predictions/Reshape_1' as output. - */ - private ModelType modelType = ModelType.mobilenetv2; - - /** - * pre-trained tensorflow image recognition model. Note that the model must match the selected model type! - */ - private String model = "https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.4_224.tgz#mobilenet_v2_1.4_224_frozen.pb"; - - /** - * cache the pre-trained tensorflow model. - */ - private boolean cacheModel = true; - - /** - * Normalized image size. - */ - private int normalizedImageSize = 224; - - /** - * number of recognized images. - */ - private int responseSize = 5; - - private boolean debugOutput = false; - - private String debugOutputPath = "image-recognition-result.png"; - - public ModelType getModelType() { - return modelType; - } - - public void setModelType(ModelType modelType) { - this.modelType = modelType; - } - - public String getModel() { - return model; - } - - public void setModel(String model) { - this.model = model; - } - - public boolean isCacheModel() { - return cacheModel; - } - - public void setCacheModel(boolean cacheModel) { - this.cacheModel = cacheModel; - } - - public int getNormalizedImageSize() { - return normalizedImageSize; - } - - public void setNormalizedImageSize(int normalizedImageSize) { - this.normalizedImageSize = normalizedImageSize; - } - - public int getResponseSize() { - return responseSize; - } - - public void setResponseSize(int responseSize) { - this.responseSize = responseSize; - } - - public boolean isDebugOutput() { - return debugOutput; - } - - public void setDebugOutput(boolean debugOutput) { - this.debugOutput = debugOutput; - } - - public String getDebugOutputPath() { - return debugOutputPath; - } - - public void setDebugOutputPath(String debugOutputPath) { - this.debugOutputPath = debugOutputPath; - } -} diff --git a/applications/processor/image-recognition-processor/src/main/resources/META-INF/dataflow-configuration-metadata.properties b/applications/processor/image-recognition-processor/src/main/resources/META-INF/dataflow-configuration-metadata.properties deleted file mode 100644 index 4b2d310f4..000000000 --- a/applications/processor/image-recognition-processor/src/main/resources/META-INF/dataflow-configuration-metadata.properties +++ /dev/null @@ -1,2 +0,0 @@ -configuration-properties.classes=\ - org.springframework.cloud.stream.app.processor.image.recognition.ImageRecognitionProcessorProperties diff --git a/applications/processor/image-recognition-processor/src/main/resources/application.yml b/applications/processor/image-recognition-processor/src/main/resources/application.yml deleted file mode 100644 index c608c20d3..000000000 --- a/applications/processor/image-recognition-processor/src/main/resources/application.yml +++ /dev/null @@ -1,16 +0,0 @@ -image: - recognition: - model: https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.4_224.tgz#mobilenet_v2_1.4_224_frozen.pb - modelType: mobilenetv2 - responseSize: 3 - normalizedImageSize: 224 - cacheModel: true -spring: - cloud: - function: - definition: imageRecognitionFunction -management: - defaults: - metrics: - export: - enabled: false \ No newline at end of file diff --git a/applications/processor/image-recognition-processor/src/test/java/org/springframework/cloud/stream/app/processor/image/recognition/ImageRecognitionProcessorTests.java b/applications/processor/image-recognition-processor/src/test/java/org/springframework/cloud/stream/app/processor/image/recognition/ImageRecognitionProcessorTests.java deleted file mode 100644 index 5695ed117..000000000 --- a/applications/processor/image-recognition-processor/src/test/java/org/springframework/cloud/stream/app/processor/image/recognition/ImageRecognitionProcessorTests.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.app.processor.image.recognition; - -import java.io.IOException; -import java.math.BigDecimal; -import java.math.RoundingMode; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; - -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; -import org.springframework.cloud.stream.binder.test.InputDestination; -import org.springframework.cloud.stream.binder.test.OutputDestination; -import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Christian Tzolov - */ -public class ImageRecognitionProcessorTests { - - private ObjectMapper objectMapper = new ObjectMapper(); - - @Test - public void testImageRecognitionProcessorMobileNetV2() throws IOException { - List> expected = deserializeAndRoundToNPlaces( - "[{\"label\":\"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\",\"probability\":0.962329626083374}," - + - "{\"label\":\"badger\",\"probability\":0.006058811210095882}," + - "{\"label\":\"ram, tup\",\"probability\":0.0010668420000001788}]", - 6); - - imageRecognitionProcessorMobileNetV2(verify(expected)); - } - - private void imageRecognitionProcessorMobileNetV2(Consumer> consumer) throws IOException { - try (ConfigurableApplicationContext context = new SpringApplicationBuilder( - TestChannelBinderConfiguration.getCompleteConfiguration(ImageRecognitionProcessorTestApplication.class)) - .web(WebApplicationType.NONE) - .run("--spring.cloud.function.definition=imageRecognitionFunction", - "--image.recognition.modelType=mobilenetv2", - "--image.recognition.responseSize=3", - "--image.recognition.debugOutput=true", - "--image.recognition.debugOutputPath=./target/image-recognition-mobilenetv2.png")) { - - InputDestination processorInput = context.getBean(InputDestination.class); - OutputDestination processorOutput = context.getBean(OutputDestination.class); - - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/giant_panda_in_beijing_zoo_1.jpg"); - processorInput.send(new GenericMessage<>(inputImage)); - Message sourceMessage = processorOutput.receive(10000); - consumer.accept(sourceMessage); - } - } - - @Test - public void testImageRecognitionProcessorMobileNetV1() throws IOException { - List> expected = deserializeAndRoundToNPlaces( - "[{\"label\":\"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\",\"probability\":0.984053909778595}," - + - "{\"label\":\"ram, tup\",\"probability\":0.0019619385711848736}," + - "{\"label\":\"Staffordshire bullterrier, Staffordshire bull terrier\",\"probability\":0.0018697341438382864}]", - 6); - - imageRecognitionProcessorMobileNetV1(verify(expected)); - } - - private void imageRecognitionProcessorMobileNetV1(Consumer> consumer) throws IOException { - try (ConfigurableApplicationContext context = new SpringApplicationBuilder( - TestChannelBinderConfiguration.getCompleteConfiguration(ImageRecognitionProcessorTestApplication.class)) - .web(WebApplicationType.NONE) - .run("--image.recognition.model=https://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224.tgz#mobilenet_v1_1.0_224_frozen.pb", - "--image.recognition.modelType=mobilenetv1", - "--image.recognition.responseSize=3", - "--image.recognition.debugOutput=true", - "--image.recognition.debugOutputPath=./target/image-recognition-mobilenetv1.png")) { - - InputDestination processorInput = context.getBean(InputDestination.class); - OutputDestination processorOutput = context.getBean(OutputDestination.class); - - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/giant_panda_in_beijing_zoo_1.jpg"); - processorInput.send(new GenericMessage<>(inputImage)); - Message sourceMessage = processorOutput.receive(10000); - consumer.accept(sourceMessage); - } - } - - @Test - public void testImageRecognitionProcessorInception() throws IOException { - List> expected = deserializeAndRoundToNPlaces( - "[{\"label\":\"giant panda\",\"probability\":0.9946685433387756}," + - "{\"label\":\"Arctic fox\",\"probability\":0.003663112409412861}," + - "{\"label\":\"ice bear\",\"probability\":3.378273395355791E-4}]", - 6); - imageRecognitionProcessorInception(verify(expected)); - } - - private void imageRecognitionProcessorInception(Consumer> consumer) throws IOException { - try (ConfigurableApplicationContext context = new SpringApplicationBuilder( - TestChannelBinderConfiguration.getCompleteConfiguration(ImageRecognitionProcessorTestApplication.class)) - .web(WebApplicationType.NONE) - .run("--image.recognition.model=https://storage.googleapis.com/scdf-tensorflow-models/image-recognition/tensorflow_inception_graph.pb", - "--image.recognition.modelType=inception", - "--image.recognition.responseSize=3", - "--image.recognition.debugOutput=true", - "--image.recognition.debugOutputPath=./target/image-recognition-inception.png")) { - - InputDestination processorInput = context.getBean(InputDestination.class); - OutputDestination processorOutput = context.getBean(OutputDestination.class); - - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/giant_panda_in_beijing_zoo_1.jpg"); - processorInput.send(new GenericMessage<>(inputImage)); - Message sourceMessage = processorOutput.receive(10000); - - consumer.accept(sourceMessage); - } - } - - private Consumer> verify(List> expected) { - return message -> { - List> actual = deserializeAndRoundToNPlaces((String) message.getHeaders() - .get(ImageRecognitionProcessorConfiguration.RECOGNIZED_OBJECTS_HEADER), 6); - assertThat(expected) - .isEqualTo(actual); - }; - } - - private List> deserializeAndRoundToNPlaces(String json, int places) { - List> result = null; - try { - result = objectMapper.readValue(json, ArrayList.class); - } - catch (JsonProcessingException e) { - throw new IllegalStateException(e.getMessage(), e); - } - - result.forEach(map -> { - if (map.containsKey("probability")) { - map.put("probability", round((double) map.get("probability"), places)); - } - }); - - return result; - - } - - private static double round(double value, int places) { - if (places < 0) { - throw new IllegalArgumentException(); - } - - BigDecimal bd = new BigDecimal(Double.toString(value)); - bd = bd.setScale(places, RoundingMode.HALF_UP); - return bd.doubleValue(); - } - - @SpringBootApplication - @Import({ ImageRecognitionProcessorConfiguration.class }) - public static class ImageRecognitionProcessorTestApplication { - } - -} diff --git a/applications/processor/image-recognition-processor/src/test/resources/images/giant_panda_in_beijing_zoo_1.jpg b/applications/processor/image-recognition-processor/src/test/resources/images/giant_panda_in_beijing_zoo_1.jpg deleted file mode 100644 index e7a44cae2..000000000 Binary files a/applications/processor/image-recognition-processor/src/test/resources/images/giant_panda_in_beijing_zoo_1.jpg and /dev/null differ diff --git a/applications/processor/image-recognition-processor/src/test/resources/images/panda.jpeg b/applications/processor/image-recognition-processor/src/test/resources/images/panda.jpeg deleted file mode 100644 index f912b9440..000000000 Binary files a/applications/processor/image-recognition-processor/src/test/resources/images/panda.jpeg and /dev/null differ diff --git a/applications/processor/object-detection-processor/README.adoc b/applications/processor/object-detection-processor/README.adoc deleted file mode 100644 index a9cdcd543..000000000 --- a/applications/processor/object-detection-processor/README.adoc +++ /dev/null @@ -1,57 +0,0 @@ -//tag::ref-doc[] -:image-root: https://github.com/spring-cloud/stream-applications/raw/master/applications/processor/object-detection-processor/src/test/resources/images - -= Object Detection Processor - -The Object Detection processor provides out-of-the-box support for the https://github.com/tensorflow/models/blob/master/research/object_detection/README.md[TensorFlow Object Detection API]. It allows for real-time localization and identification of multiple objects in a single image or image stream. The Object Detection processor is built on top of the https://github.com/spring-cloud/stream-applications/tree/master/functions/function/object-detection-function[Object Detection Function]. - -You have to provide the Processor with a pre-trained https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md[object detection model], and the corresponding https://github.com/tensorflow/models/tree/865c14c/research/object_detection/data[object labels]. - -Here are some sensible configuration defaults: - -* `object.detection.model` : `https://storage.googleapis.com/scdf-tensorflow-models/object-detection/faster_rcnn_resnet101_coco_2018_01_28_frozen_inference_graph.pb` -* `object.detection.labels` : `https://storage.googleapis.com/scdf-tensorflow-models/object-detection/mscoco_label_map.pbtxt` -* `object.detection.with-masks` : `false` - -The following diagram shows a https://dataflow.spring.io/docs/concepts/streams/[Spring Cloud Data Flow], streaming pipeline, that predicts, in real-time, the object types in input image stream. - -image::{image-root}/scdf-tensorflow-object-detection-arch.png[] - -Processor's input is an image byte array, and the output is an augmented image, and a header, called `detected_objects`, that provides textual description of the detected objects: - -[source,json] -.... -{ - "labels" : [ - {"name":"person", "confidence":0.9996774,"x1":0.0,"y1":0.3940161,"x2":0.9465165,"y2":0.5592592,"cid":1}, - {"name":"person", "confidence":0.9996604,"x1":0.047891676,"y1":0.03169123,"x2":0.941098,"y2":0.2085562,"cid":1}, - {"name":"backpack", "confidence":0.96534747,"x1":0.15588468,"y1":0.85957795,"x2":0.5091308,"y2":0.9908878,"cid":23}, - {"name":"backpack", "confidence":0.963343,"x1":0.1273736,"y1":0.57658505,"x2":0.47765,"y2":0.6986431,"cid":23} - ] -} -.... - -The `detected_objects` header format is: - -* *object-name*:**confidence** - human readable name of the detected object (e.g. label) with its confidence as a float between [0-1] -* *x1*, *y1*, *x2*, *y2* - Response also provides the bounding box of the detected objects represented as `(x1, y1, x2, y2)`. The coordinates are relative to the size of the image size. -* *cid* - Classification identifier as defined in the provided https://github.com/tensorflow/models/tree/865c14c/research/object_detection/data[labels] configuration file. - -== Payload - -The incoming type is `byte[]`, and the content type is `application/octet-stream`. The processor processes the input `byte[]` image and outputs an augmented `byte[]` image payload and a JSON header (`detected_objects`). - -== Options - -//tag::configuration-properties[] -$$object.detection.cache-model$$:: $$$$ *($$Boolean$$, default: `$$$$`)* -$$object.detection.confidence$$:: $$$$ *($$Float$$, default: `$$$$`)* -$$object.detection.debug-output$$:: $$$$ *($$Boolean$$, default: `$$$$`)* -$$object.detection.debug-output-path$$:: $$$$ *($$String$$, default: `$$$$`)* -$$object.detection.labels$$:: $$Labels URI.$$ *($$String$$, default: `$$$$`)* -$$object.detection.model$$:: $$pre-trained tensorflow object detection model.$$ *($$String$$, default: `$$$$`)* -$$object.detection.response-size$$:: $$$$ *($$Integer$$, default: `$$$$`)* -$$object.detection.with-masks$$:: $$$$ *($$Boolean$$, default: `$$$$`)* -//end::configuration-properties[] - -//end::ref-doc[] diff --git a/applications/processor/object-detection-processor/pom.xml b/applications/processor/object-detection-processor/pom.xml deleted file mode 100644 index 440f60136..000000000 --- a/applications/processor/object-detection-processor/pom.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - 4.0.0 - object-detection-processor - object-detection-processor - Object Detection (tensorflow) processor apps - jar - - - org.springframework.cloud.stream.app - stream-applications-core - 5.0.0-SNAPSHOT - ../../stream-applications-core/pom.xml - - - - - org.springframework.boot - spring-boot-configuration-processor - - - org.springframework.cloud.fn - object-detection-function - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.0.0 - - false - - - - org.springframework.cloud - spring-cloud-dataflow-apps-docs-plugin - - - org.springframework.cloud - spring-cloud-dataflow-apps-generator-plugin - - - object-detection - processor - ${project.version} - org.springframework.cloud.stream.app.processor.object.detection.ObjectDetectionProcessorConfiguration.class - objectDetection - - - - - org.springframework.cloud.stream.app - object-detection-processor - ${project.version} - - - - - - - - - - diff --git a/applications/processor/object-detection-processor/src/main/java/org/springframework/cloud/stream/app/processor/object/detection/ObjectDetectionProcessorConfiguration.java b/applications/processor/object-detection-processor/src/main/java/org/springframework/cloud/stream/app/processor/object/detection/ObjectDetectionProcessorConfiguration.java deleted file mode 100644 index fdce5fa5a..000000000 --- a/applications/processor/object-detection-processor/src/main/java/org/springframework/cloud/stream/app/processor/object/detection/ObjectDetectionProcessorConfiguration.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.app.processor.object.detection; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.List; -import java.util.function.Function; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.tensorflow.deprecated.JsonMapperFunction; -import org.springframework.cloud.fn.object.detection.ObjectDetectionImageAugmenter; -import org.springframework.cloud.fn.object.detection.ObjectDetectionService; -import org.springframework.cloud.fn.object.detection.domain.ObjectDetection; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.util.CollectionUtils; - -/** - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties(ObjectDetectionProcessorProperties.class) -public class ObjectDetectionProcessorConfiguration { - - private static final Log logger = LogFactory.getLog(ObjectDetectionProcessorConfiguration.class); - - /** - * Name of the Message header containing the JSON encoded detected objects. - */ - public static final String DETECTED_OBJECTS_HEADER = "detected_objects"; - - @Bean - public ObjectDetectionService objectDetectionService(ObjectDetectionProcessorProperties properties) { - return new ObjectDetectionService(properties.getModel(), - properties.getLabels(), properties.getConfidence(), properties.isWithMasks(), - properties.isCacheModel()); - } - - @Bean - public Function, Message> objectDetection( - ObjectDetectionService objectDetectionService, - ObjectDetectionProcessorProperties properties) { - - return input -> { - // You can use file:, http: or classpath: to provide the path to the input image. - byte[] inputImage = input.getPayload(); - - List detectedObjects = objectDetectionService.detect(inputImage); - - if (!CollectionUtils.isEmpty(detectedObjects) && properties.getResponseSize() < detectedObjects.size()) { - detectedObjects = detectedObjects.subList(0, properties.getResponseSize()); - } - - // Draw the predicted labels on top of the input image. - byte[] augmentedImage = new ObjectDetectionImageAugmenter().apply(inputImage, detectedObjects); - - String jsonDetectedObjects = new JsonMapperFunction().apply(detectedObjects); - - Message outMessage = MessageBuilder - .withPayload(augmentedImage) - .setHeader(DETECTED_OBJECTS_HEADER, jsonDetectedObjects) - .build(); - - if (properties.isDebugOutput()) { - try { - logger.info("detected objects = " + jsonDetectedObjects); - IOUtils.write(augmentedImage, new FileOutputStream(properties.getDebugOutputPath())); - } - catch (IOException e) { - logger.warn("Cloud not produce debug output", e); - } - } - - return outMessage; - - }; - } - -} diff --git a/applications/processor/object-detection-processor/src/main/java/org/springframework/cloud/stream/app/processor/object/detection/ObjectDetectionProcessorProperties.java b/applications/processor/object-detection-processor/src/main/java/org/springframework/cloud/stream/app/processor/object/detection/ObjectDetectionProcessorProperties.java deleted file mode 100644 index ad35f87af..000000000 --- a/applications/processor/object-detection-processor/src/main/java/org/springframework/cloud/stream/app/processor/object/detection/ObjectDetectionProcessorProperties.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.app.processor.object.detection; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("object.detection") -@Validated -public class ObjectDetectionProcessorProperties { - - /** - * pre-trained tensorflow object detection model. - */ - private String model = "https://download.tensorflow.org/models/object_detection/ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz#frozen_inference_graph.pb"; - - /** - * Labels URI. - */ - private String labels = "https://storage.googleapis.com/scdf-tensorflow-models/object-detection/mscoco_label_map.pbtxt"; - - private float confidence = 0.4f; - - private boolean withMasks; - - private boolean cacheModel = true; - - private boolean debugOutput = false; - - private String debugOutputPath = "object-detection-result.png"; - - private int responseSize = Integer.MAX_VALUE; - - public boolean isDebugOutput() { - return debugOutput; - } - - public void setDebugOutput(boolean debugOutput) { - this.debugOutput = debugOutput; - } - - public String getDebugOutputPath() { - return debugOutputPath; - } - - public void setDebugOutputPath(String debugOutputPath) { - this.debugOutputPath = debugOutputPath; - } - - public String getModel() { - return model; - } - - public void setModel(String model) { - this.model = model; - } - - public String getLabels() { - return labels; - } - - public void setLabels(String labels) { - this.labels = labels; - } - - public float getConfidence() { - return confidence; - } - - public void setConfidence(float confidence) { - this.confidence = confidence; - } - - public boolean isWithMasks() { - return withMasks; - } - - public void setWithMasks(boolean withMasks) { - this.withMasks = withMasks; - } - - public boolean isCacheModel() { - return cacheModel; - } - - public void setCacheModel(boolean cacheModel) { - this.cacheModel = cacheModel; - } - - public int getResponseSize() { - return responseSize; - } - - public void setResponseSize(int responseSize) { - this.responseSize = responseSize; - } -} diff --git a/applications/processor/object-detection-processor/src/main/resources/META-INF/dataflow-configuration-metadata.properties b/applications/processor/object-detection-processor/src/main/resources/META-INF/dataflow-configuration-metadata.properties deleted file mode 100644 index 4978c26c6..000000000 --- a/applications/processor/object-detection-processor/src/main/resources/META-INF/dataflow-configuration-metadata.properties +++ /dev/null @@ -1,2 +0,0 @@ -configuration-properties.classes=\ - org.springframework.cloud.stream.app.processor.object.detection.ObjectDetectionProcessorProperties diff --git a/applications/processor/object-detection-processor/src/test/java/org/springframework/cloud/stream/app/processor/object/detection/ObjectDetectionProcessorTests.java b/applications/processor/object-detection-processor/src/test/java/org/springframework/cloud/stream/app/processor/object/detection/ObjectDetectionProcessorTests.java deleted file mode 100644 index 76a36cad3..000000000 --- a/applications/processor/object-detection-processor/src/test/java/org/springframework/cloud/stream/app/processor/object/detection/ObjectDetectionProcessorTests.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.app.processor.object.detection; - -import java.io.IOException; - -import org.junit.jupiter.api.Test; - -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; -import org.springframework.cloud.stream.binder.test.InputDestination; -import org.springframework.cloud.stream.binder.test.OutputDestination; -import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Christian Tzolov - */ -public class ObjectDetectionProcessorTests { - - @Test - public void testObjectDetectionProcessor() throws IOException { - try (ConfigurableApplicationContext context = new SpringApplicationBuilder( - TestChannelBinderConfiguration.getCompleteConfiguration(ObjectDetectionProcessorTestApplication.class)) - .web(WebApplicationType.NONE) - .run("--spring.cloud.function.definition=objectDetection", - "--object.detection.model=https://download.tensorflow.org/models/object_detection/ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz#frozen_inference_graph.pb", - "--object.detection.labels=https://storage.googleapis.com/scdf-tensorflow-models/object-detection/mscoco_label_map.pbtxt", - "--object.detection.responseSize=10", - "--object.detection.debugOutput=true", - "--object.detection.debugOutputPath=./target/object-detection-1.png")) { - - InputDestination processorInput = context.getBean(InputDestination.class); - OutputDestination processorOutput = context.getBean(OutputDestination.class); - - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/pivotal.jpeg"); - processorInput.send(new GenericMessage<>(inputImage)); - Message sourceMessage = processorOutput.receive(10000); - String jsonRecognizedObjects = (String) sourceMessage.getHeaders().get(ObjectDetectionProcessorConfiguration.DETECTED_OBJECTS_HEADER); - assertThat(jsonRecognizedObjects).isNotEmpty(); - //assertThat(jsonRecognizedObjects) - // .isEqualTo("[{\"label\":\"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\",\"probability\":0.962329626083374}," + - // "{\"label\":\"badger\",\"probability\":0.006058811210095882}," + - // "{\"label\":\"ram, tup\",\"probability\":0.0010668420000001788}]"); - } - } - - //@Test - public void testObjectDetectionProcessoriNaturalistSpecies() throws IOException { - try (ConfigurableApplicationContext context = new SpringApplicationBuilder( - TestChannelBinderConfiguration.getCompleteConfiguration(ObjectDetectionProcessorTestApplication.class)) - .web(WebApplicationType.NONE) - .run( - //"--spring.cloud.function.definition=objectDetection", - "--object.detection.model=https://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_fgvc_2018_07_19.tar.gz#frozen_inference_graph.pb", - "--object.detection.labels=https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/fgvc_2854_classes_label_map.pbtxt", - "--object.detection..debugOutput=true", - "--object.detection..debugOutputPath=./target/object-detection-2.png")) { - - InputDestination processorInput = context.getBean(InputDestination.class); - OutputDestination processorOutput = context.getBean(OutputDestination.class); - - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/animals2.jpg"); - processorInput.send(new GenericMessage<>(inputImage)); - Message sourceMessage = processorOutput.receive(10000); - String jsonRecognizedObjects = (String) sourceMessage.getHeaders().get(ObjectDetectionProcessorConfiguration.DETECTED_OBJECTS_HEADER); - //assertThat(jsonRecognizedObjects) - // .isEqualTo("[{\"label\":\"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\",\"probability\":0.962329626083374}," + - // "{\"label\":\"badger\",\"probability\":0.006058811210095882}," + - // "{\"label\":\"ram, tup\",\"probability\":0.0010668420000001788}]"); - } - } - - @SpringBootApplication - @Import({ ObjectDetectionProcessorConfiguration.class }) - public static class ObjectDetectionProcessorTestApplication { - } - -} diff --git a/applications/processor/object-detection-processor/src/test/resources/images/animals2.jpg b/applications/processor/object-detection-processor/src/test/resources/images/animals2.jpg deleted file mode 100644 index 71cb8558a..000000000 Binary files a/applications/processor/object-detection-processor/src/test/resources/images/animals2.jpg and /dev/null differ diff --git a/applications/processor/object-detection-processor/src/test/resources/images/object-detection.jpg b/applications/processor/object-detection-processor/src/test/resources/images/object-detection.jpg deleted file mode 100644 index 9eb325ac5..000000000 Binary files a/applications/processor/object-detection-processor/src/test/resources/images/object-detection.jpg and /dev/null differ diff --git a/applications/processor/object-detection-processor/src/test/resources/images/panda.jpeg b/applications/processor/object-detection-processor/src/test/resources/images/panda.jpeg deleted file mode 100644 index f912b9440..000000000 Binary files a/applications/processor/object-detection-processor/src/test/resources/images/panda.jpeg and /dev/null differ diff --git a/applications/processor/object-detection-processor/src/test/resources/images/pets.jpg b/applications/processor/object-detection-processor/src/test/resources/images/pets.jpg deleted file mode 100644 index 8fb17343d..000000000 Binary files a/applications/processor/object-detection-processor/src/test/resources/images/pets.jpg and /dev/null differ diff --git a/applications/processor/object-detection-processor/src/test/resources/images/pivotal.jpeg b/applications/processor/object-detection-processor/src/test/resources/images/pivotal.jpeg deleted file mode 100644 index 36d7494f1..000000000 Binary files a/applications/processor/object-detection-processor/src/test/resources/images/pivotal.jpeg and /dev/null differ diff --git a/applications/processor/object-detection-processor/src/test/resources/images/scdf-tensorflow-object-detection-arch.png b/applications/processor/object-detection-processor/src/test/resources/images/scdf-tensorflow-object-detection-arch.png deleted file mode 100644 index 2acde16cf..000000000 Binary files a/applications/processor/object-detection-processor/src/test/resources/images/scdf-tensorflow-object-detection-arch.png and /dev/null differ diff --git a/applications/processor/object-detection-processor/src/test/resources/images/tourists.jpg b/applications/processor/object-detection-processor/src/test/resources/images/tourists.jpg deleted file mode 100644 index d1ed10af0..000000000 Binary files a/applications/processor/object-detection-processor/src/test/resources/images/tourists.jpg and /dev/null differ diff --git a/applications/processor/pom.xml b/applications/processor/pom.xml index 9124d690d..859bd2a67 100644 --- a/applications/processor/pom.xml +++ b/applications/processor/pom.xml @@ -21,9 +21,6 @@ transform-processor script-processor twitter-trend-processor - image-recognition-processor - object-detection-processor - semantic-segmentation-processor diff --git a/applications/processor/script-processor/pom.xml b/applications/processor/script-processor/pom.xml index fa4c3bb08..2551418e5 100644 --- a/applications/processor/script-processor/pom.xml +++ b/applications/processor/script-processor/pom.xml @@ -78,8 +78,8 @@ org.springframework.cloud.fn - payload-converter-function - ${java-functions.version} + spring-payload-converter-function + org.springframework.boot diff --git a/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorConfiguration.java b/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorConfiguration.java index 9f66d08d1..770974379 100644 --- a/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorConfiguration.java +++ b/applications/processor/script-processor/src/main/java/org/springframework/cloud/stream/app/processor/script/ScriptProcessorConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2020 the original author or authors. + * Copyright 2016-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,6 +30,7 @@ import org.springframework.core.io.Resource; import org.springframework.integration.handler.MessageProcessor; import org.springframework.integration.scripting.ScriptVariableGenerator; +import org.springframework.integration.scripting.dsl.ScriptSpec; import org.springframework.integration.scripting.dsl.Scripts; import org.springframework.messaging.Message; @@ -47,7 +48,7 @@ * @author Artme Bilan * @author Soby Chacko */ -@Configuration +@Configuration(proxyBeanMethods = false) @EnableConfigurationProperties(ScriptProcessorProperties.class) @Import(ScriptVariableGeneratorConfiguration.class) public class ScriptProcessorConfiguration { @@ -69,12 +70,12 @@ public ScriptProcessorConfiguration(ScriptProcessorProperties properties, Script } @Bean - public Function, Object> scriptProcessorFunction() { - return processor()::processMessage; + public Function, Object> scriptProcessorFunction(MessageProcessor messageProcessor) { + return messageProcessor::processMessage; } @Bean - public MessageProcessor processor() { + public ScriptSpec processor() { String language = this.properties.getLanguage(); String script = this.properties.getScript(); logger.info(String.format("Input script is '%s', language is '%s'", script, language)); @@ -82,8 +83,7 @@ public MessageProcessor processor() { return Scripts.processor(scriptResource) .lang(language) - .variableGenerator(scriptVariableGenerator) - .get(); + .variableGenerator(scriptVariableGenerator); } private static String decodeScript(String script) { @@ -94,4 +94,5 @@ private static String decodeScript(String script) { } return toProcess.replaceAll(NEWLINE_ESCAPE, "\n").replaceAll(DOUBLE_DOUBLE_QUOTE, "\""); } + } diff --git a/applications/processor/semantic-segmentation-processor/README.adoc b/applications/processor/semantic-segmentation-processor/README.adoc deleted file mode 100644 index 1716fceda..000000000 --- a/applications/processor/semantic-segmentation-processor/README.adoc +++ /dev/null @@ -1,42 +0,0 @@ -//tag::ref-doc[] - -= Semantic Segmentation Processor - -Image Semantic Segmentation based on the state-of-art https://github.com/tensorflow/models/tree/master/research/deeplab[DeepLab] Tensorflow model. - -The `Semantic Segmentation` is the process of associating each pixel of an image with a class label, (such as flower, person, road, sky, ocean, or car). -Unlike the `Instance Segmentation`, which produces instance-aware region masks, the `Semantic Segmentation` produces class-aware masks. -For implementing `Instance Segmentation` consult the https://github.com/spring-cloud/stream-applications/tree/master/functions/function/object-detection-function[Object Detection Service] instead. - -The `Semantic Segmentation Processor` uses the https://github.com/spring-cloud/stream-applications/tree/master/functions/function/semantic-segmentation-function[Semantic Segmentation Function] library and the https://github.com/spring-cloud/stream-applications/tree/master/functions/common/tensorflow-common[TensorFlow Service]. - -== Payload - -The incoming type is `byte[]`, and the content type is `application/octet-stream`. The processor processes the input `byte[]` image and outputs augmented `byte[]` image payload and json header. - -Processor's input is an image byte array, and the output is an augmented image byte array, and a JSON header `semantic_segmentation` in this format: - -[source,json] -.... -[ - [ 0, 0, 0 ], - [ 127, 127, 127 ], - [ 255, 255, 255 ], - ... -] -.... - -The output header json format represents the color pixel map computed from the input image. - -== Options - -//tag::configuration-properties[] -$$semantic.segmentation.color-map-uri$$:: $$Every pre-trained model is based on certain object color maps. The pre-defined options are: - classpath:/colormap/citymap_colormap.json - classpath:/colormap/ade20k_colormap.json - classpath:/colormap/black_white_colormap.json - classpath:/colormap/mapillary_colormap.json$$ *($$String$$, default: `$$$$`)* -$$semantic.segmentation.debug-output$$:: $$save output image inn the local debugOutputPath path.$$ *($$Boolean$$, default: `$$$$`)* -$$semantic.segmentation.debug-output-path$$:: $$$$ *($$String$$, default: `$$$$`)* -$$semantic.segmentation.mask-transparency$$:: $$The alpha color of the computed segmentation mask image.$$ *($$Float$$, default: `$$$$`)* -$$semantic.segmentation.model$$:: $$pre-trained tensorflow semantic segmentation model.$$ *($$String$$, default: `$$$$`)* -$$semantic.segmentation.output-type$$:: $$Specifies the output image type. You can return either the input image with the computed mask overlay, or the mask alone.$$ *($$OutputType$$, default: `$$$$`, possible values: `blended`,`mask`)* -//end::configuration-properties[] - -//end::ref-doc[] diff --git a/applications/processor/semantic-segmentation-processor/pom.xml b/applications/processor/semantic-segmentation-processor/pom.xml deleted file mode 100644 index 5605cc5ed..000000000 --- a/applications/processor/semantic-segmentation-processor/pom.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - 4.0.0 - semantic-segmentation-processor - semantic-segmentation-processor - Semantic Segmentation (tensorflow) processor apps - jar - - - org.springframework.cloud.stream.app - stream-applications-core - 5.0.0-SNAPSHOT - ../../stream-applications-core/pom.xml - - - - - org.springframework.boot - spring-boot-configuration-processor - - - org.springframework.cloud.fn - semantic-segmentation-function - - - org.springframework.cloud.fn - object-detection-function - - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.0.0 - - false - - - - org.springframework.cloud - spring-cloud-dataflow-apps-docs-plugin - - - org.springframework.cloud - spring-cloud-dataflow-apps-generator-plugin - - - semantic-segmentation - processor - ${project.version} - org.springframework.cloud.stream.app.processor.semantic.segmentation.SemanticSegmentationProcessorConfiguration.class - semanticSegmentationFunction - - - - - org.springframework.cloud.stream.app - semantic-segmentation-processor - ${project.version} - - - - - - - - - - diff --git a/applications/processor/semantic-segmentation-processor/src/main/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorConfiguration.java b/applications/processor/semantic-segmentation-processor/src/main/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorConfiguration.java deleted file mode 100644 index 7a6e55546..000000000 --- a/applications/processor/semantic-segmentation-processor/src/main/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorConfiguration.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.app.processor.semantic.segmentation; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.function.Function; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.tensorflow.deprecated.JsonMapperFunction; -import org.springframework.cloud.fn.semantic.segmentation.SegmentationColorMap; -import org.springframework.cloud.fn.semantic.segmentation.SemanticSegmentation; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; - -/** - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties(SemanticSegmentationProcessorProperties.class) -public class SemanticSegmentationProcessorConfiguration { - - private static final Log logger = LogFactory.getLog(SemanticSegmentationProcessorConfiguration.class); - - /** - * Output header name. - */ - public static final String SEMANTIC_SEGMENTATION_HEADER = "semantic_segmentation"; - - @Bean - public SemanticSegmentation semanticSegmentation(SemanticSegmentationProcessorProperties properties) { - return new SemanticSegmentation(properties.getModel(), - SegmentationColorMap.loadColorMap(properties.getColorMapUri()), null, - properties.getMaskTransparency()); - } - - @Bean - public Function, Message> semanticSegmentationFunction( - SemanticSegmentation semanticSegmentation, - SemanticSegmentationProcessorProperties properties) { - - return input -> { - // You can use file:, http: or classpath: to provide the path to the input image. - byte[] inputImage = input.getPayload(); - - byte[] outputImage = (properties.getOutputType() == SemanticSegmentationProcessorProperties.OutputType.blended) ? - semanticSegmentation.blendMask(inputImage) : semanticSegmentation.maskImage(inputImage); - - long[][] maskPixels = semanticSegmentation.maskPixels(inputImage); - String jsonMaskPixels = new JsonMapperFunction().apply(maskPixels); - - Message outMessage = MessageBuilder - .withPayload(outputImage) - .setHeader(SEMANTIC_SEGMENTATION_HEADER, jsonMaskPixels) - .build(); - - if (properties.isDebugOutput()) { - try { - IOUtils.write(outputImage, new FileOutputStream(properties.getDebugOutputPath())); - } - catch (IOException e) { - logger.warn("Cloud not produce debug output", e); - } - } - - return outMessage; - }; - } -} diff --git a/applications/processor/semantic-segmentation-processor/src/main/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorProperties.java b/applications/processor/semantic-segmentation-processor/src/main/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorProperties.java deleted file mode 100644 index fec7b807b..000000000 --- a/applications/processor/semantic-segmentation-processor/src/main/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorProperties.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.app.processor.semantic.segmentation; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("semantic.segmentation") -@Validated -public class SemanticSegmentationProcessorProperties { - - enum OutputType { - /** Input image augmented with the segmentation mask on top. */ - blended, - /** Image of the segmentation mask. */ - mask - } - - /** - * pre-trained tensorflow semantic segmentation model. - */ - private String model = "https://download.tensorflow.org/models/deeplabv3_mnv2_cityscapes_train_2018_02_05.tar.gz#frozen_inference_graph.pb"; - - /** - * Specifies the output image type. You can return either the input image with the computed mask overlay, or - * the mask alone. - */ - private OutputType outputType = OutputType.blended; - - /** - * Every pre-trained model is based on certain object color maps. - * The pre-defined options are: - * - classpath:/colormap/citymap_colormap.json - * - classpath:/colormap/ade20k_colormap.json - * - classpath:/colormap/black_white_colormap.json - * - classpath:/colormap/mapillary_colormap.json - */ - private String colorMapUri = "classpath:/colormap/citymap_colormap.json"; - - /** - * The alpha color of the computed segmentation mask image. - */ - private float maskTransparency = 0.45f; - - /** - * save output image inn the local debugOutputPath path. - */ - private boolean debugOutput = false; - - private String debugOutputPath = "semantic-segmentation-result.png"; - - public OutputType getOutputType() { - return outputType; - } - - public void setOutputType(OutputType outputType) { - this.outputType = outputType; - } - - public boolean isDebugOutput() { - return debugOutput; - } - - public void setDebugOutput(boolean debugOutput) { - this.debugOutput = debugOutput; - } - - public String getDebugOutputPath() { - return debugOutputPath; - } - - public void setDebugOutputPath(String debugOutputPath) { - this.debugOutputPath = debugOutputPath; - } - - public String getModel() { - return model; - } - - public void setModel(String model) { - this.model = model; - } - - public String getColorMapUri() { - return colorMapUri; - } - - public void setColorMapUri(String colorMapUri) { - this.colorMapUri = colorMapUri; - } - - public float getMaskTransparency() { - return maskTransparency; - } - - public void setMaskTransparency(float maskTransparency) { - this.maskTransparency = maskTransparency; - } - -} diff --git a/applications/processor/semantic-segmentation-processor/src/main/resources/META-INF/dataflow-configuration-metadata.properties b/applications/processor/semantic-segmentation-processor/src/main/resources/META-INF/dataflow-configuration-metadata.properties deleted file mode 100644 index 104f7436f..000000000 --- a/applications/processor/semantic-segmentation-processor/src/main/resources/META-INF/dataflow-configuration-metadata.properties +++ /dev/null @@ -1,2 +0,0 @@ -configuration-properties.classes=\ - org.springframework.cloud.stream.app.processor.semantic.segmentation.SemanticSegmentationProcessorProperties diff --git a/applications/processor/semantic-segmentation-processor/src/test/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorTests.java b/applications/processor/semantic-segmentation-processor/src/test/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorTests.java deleted file mode 100644 index d95adc2f1..000000000 --- a/applications/processor/semantic-segmentation-processor/src/test/java/org/springframework/cloud/stream/app/processor/semantic/segmentation/SemanticSegmentationProcessorTests.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.stream.app.processor.semantic.segmentation; - -import java.io.IOException; - -import org.junit.jupiter.api.Test; - -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; -import org.springframework.cloud.stream.binder.test.InputDestination; -import org.springframework.cloud.stream.binder.test.OutputDestination; -import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Christian Tzolov - */ -public class SemanticSegmentationProcessorTests { - - @Test - public void testSemanticSegmentationProcessor() throws IOException { - try (ConfigurableApplicationContext context = new SpringApplicationBuilder( - TestChannelBinderConfiguration.getCompleteConfiguration(SemanticSegmentationTestApplication.class)) - .web(WebApplicationType.NONE) - .run("--spring.cloud.function.definition=semanticSegmentationFunction", - "--semantic.segmentation.model=https://download.tensorflow.org/models/deeplabv3_mnv2_cityscapes_train_2018_02_05.tar.gz#frozen_inference_graph.pb", - "--semantic.segmentation.colorMapUri=classpath:/colormap/citymap_colormap.json", - "--semantic.segmentation.outputType=blended", - "--semantic.segmentation.debugOutput=true", - "--semantic.segmentation.debugOutputPath=./target/semantic-segmentation-1.png")) { - - InputDestination processorInput = context.getBean(InputDestination.class); - OutputDestination processorOutput = context.getBean(OutputDestination.class); - - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/amsterdam-cityscape1.jpg"); - processorInput.send(new GenericMessage<>(inputImage)); - Message sourceMessage = processorOutput.receive(10000); - String jsonRecognizedObjects = (String) sourceMessage.getHeaders().get( - SemanticSegmentationProcessorConfiguration.SEMANTIC_SEGMENTATION_HEADER); - assertThat(jsonRecognizedObjects).isNotEmpty(); - //assertThat(jsonRecognizedObjects) - // .isEqualTo("[{\"label\":\"giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca\",\"probability\":0.962329626083374}," + - // "{\"label\":\"badger\",\"probability\":0.006058811210095882}," + - // "{\"label\":\"ram, tup\",\"probability\":0.0010668420000001788}]"); - } - } - - - @SpringBootApplication - @Import(SemanticSegmentationProcessorConfiguration.class) - public static class SemanticSegmentationTestApplication { - } - -} diff --git a/applications/processor/semantic-segmentation-processor/src/test/resources/images/amsterdam-cityscape1.jpg b/applications/processor/semantic-segmentation-processor/src/test/resources/images/amsterdam-cityscape1.jpg deleted file mode 100644 index d77cae406..000000000 Binary files a/applications/processor/semantic-segmentation-processor/src/test/resources/images/amsterdam-cityscape1.jpg and /dev/null differ diff --git a/applications/processor/splitter-processor/README.adoc b/applications/processor/splitter-processor/README.adoc index 482fa9da6..d82b18b23 100644 --- a/applications/processor/splitter-processor/README.adoc +++ b/applications/processor/splitter-processor/README.adoc @@ -20,7 +20,7 @@ If the incoming type is `byte[]` and the content type is set to `text/plain` or $$splitter.apply-sequence$$:: $$Add correlation/sequence information in headers to facilitate later aggregation.$$ *($$Boolean$$, default: `$$true$$`)* $$splitter.charset$$:: $$The charset to use when converting bytes in text-based files to String.$$ *($$String$$, default: `$$$$`)* $$splitter.delimiters$$:: $$When expression is null, delimiters to use when tokenizing {@link String} payloads.$$ *($$String$$, default: `$$$$`)* -$$splitter.expression$$:: $$A SpEL expression for splitting payloads.$$ *($$String$$, default: `$$$$`)* +$$splitter.expression$$:: $$A SpEL expression for splitting payloads.$$ *($$Expression$$, default: `$$$$`)* $$splitter.file-markers$$:: $$Set to true or false to use a {@code FileSplitter} (to split text-based files by line) that includes (or not) beginning/end of file markers.$$ *($$Boolean$$, default: `$$$$`)* $$splitter.markers-json$$:: $$When 'fileMarkers == true', specify if they should be produced as FileSplitter.FileMarker objects or JSON.$$ *($$Boolean$$, default: `$$true$$`)* //end::configuration-properties[] diff --git a/applications/processor/splitter-processor/pom.xml b/applications/processor/splitter-processor/pom.xml index 2ba8e99e4..6237244b3 100644 --- a/applications/processor/splitter-processor/pom.xml +++ b/applications/processor/splitter-processor/pom.xml @@ -17,11 +17,11 @@ org.springframework.cloud.fn - splitter-function + spring-splitter-function org.springframework.cloud.fn - payload-converter-function + spring-payload-converter-function @@ -47,19 +47,18 @@ splitter processor ${project.version} - org.springframework.cloud.fn.splitter.SplitterFunctionConfiguration.class - + AUTOCONFIGURATION byteArrayTextToString|splitterFunction org.springframework.cloud.fn - payload-converter-function + spring-payload-converter-function org.springframework.cloud.fn - splitter-function + spring-splitter-function diff --git a/applications/processor/transform-processor/README.adoc b/applications/processor/transform-processor/README.adoc index 4a4eeb4fe..b998f0ecb 100644 --- a/applications/processor/transform-processor/README.adoc +++ b/applications/processor/transform-processor/README.adoc @@ -21,7 +21,7 @@ The incoming message can contain any type of payload. == Options //tag::configuration-properties[] -$$spel.function.expression$$:: $$A SpEL expression to apply.$$ *($$String$$, default: `$$$$`)* +$$spel.function.expression$$:: $$A SpEL expression to apply.$$ *($$Expression$$, default: `$$$$`)* //end::configuration-properties[] //end::ref-doc[] diff --git a/applications/processor/transform-processor/pom.xml b/applications/processor/transform-processor/pom.xml index 01bb02fd2..05fabf930 100644 --- a/applications/processor/transform-processor/pom.xml +++ b/applications/processor/transform-processor/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - spel-function + spring-spel-function @@ -43,13 +43,13 @@ transform processor ${project.version} - org.springframework.cloud.fn.spel.SpelFunctionConfiguration.class + AUTOCONFIGURATION byteArrayTextToString|spelFunction org.springframework.cloud.fn - spel-function + spring-spel-function diff --git a/applications/processor/twitter-trend-processor/README.adoc b/applications/processor/twitter-trend-processor/README.adoc index 691466f86..780f448e6 100644 --- a/applications/processor/twitter-trend-processor/README.adoc +++ b/applications/processor/twitter-trend-processor/README.adoc @@ -31,12 +31,12 @@ Properties grouped by prefix: === twitter.trend.closest -$$lat$$:: $$If provided with a long parameter the available trend locations will be sorted by distance, nearest to furthest, to the co-ordinate pair. The valid ranges for longitude is -180.0 to +180.0 (West is negative, East is positive) inclusive.$$ *($$Expression$$, default: `$$$$`)* -$$lon$$:: $$If provided with a lat parameter the available trend locations will be sorted by distance, nearest to furthest, to the co-ordinate pair. The valid ranges for longitude is -180.0 to +180.0 (West is negative, East is positive) inclusive.$$ *($$Expression$$, default: `$$$$`)* +$$lat$$:: $$If provided with a long parameter the available trend locations will be sorted by distance, nearest to the furthest, to the co-ordinate pair. The valid ranges for longitude is -180.0 to +180.0 (West is negative, East is positive) inclusive.$$ *($$Expression$$, default: `$$$$`)* +$$lon$$:: $$If provided with a lat parameter the available trend locations will be sorted by distance, nearest to the furthest, to the co-ordinate pair. The valid ranges for longitude is -180.0 to +180.0 (West is negative, East is positive) inclusive.$$ *($$Expression$$, default: `$$$$`)* === twitter.trend -$$location-id$$:: $$The Yahoo! Where On Earth ID of the location to return trending information for. Global information is available by using 1 as the WOEID.$$ *($$Expression$$, default: `$$payload$$`)* +$$location-id$$:: $$The Yahoo! Where On Earth ID of the location to return trending information for. Global information is available by using 1 as the WOEID.$$ *($$Expression$$, default: `$$$$`)* $$trend-query-type$$:: $$$$ *($$TrendQueryType$$, default: `$$$$`, possible values: `trend`,`trendLocation`)* //end::configuration-properties[] diff --git a/applications/processor/twitter-trend-processor/pom.xml b/applications/processor/twitter-trend-processor/pom.xml index b3238de76..d983c1b7c 100644 --- a/applications/processor/twitter-trend-processor/pom.xml +++ b/applications/processor/twitter-trend-processor/pom.xml @@ -18,7 +18,7 @@ org.springframework.cloud.fn - twitter-function + spring-twitter-function org.springframework.boot @@ -28,13 +28,6 @@ org.mock-server mockserver-netty - ${mockserver.version} - test - - - org.mock-server - mockserver-client-java - ${mockserver.version} test @@ -61,14 +54,14 @@ twitter-trend processor ${project.version} - org.springframework.cloud.fn.twitter.trend.TwitterTrendFunctionConfiguration.class + AUTOCONFIGURATION twitterTrendFunction org.springframework.cloud.fn - twitter-function + spring-twitter-function diff --git a/applications/processor/twitter-trend-processor/src/test/java/org/springframework/cloud/stream/app/processor/twitter/trend/TwitterTrendProcessorIntegrationTests.java b/applications/processor/twitter-trend-processor/src/test/java/org/springframework/cloud/stream/app/processor/twitter/trend/TwitterTrendProcessorIntegrationTests.java index 723027e36..94a968c4e 100644 --- a/applications/processor/twitter-trend-processor/src/test/java/org/springframework/cloud/stream/app/processor/twitter/trend/TwitterTrendProcessorIntegrationTests.java +++ b/applications/processor/twitter-trend-processor/src/test/java/org/springframework/cloud/stream/app/processor/twitter/trend/TwitterTrendProcessorIntegrationTests.java @@ -18,7 +18,6 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.concurrent.TimeUnit; import java.util.function.Function; import org.junit.jupiter.api.AfterAll; @@ -36,17 +35,14 @@ import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties; import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils; -import org.springframework.cloud.fn.twitter.trend.TwitterTrendFunctionConfiguration; import org.springframework.cloud.stream.binder.test.InputDestination; import org.springframework.cloud.stream.binder.test.OutputDestination; import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; import org.springframework.messaging.Message; import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.util.TestSocketUtils; import static org.assertj.core.api.Assertions.assertThat; import static org.mockserver.matchers.Times.exactly; @@ -56,22 +52,20 @@ /** * @author Christian Tzolov + * @author Artem Bilan */ public class TwitterTrendProcessorIntegrationTests { - private static final String MOCK_SERVER_IP = "127.0.0.1"; - - private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort(); - private static ClientAndServer mockServer; private static MockServerClient mockClient; + private static HttpRequest trendsRequest; @BeforeAll public static void startServer() { - mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT); - mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT); + mockServer = ClientAndServer.startClientAndServer(); + mockClient = new MockServerClient("localhost", mockServer.getPort()); trendsRequest = setExpectation(request() .withMethod("GET") @@ -91,6 +85,7 @@ public void testTwitterTrendPayload() { .web(WebApplicationType.NONE) .run("--spring.cloud.function.definition=twitterTrendFunction", + "--twitter.trend.trend-query-type=trend", "--twitter.trend.locationId='2972'", "--twitter.connection.rawJson=true", @@ -110,11 +105,6 @@ public void testTwitterTrendPayload() { assertThat(outputMessage).isNotNull(); mockClient.verify(trendsRequest, once()); - - //Resource trendsResource = new DefaultResourceLoader().getResource("classpath:/response/trends.json"); - //String expected = new String(StreamUtils.copyToByteArray(trendsResource.getInputStream()), StandardCharsets.UTF_8).trim(); - //String actual = new String(outputMessage.getPayload(), StandardCharsets.UTF_8); - //JSONAssert.assertEquals(expected, actual, JSONCompareMode.LENIENT); } } @@ -127,15 +117,14 @@ public static HttpRequest setExpectation(HttpRequest request) { new Header("Content-Type", "application/json; charset=utf-8"), new Header("Cache-Control", "public, max-age=86400")) .withBody(TwitterTestUtils.asString("classpath:/response/trends.json")) - .withDelay(TimeUnit.SECONDS, 1) ); return request; } @SpringBootConfiguration @EnableAutoConfiguration - @Import(TwitterTrendFunctionConfiguration.class) public static class TestTwitterTrendProcessorApplication { + @Bean @Primary public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionProperties properties, @@ -143,11 +132,11 @@ public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionPrope Function mockedConfiguration = toConfigurationBuilder.andThen( - new TwitterTestUtils().mockTwitterUrls( - String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT))); + new TwitterTestUtils().mockTwitterUrls("http://localhost:" + mockServer.getPort())); return mockedConfiguration.apply(properties).build(); } + } } diff --git a/applications/sink/analytics-sink/pom.xml b/applications/sink/analytics-sink/pom.xml index 40c660015..2987b4d6a 100644 --- a/applications/sink/analytics-sink/pom.xml +++ b/applications/sink/analytics-sink/pom.xml @@ -18,7 +18,7 @@ org.springframework.cloud.fn - analytics-consumer + spring-analytics-consumer org.awaitility @@ -55,13 +55,13 @@ analytics sink ${project.version} - org.springframework.cloud.fn.consumer.analytics.AnalyticsConsumerConfiguration.class + AUTOCONFIGURATION byteArrayTextToString|analyticsConsumer org.springframework.cloud.fn - analytics-consumer + spring-analytics-consumer diff --git a/applications/sink/cassandra-sink/README.adoc b/applications/sink/cassandra-sink/README.adoc index b9b648230..703bf16f3 100644 --- a/applications/sink/cassandra-sink/README.adoc +++ b/applications/sink/cassandra-sink/README.adoc @@ -24,7 +24,7 @@ $$entity-base-packages$$:: $$Base packages to scan for entities annotated with T $$init-script$$:: $$Resource with CQL scripts (delimited by ';') to initialize keyspace schema.$$ *($$Resource$$, default: `$$$$`)* $$skip-ssl-validation$$:: $$Flag to validate the Servers' SSL certs.$$ *($$Boolean$$, default: `$$false$$`)* -=== cassandra +=== cassandra.consumer $$consistency-level$$:: $$The consistency level for write operation.$$ *($$ConsistencyLevel$$, default: `$$$$`)* $$ingest-query$$:: $$Ingest Cassandra query.$$ *($$String$$, default: `$$$$`)* diff --git a/applications/sink/cassandra-sink/pom.xml b/applications/sink/cassandra-sink/pom.xml index a85a7047f..df604d710 100644 --- a/applications/sink/cassandra-sink/pom.xml +++ b/applications/sink/cassandra-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - cassandra-consumer + spring-cassandra-consumer @@ -43,15 +43,13 @@ cassandra sink ${project.version} - - org.springframework.cloud.fn.consumer.cassandra.CassandraConsumerConfiguration.class - - + AUTOCONFIGURATION + cassandraConsumer org.springframework.cloud.fn - cassandra-consumer + spring-cassandra-consumer diff --git a/applications/sink/elasticsearch-sink/README.adoc b/applications/sink/elasticsearch-sink/README.adoc index e7a63e4b0..bfd690964 100644 --- a/applications/sink/elasticsearch-sink/README.adoc +++ b/applications/sink/elasticsearch-sink/README.adoc @@ -20,11 +20,11 @@ Properties grouped by prefix: === elasticsearch.consumer -$$async$$:: $$Indicates whether the indexing operation is async or not. By default indexing is done synchronously.$$ *($$Boolean$$, default: `$$false$$`)* +$$async$$:: $$Indicates whether the indexing operation is async or not. By default, indexing is done synchronously.$$ *($$Boolean$$, default: `$$false$$`)* $$batch-size$$:: $$Number of items to index for each request. It defaults to 1. For values greater than 1 bulk indexing API will be used.$$ *($$Integer$$, default: `$$1$$`)* $$group-timeout$$:: $$Timeout in milliseconds after which message group is flushed when bulk indexing is active. It defaults to -1, meaning no automatic flush of idle message groups occurs.$$ *($$Long$$, default: `$$-1$$`)* -$$id$$:: $$The id of the document to index. If set, the INDEX_ID header value overrides this property on a per message basis.$$ *($$Expression$$, default: `$$$$`)* -$$index$$:: $$Name of the index. If set, the INDEX_NAME header value overrides this property on a per message basis.$$ *($$String$$, default: `$$$$`)* +$$id$$:: $$The id of the document to index. If set, the INDEX_ID header value overrides this property on a per-message basis.$$ *($$Expression$$, default: `$$$$`)* +$$index$$:: $$Name of the index. If set, the INDEX_NAME header value overrides this property on a per-message basis.$$ *($$String$$, default: `$$$$`)* $$routing$$:: $$Indicates the shard to route to. If not provided, Elasticsearch will default to a hash of the document id.$$ *($$String$$, default: `$$$$`)* $$timeout-seconds$$:: $$Timeout for the shard to be available. If not set, it defaults to 1 minute set by the Elasticsearch client.$$ *($$Long$$, default: `$$0$$`)* diff --git a/applications/sink/elasticsearch-sink/pom.xml b/applications/sink/elasticsearch-sink/pom.xml index 01fa91c34..cf117e5a2 100644 --- a/applications/sink/elasticsearch-sink/pom.xml +++ b/applications/sink/elasticsearch-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - elasticsearch-consumer + spring-elasticsearch-consumer org.awaitility @@ -72,14 +72,14 @@ elasticsearch sink ${project.version} - org.springframework.cloud.fn.consumer.elasticsearch.ElasticsearchConsumerConfiguration.class + AUTOCONFIGURATION byteArrayTextToString|elasticsearchConsumer org.springframework.cloud.fn - elasticsearch-consumer + spring-elasticsearch-consumer diff --git a/applications/sink/file-sink/README.adoc b/applications/sink/file-sink/README.adoc index 20487057c..889495f58 100644 --- a/applications/sink/file-sink/README.adoc +++ b/applications/sink/file-sink/README.adoc @@ -15,10 +15,10 @@ The file sink app writes each message it receives to a file. The `file-sink` has the following options: //tag::configuration-properties[] -$$file.consumer.binary$$:: $$A flag to indicate whether adding a newline after the write should be suppressed.$$ *($$Boolean$$, default: `$$false$$`)* +$$file.consumer.binary$$:: $$A flag to indicate whether adding a newline after the write operation should be suppressed.$$ *($$Boolean$$, default: `$$false$$`)* $$file.consumer.charset$$:: $$The charset to use when writing text content.$$ *($$String$$, default: `$$UTF-8$$`)* $$file.consumer.directory$$:: $$The parent directory of the target file.$$ *($$File$$, default: `$$$$`)* -$$file.consumer.directory-expression$$:: $$The expression to evaluate for the parent directory of the target file.$$ *($$String$$, default: `$$$$`)* +$$file.consumer.directory-expression$$:: $$The expression to evaluate for the parent directory of the target file.$$ *($$Expression$$, default: `$$$$`)* $$file.consumer.mode$$:: $$The FileExistsMode to use if the target file already exists.$$ *($$FileExistsMode$$, default: `$$$$`, possible values: `APPEND`,`APPEND_NO_FLUSH`,`FAIL`,`IGNORE`,`REPLACE`,`REPLACE_IF_MODIFIED`)* $$file.consumer.name$$:: $$The name of the target file.$$ *($$String$$, default: `$$file-consumer$$`)* $$file.consumer.name-expression$$:: $$The expression to evaluate for the name of the target file.$$ *($$String$$, default: `$$$$`)* diff --git a/applications/sink/file-sink/pom.xml b/applications/sink/file-sink/pom.xml index 9efc18a3f..269f46f1f 100644 --- a/applications/sink/file-sink/pom.xml +++ b/applications/sink/file-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - file-consumer + spring-file-consumer @@ -43,13 +43,13 @@ file sink ${project.version} - org.springframework.cloud.fn.consumer.file.FileConsumerConfiguration.class - + AUTOCONFIGURATION + fileConsumer org.springframework.cloud.fn - file-consumer + spring-file-consumer diff --git a/applications/sink/ftp-sink/README.adoc b/applications/sink/ftp-sink/README.adoc index c6c38c697..3cda28c82 100644 --- a/applications/sink/ftp-sink/README.adoc +++ b/applications/sink/ftp-sink/README.adoc @@ -37,14 +37,14 @@ Properties grouped by prefix: === ftp.consumer -$$auto-create-dir$$:: $$Whether or not to create the remote directory.$$ *($$Boolean$$, default: `$$true$$`)* +$$auto-create-dir$$:: $$Whether to create the remote directory.$$ *($$Boolean$$, default: `$$true$$`)* $$filename-expression$$:: $$A SpEL expression to generate the remote file name.$$ *($$String$$, default: `$$$$`)* $$mode$$:: $$Action to take if the remote file already exists.$$ *($$FileExistsMode$$, default: `$$$$`, possible values: `APPEND`,`APPEND_NO_FLUSH`,`FAIL`,`IGNORE`,`REPLACE`,`REPLACE_IF_MODIFIED`)* $$remote-dir$$:: $$The remote FTP directory.$$ *($$String$$, default: `$$/$$`)* $$remote-file-separator$$:: $$The remote file separator.$$ *($$String$$, default: `$$/$$`)* $$temporary-remote-dir$$:: $$A temporary directory where the file will be written if '#isUseTemporaryFilename()' is true.$$ *($$String$$, default: `$$/$$`)* $$tmp-file-suffix$$:: $$The suffix to use while the transfer is in progress.$$ *($$String$$, default: `$$.tmp$$`)* -$$use-temporary-filename$$:: $$Whether or not to write to a temporary file and rename.$$ *($$Boolean$$, default: `$$true$$`)* +$$use-temporary-filename$$:: $$Whether to write to a temporary file and rename.$$ *($$Boolean$$, default: `$$true$$`)* === ftp.factory diff --git a/applications/sink/ftp-sink/pom.xml b/applications/sink/ftp-sink/pom.xml index f5ad426ed..6f417a0e6 100644 --- a/applications/sink/ftp-sink/pom.xml +++ b/applications/sink/ftp-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - ftp-consumer + spring-ftp-consumer @@ -43,12 +43,13 @@ ftp sink ${project.version} - org.springframework.cloud.fn.consumer.ftp.FtpConsumerConfiguration.class + AUTOCONFIGURATION + ftpConsumer org.springframework.cloud.fn - ftp-consumer + spring-ftp-consumer diff --git a/applications/sink/jdbc-sink/pom.xml b/applications/sink/jdbc-sink/pom.xml index e388ff359..e8df6b76c 100644 --- a/applications/sink/jdbc-sink/pom.xml +++ b/applications/sink/jdbc-sink/pom.xml @@ -18,7 +18,7 @@ org.springframework.cloud.fn - jdbc-consumer + spring-jdbc-consumer com.h2database @@ -54,12 +54,13 @@ jdbc sink ${project.version} - org.springframework.cloud.fn.consumer.jdbc.JdbcConsumerConfiguration.class + AUTOCONFIGURATION + jdbcConsumer org.springframework.cloud.fn - jdbc-consumer + spring-jdbc-consumer diff --git a/applications/sink/kafka-sink/README.adoc b/applications/sink/kafka-sink/README.adoc index 52f337c40..90638632a 100644 --- a/applications/sink/kafka-sink/README.adoc +++ b/applications/sink/kafka-sink/README.adoc @@ -51,6 +51,7 @@ $$value-serializer$$:: $$Serializer class for values.$$ *($$Class$$, default: === spring.kafka.template $$default-topic$$:: $$Default topic to which messages are sent.$$ *($$String$$, default: `$$$$`)* +$$observation-enabled$$:: $$Whether to enable observation.$$ *($$Boolean$$, default: `$$false$$`)* $$transaction-id-prefix$$:: $$Transaction id prefix, override the transaction id prefix in the producer factory.$$ *($$String$$, default: `$$$$`)* //end::configuration-properties[] diff --git a/applications/sink/kafka-sink/pom.xml b/applications/sink/kafka-sink/pom.xml index 99588869b..5bcd294fe 100644 --- a/applications/sink/kafka-sink/pom.xml +++ b/applications/sink/kafka-sink/pom.xml @@ -19,7 +19,7 @@ org.springframework.cloud.fn - kafka-publisher + spring-kafka-publisher diff --git a/applications/sink/kafka-sink/src/test/java/org/springframework/cloud/stream/app/sink/kafka/KafkaSinkTests.java b/applications/sink/kafka-sink/src/test/java/org/springframework/cloud/stream/app/sink/kafka/KafkaSinkTests.java index 2416e8ebd..21d516706 100644 --- a/applications/sink/kafka-sink/src/test/java/org/springframework/cloud/stream/app/sink/kafka/KafkaSinkTests.java +++ b/applications/sink/kafka-sink/src/test/java/org/springframework/cloud/stream/app/sink/kafka/KafkaSinkTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2023-2023 the original author or authors. + * Copyright 2023-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,7 +46,7 @@ "kafka.publisher.topic=" + KafkaSinkTests.TEST_TOPIC, "kafka.publisher.mappedHeaders=mapped" }) -@EmbeddedKafka(bootstrapServersProperty = "spring.kafka.bootstrap-servers") +@EmbeddedKafka(kraft = false) @DirtiesContext public class KafkaSinkTests { diff --git a/applications/sink/log-sink/README.adoc b/applications/sink/log-sink/README.adoc index 44b7671e6..75d217a86 100644 --- a/applications/sink/log-sink/README.adoc +++ b/applications/sink/log-sink/README.adoc @@ -13,9 +13,9 @@ The **$$log$$** $$sink$$ has the following options: //tag::configuration-properties[] -$$log.expression$$:: $$A SpEL expression (against the incoming message) to evaluate as the logged message.$$ *($$String$$, default: `$$payload$$`)* -$$log.level$$:: $$The level at which to log messages.$$ *($$Level$$, default: `$$$$`, possible values: `FATAL`,`ERROR`,`WARN`,`INFO`,`DEBUG`,`TRACE`)* -$$log.name$$:: $$The name of the logger to use.$$ *($$String$$, default: `$$$$`)* +$$log.consumer.expression$$:: $$A SpEL expression (against the incoming message) to evaluate as the logged message.$$ *($$String$$, default: `$$payload$$`)* +$$log.consumer.level$$:: $$The level at which to log messages.$$ *($$Level$$, default: `$$$$`, possible values: `FATAL`,`ERROR`,`WARN`,`INFO`,`DEBUG`,`TRACE`)* +$$log.consumer.name$$:: $$The name of the logger to use.$$ *($$String$$, default: `$$$$`)* //end::configuration-properties[] //end::ref-doc[] diff --git a/applications/sink/log-sink/pom.xml b/applications/sink/log-sink/pom.xml index 2f7907f4c..44e004d1b 100644 --- a/applications/sink/log-sink/pom.xml +++ b/applications/sink/log-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - log-consumer + spring-log-consumer org.awaitility @@ -54,14 +54,14 @@ log sink ${project.version} - org.springframework.cloud.fn.consumer.log.LogConsumerConfiguration.class + AUTOCONFIGURATION byteArrayTextToString|logConsumer org.springframework.cloud.fn - log-consumer + spring-log-consumer diff --git a/applications/sink/mongodb-sink/pom.xml b/applications/sink/mongodb-sink/pom.xml index d0fc45974..34f418395 100644 --- a/applications/sink/mongodb-sink/pom.xml +++ b/applications/sink/mongodb-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - mongodb-consumer + spring-mongodb-consumer io.projectreactor @@ -48,14 +48,13 @@ mongodb sink ${project.version} - org.springframework.cloud.fn.consumer.mongo.MongoDbConsumerConfiguration.class - + AUTOCONFIGURATION byteArrayTextToString|mongodbConsumer org.springframework.cloud.fn - mongodb-consumer + spring-mongodb-consumer org.springframework.cloud.fn diff --git a/applications/sink/mqtt-sink/README.adoc b/applications/sink/mqtt-sink/README.adoc index fc8fd079e..ebdaa5a2c 100644 --- a/applications/sink/mqtt-sink/README.adoc +++ b/applications/sink/mqtt-sink/README.adoc @@ -30,12 +30,12 @@ $$username$$:: $$the username to use when connecting to the broker.$$ *($$String === mqtt.consumer -$$async$$:: $$whether or not to use async sends.$$ *($$Boolean$$, default: `$$false$$`)* -$$charset$$:: $$the charset used to convert a String payload to byte[].$$ *($$String$$, default: `$$UTF-8$$`)* -$$client-id$$:: $$identifies the client.$$ *($$String$$, default: `$$stream.client.id.sink$$`)* -$$qos$$:: $$the quality of service to use.$$ *($$Integer$$, default: `$$1$$`)* -$$retained$$:: $$whether to set the 'retained' flag.$$ *($$Boolean$$, default: `$$false$$`)* -$$topic$$:: $$the topic to which the sink will publish.$$ *($$String$$, default: `$$stream.mqtt$$`)* +$$async$$:: $$Whether to use async sends.$$ *($$Boolean$$, default: `$$false$$`)* +$$charset$$:: $$The charset used to convert a String payload to byte[].$$ *($$String$$, default: `$$UTF-8$$`)* +$$client-id$$:: $$Identifies the client.$$ *($$String$$, default: `$$stream.client.id.sink$$`)* +$$qos$$:: $$The quality of service to use.$$ *($$Integer$$, default: `$$1$$`)* +$$retained$$:: $$Whether to set the 'retained' flag.$$ *($$Boolean$$, default: `$$false$$`)* +$$topic$$:: $$The topic to which the sink will publish.$$ *($$String$$, default: `$$stream.mqtt$$`)* //end::configuration-properties[] //end::ref-doc[] diff --git a/applications/sink/mqtt-sink/pom.xml b/applications/sink/mqtt-sink/pom.xml index 4a751ded4..bc3f46c63 100644 --- a/applications/sink/mqtt-sink/pom.xml +++ b/applications/sink/mqtt-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - mqtt-consumer + spring-mqtt-consumer org.testcontainers @@ -49,13 +49,13 @@ mqtt sink ${project.version} - org.springframework.cloud.fn.consumer.mqtt.MqttConsumerConfiguration.class - + AUTOCONFIGURATION + mqttConsumer org.springframework.cloud.fn - mqtt-consumer + spring-mqtt-consumer diff --git a/applications/sink/pgcopy-sink/pom.xml b/applications/sink/pgcopy-sink/pom.xml index 037e5dc40..b2aafc3de 100644 --- a/applications/sink/pgcopy-sink/pom.xml +++ b/applications/sink/pgcopy-sink/pom.xml @@ -28,11 +28,6 @@ postgresql compile - - org.springframework.cloud - spring-cloud-stream-test-binder - test - diff --git a/applications/sink/rabbit-sink/README.adoc b/applications/sink/rabbit-sink/README.adoc index 70a8cc00e..93b26eae7 100644 --- a/applications/sink/rabbit-sink/README.adoc +++ b/applications/sink/rabbit-sink/README.adoc @@ -13,14 +13,13 @@ The **$$rabbit$$** $$sink$$ has the following options: Properties grouped by prefix: -=== rabbit +=== rabbit.consumer $$converter-bean-name$$:: $$The bean name for a custom message converter; if omitted, a SimpleMessageConverter is used. If 'jsonConverter', a Jackson2JsonMessageConverter bean will be created for you.$$ *($$String$$, default: `$$$$`)* $$exchange$$:: $$Exchange name - overridden by exchangeNameExpression, if supplied.$$ *($$String$$, default: `$$$$`)* $$exchange-expression$$:: $$A SpEL expression that evaluates to an exchange name.$$ *($$Expression$$, default: `$$$$`)* -$$headers-mapped-last$$:: $$When mapping headers for the outbound message, determine whether the headers are mapped before the message is converted, or afterwards.$$ *($$Boolean$$, default: `$$true$$`)* +$$headers-mapped-last$$:: $$When mapping headers for the outbound message, determine whether the headers are mapped before the message is converted, or afterward.$$ *($$Boolean$$, default: `$$true$$`)* $$mapped-request-headers$$:: $$Headers that will be mapped.$$ *($$String[]$$, default: `$$[*]$$`)* -$$own-connection$$:: $$When true, use a separate connection based on the boot properties.$$ *($$Boolean$$, default: `$$false$$`)* $$persistent-delivery-mode$$:: $$Default delivery mode when 'amqp_deliveryMode' header is not present, true for PERSISTENT.$$ *($$Boolean$$, default: `$$false$$`)* $$routing-key$$:: $$Routing key - overridden by routingKeyExpression, if supplied.$$ *($$String$$, default: `$$$$`)* $$routing-key-expression$$:: $$A SpEL expression that evaluates to a routing key.$$ *($$Expression$$, default: `$$$$`)* @@ -32,6 +31,7 @@ $$addresses$$:: $$Comma-separated list of addresses to which the client should c $$channel-rpc-timeout$$:: $$Continuation timeout for RPC calls in channels. Set it to zero to wait forever.$$ *($$Duration$$, default: `$$10m$$`)* $$connection-timeout$$:: $$Connection timeout. Set it to zero to wait forever.$$ *($$Duration$$, default: `$$$$`)* $$host$$:: $$RabbitMQ host. Ignored if an address is set.$$ *($$String$$, default: `$$localhost$$`)* +$$max-inbound-message-body-size$$:: $$Maximum size of the body of inbound (received) messages.$$ *($$DataSize$$, default: `$$64MB$$`)* $$password$$:: $$Login to authenticate against the broker.$$ *($$String$$, default: `$$guest$$`)* $$port$$:: $$RabbitMQ port. Ignored if an address is set. Default to 5672, or 5671 if SSL is enabled.$$ *($$Integer$$, default: `$$$$`)* $$publisher-confirm-type$$:: $$Type of publisher confirms to use.$$ *($$ConfirmType$$, default: `$$$$`, possible values: `SIMPLE`,`CORRELATED`,`NONE`)* diff --git a/applications/sink/rabbit-sink/pom.xml b/applications/sink/rabbit-sink/pom.xml index 36aa64f6d..50e248ef1 100644 --- a/applications/sink/rabbit-sink/pom.xml +++ b/applications/sink/rabbit-sink/pom.xml @@ -27,14 +27,9 @@ org.springframework.cloud.fn - rabbit-consumer + spring-rabbit-consumer - org.springframework.cloud - spring-cloud-stream-test-binder - test - - org.testcontainers testcontainers ${testcontainers.version} @@ -76,13 +71,13 @@ rabbit sink ${project.version} - org.springframework.cloud.fn.consumer.rabbit.RabbitConsumerConfiguration.class - + AUTOCONFIGURATION + rabbitConsumer org.springframework.cloud.fn - rabbit-consumer + spring-rabbit-consumer diff --git a/applications/sink/redis-sink/README.adoc b/applications/sink/redis-sink/README.adoc index ea07a136e..7ecf5ce2f 100644 --- a/applications/sink/redis-sink/README.adoc +++ b/applications/sink/redis-sink/README.adoc @@ -14,11 +14,11 @@ Properties grouped by prefix: === redis.consumer $$key$$:: $$A literal key name to use when storing to a key.$$ *($$String$$, default: `$$$$`)* -$$key-expression$$:: $$A SpEL expression to use for storing to a key.$$ *($$String$$, default: `$$$$`)* +$$key-expression$$:: $$A SpEL expression to use for storing to a key.$$ *($$Expression$$, default: `$$$$`)* $$queue$$:: $$A literal queue name to use when storing in a queue.$$ *($$String$$, default: `$$$$`)* -$$queue-expression$$:: $$A SpEL expression to use for queue.$$ *($$String$$, default: `$$$$`)* +$$queue-expression$$:: $$A SpEL expression to use for queue.$$ *($$Expression$$, default: `$$$$`)* $$topic$$:: $$A literal topic name to use when publishing to a topic.$$ *($$String$$, default: `$$$$`)* -$$topic-expression$$:: $$A SpEL expression to use for topic.$$ *($$String$$, default: `$$$$`)* +$$topic-expression$$:: $$A SpEL expression to use for topic.$$ *($$Expression$$, default: `$$$$`)* === spring.data.redis diff --git a/applications/sink/redis-sink/pom.xml b/applications/sink/redis-sink/pom.xml index eb9585b7a..1bff70bb5 100644 --- a/applications/sink/redis-sink/pom.xml +++ b/applications/sink/redis-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - redis-consumer + spring-redis-consumer @@ -79,13 +79,13 @@ redis sink ${project.version} - org.springframework.cloud.fn.consumer.redis.RedisConsumerConfiguration.class - + AUTOCONFIGURATION + redisConsumer org.springframework.cloud.fn - redis-consumer + spring-redis-consumer diff --git a/applications/sink/redis-sink/src/test/java/org/springframework/cloud/stream/app/sink/redis/RedisSinkTests.java b/applications/sink/redis-sink/src/test/java/org/springframework/cloud/stream/app/sink/redis/RedisSinkTests.java index f89e7c3d8..7c05255cf 100644 --- a/applications/sink/redis-sink/src/test/java/org/springframework/cloud/stream/app/sink/redis/RedisSinkTests.java +++ b/applications/sink/redis-sink/src/test/java/org/springframework/cloud/stream/app/sink/redis/RedisSinkTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 the original author or authors. + * Copyright 2020-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,12 +24,10 @@ import org.springframework.boot.WebApplicationType; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.cloud.fn.consumer.redis.RedisConsumerConfiguration; import org.springframework.cloud.fn.consumer.redis.RedisTestContainerSupport; import org.springframework.cloud.stream.binder.test.InputDestination; import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.Import; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.data.redis.support.collections.DefaultRedisList; import org.springframework.data.redis.support.collections.RedisList; @@ -47,32 +45,30 @@ public class RedisSinkTests implements RedisTestContainerSupport { @Test public void testRedisSink() { + String key = "foo"; try (ConfigurableApplicationContext context = new SpringApplicationBuilder( TestChannelBinderConfiguration .getCompleteConfiguration(RedisSinkTestApplication.class)) .web(WebApplicationType.NONE) .run("--spring.cloud.function.definition=redisConsumer", + "--spring.cloud.stream.bindings.redisConsumer-in-0.consumer.use-native-decoding=true", "--spring.data.redis.url=" + RedisTestContainerSupport.getUri(), - "--redis.consumer.key=foo")) { + "--redis.consumer.key=" + key)) { - //Setup - String key = "foo"; - - final StringRedisTemplate redisTemplate = context.getBean(StringRedisTemplate.class); + StringRedisTemplate redisTemplate = context.getBean(StringRedisTemplate.class); redisTemplate.delete(key); - RedisList redisList = new DefaultRedisList<>(key, redisTemplate); List list = new ArrayList<>(); list.add("Manny"); list.add("Moe"); list.add("Jack"); - //Execute Message> message = new GenericMessage<>(list); InputDestination source = context.getBean(InputDestination.class); source.send(message); + RedisList redisList = new DefaultRedisList<>(key, redisTemplate); assertThat(redisList.size()).isEqualTo(3); assertThat(redisList.get(0)).isEqualTo("Manny"); assertThat(redisList.get(1)).isEqualTo("Moe"); @@ -81,7 +77,6 @@ public void testRedisSink() { } @SpringBootApplication - @Import(RedisConsumerConfiguration.class) public static class RedisSinkTestApplication { } } diff --git a/applications/sink/router-sink/pom.xml b/applications/sink/router-sink/pom.xml index 0c1ece0e2..bd92aeb66 100644 --- a/applications/sink/router-sink/pom.xml +++ b/applications/sink/router-sink/pom.xml @@ -54,7 +54,7 @@ org.springframework.cloud.fn - payload-converter-function + spring-payload-converter-function org.springframework.boot diff --git a/applications/sink/router-sink/src/main/java/org/springframework/cloud/stream/app/sink/router/RouterSinkConfiguration.java b/applications/sink/router-sink/src/main/java/org/springframework/cloud/stream/app/sink/router/RouterSinkConfiguration.java index 836dde298..6ecf7423f 100644 --- a/applications/sink/router-sink/src/main/java/org/springframework/cloud/stream/app/sink/router/RouterSinkConfiguration.java +++ b/applications/sink/router-sink/src/main/java/org/springframework/cloud/stream/app/sink/router/RouterSinkConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 the original author or authors. + * Copyright 2016-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,6 +36,7 @@ import org.springframework.integration.router.AbstractMessageRouter; import org.springframework.integration.router.ExpressionEvaluatingRouter; import org.springframework.integration.router.MethodInvokingRouter; +import org.springframework.integration.scripting.dsl.ScriptSpec; import org.springframework.integration.scripting.dsl.Scripts; import org.springframework.lang.Nullable; import org.springframework.messaging.Message; @@ -93,12 +94,11 @@ public AbstractMessageRouter router(BindingService bindingService, StreamBridge @Bean @ConditionalOnProperty("router.script") - public MessageProcessor scriptProcessor() { + public ScriptSpec scriptProcessor() { return Scripts.processor(this.properties.getScript()) .lang("groovy") .refreshCheckDelay(this.properties.getRefreshDelay()) - .variables(obtainScriptVariables(this.properties)) - .get(); + .variables(obtainScriptVariables(this.properties)); } private static Map obtainScriptVariables(RouterSinkProperties properties) { diff --git a/applications/sink/rsocket-sink/pom.xml b/applications/sink/rsocket-sink/pom.xml index 7aaebde95..47a772029 100644 --- a/applications/sink/rsocket-sink/pom.xml +++ b/applications/sink/rsocket-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - rsocket-consumer + spring-rsocket-consumer io.projectreactor @@ -53,12 +53,13 @@ rsocket sink ${project.version} - org.springframework.cloud.fn.consumer.rsocket.RsocketConsumerConfiguration.class + AUTOCONFIGURATION + rsocketConsumer org.springframework.cloud.fn - rsocket-consumer + spring-rsocket-consumer diff --git a/applications/sink/rsocket-sink/src/test/java/org/springframework/cloud/stream/app/rsocket/sink/RSocketSinkTests.java b/applications/sink/rsocket-sink/src/test/java/org/springframework/cloud/stream/app/rsocket/sink/RSocketSinkTests.java index 185166e91..8524eaa67 100644 --- a/applications/sink/rsocket-sink/src/test/java/org/springframework/cloud/stream/app/rsocket/sink/RSocketSinkTests.java +++ b/applications/sink/rsocket-sink/src/test/java/org/springframework/cloud/stream/app/rsocket/sink/RSocketSinkTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 the original author or authors. + * Copyright 2020-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,13 +16,11 @@ package org.springframework.cloud.stream.app.rsocket.sink; -import java.util.function.Function; +import java.time.Duration; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.core.publisher.ReplayProcessor; +import reactor.core.publisher.Sinks; import reactor.test.StepVerifier; import org.springframework.beans.factory.annotation.Autowired; @@ -36,11 +34,9 @@ import org.springframework.cloud.stream.binder.test.InputDestination; import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Import; import org.springframework.integration.support.MessageBuilder; import org.springframework.messaging.Message; import org.springframework.messaging.handler.annotation.MessageMapping; -import org.springframework.messaging.support.GenericMessage; import org.springframework.stereotype.Controller; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.util.ReflectionTestUtils; @@ -48,7 +44,7 @@ /** * @author Soby Chacko */ -@SpringBootTest(properties = {"spring.rsocket.server.port=0"}, classes = RSocketSinkTests.RSocketserverApplication.class) +@SpringBootTest(properties = {"spring.rsocket.server.port=0"}, classes = RSocketSinkTests.RSocketServerApplication.class) @DirtiesContext public class RSocketSinkTests { @@ -71,15 +67,11 @@ void testRsocketConsumer() { final int port = server.address().getPort(); applicationContextRunner.withPropertyValues( - "spring.cloud.function.definition=rsocketConsumer", - "rsocket.consumer.port=" + port, - "rsocket.consumer.route=test-route") + "spring.cloud.function.definition=rsocketFunctionConsumer", + "rsocket.consumer.port=" + port, + "rsocket.consumer.route=test-route") .run(context -> { - Function>, Mono> rsocketConsumer = context.getBean("rsocketConsumer", Function.class); - rsocketConsumer.apply(Flux.just(new GenericMessage<>("Hello RSocket"))) - .subscribe(); - - final StepVerifier stepVerifier = StepVerifier.create(RSocketserverApplication.fireForgetPayloads) + final StepVerifier stepVerifier = StepVerifier.create(RSocketServerApplication.fireForgetPayloads.asMono()) .expectNext("Hello RSocket") .thenCancel() .verifyLater(); @@ -88,25 +80,24 @@ void testRsocketConsumer() { InputDestination source = context.getBean(InputDestination.class); source.send(message); - stepVerifier.verify(); + stepVerifier.verify(Duration.ofSeconds(10)); }); } - @EnableAutoConfiguration + @EnableAutoConfiguration(exclude = RsocketConsumerConfiguration.class) @SpringBootConfiguration @Controller - static class RSocketserverApplication { - static final ReplayProcessor fireForgetPayloads = ReplayProcessor.create(); + static class RSocketServerApplication { + static final Sinks.One fireForgetPayloads = Sinks.one(); @MessageMapping("test-route") void someMethod(String payload) { - this.fireForgetPayloads.onNext(payload); + this.fireForgetPayloads.tryEmitValue(payload); } } @EnableAutoConfiguration @SpringBootConfiguration - @Import(RsocketConsumerConfiguration.class) static class RsocketSinkTestApplication { } diff --git a/applications/sink/s3-sink/README.adoc b/applications/sink/s3-sink/README.adoc index dbf1bc2d7..6ec4d5037 100644 --- a/applications/sink/s3-sink/README.adoc +++ b/applications/sink/s3-sink/README.adoc @@ -44,6 +44,7 @@ $$static$$:: $$$$ *($$String$$, default: `$$$$`)* $$accelerate-mode-enabled$$:: $$Option to enable using the accelerate endpoint when accessing S3. Accelerate endpoints allow faster transfer of objects by using Amazon CloudFront's globally distributed edge locations.$$ *($$Boolean$$, default: `$$$$`)* $$checksum-validation-enabled$$:: $$Option to disable doing a validation of the checksum of an object stored in S3.$$ *($$Boolean$$, default: `$$$$`)* $$chunked-encoding-enabled$$:: $$Option to enable using chunked encoding when signing the request payload for {@link software.amazon.awssdk.services.s3.model.PutObjectRequest} and {@link software.amazon.awssdk.services.s3.model.UploadPartRequest}.$$ *($$Boolean$$, default: `$$$$`)* +$$cross-region-enabled$$:: $$Enables cross-region bucket access.$$ *($$Boolean$$, default: `$$$$`)* $$endpoint$$:: $$Overrides the default endpoint.$$ *($$URI$$, default: `$$$$`)* $$path-style-access-enabled$$:: $$Option to enable using path style access for accessing S3 objects instead of DNS style access. DNS style access is preferred as it will result in better load balancing when accessing S3.$$ *($$Boolean$$, default: `$$$$`)* $$region$$:: $$Overrides the default region.$$ *($$String$$, default: `$$$$`)* diff --git a/applications/sink/s3-sink/pom.xml b/applications/sink/s3-sink/pom.xml index c015855bc..4d75ce7bf 100644 --- a/applications/sink/s3-sink/pom.xml +++ b/applications/sink/s3-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - s3-consumer + spring-s3-consumer org.springframework.integration @@ -58,13 +58,13 @@ s3 sink ${project.version} - org.springframework.cloud.fn.consumer.s3.AwsS3ConsumerConfiguration.class - + AUTOCONFIGURATION + s3Consumer org.springframework.cloud.fn - s3-consumer + spring-s3-consumer diff --git a/applications/sink/s3-sink/src/test/java/org/springframework/cloud/stream/app/s3/sink/AwsS3SinkTests.java b/applications/sink/s3-sink/src/test/java/org/springframework/cloud/stream/app/s3/sink/AwsS3SinkTests.java index f47c29540..24be6bb9e 100644 --- a/applications/sink/s3-sink/src/test/java/org/springframework/cloud/stream/app/s3/sink/AwsS3SinkTests.java +++ b/applications/sink/s3-sink/src/test/java/org/springframework/cloud/stream/app/s3/sink/AwsS3SinkTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 the original author or authors. + * Copyright 2016-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package org.springframework.cloud.stream.app.s3.sink; import java.io.File; +import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; @@ -40,7 +41,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.consumer.s3.AwsS3ConsumerConfiguration; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.cloud.stream.binder.test.InputDestination; import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.context.annotation.Bean; @@ -54,7 +55,6 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.BDDMockito.willReturn; import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, @@ -62,6 +62,7 @@ "spring.cloud.aws.credentials.accessKey=" + AwsS3SinkTests.AWS_ACCESS_KEY, "spring.cloud.aws.credentials.secretKey=" + AwsS3SinkTests.AWS_SECRET_KEY, "spring.cloud.aws.region.static=" + AwsS3SinkTests.AWS_REGION, + "spring.cloud.aws.s3.endpoint=s3://test.endpoint", "s3.consumer.bucket=" + AwsS3SinkTests.S3_BUCKET, "s3.consumer.acl=PUBLIC_READ_WRITE"}) @DirtiesContext @@ -78,7 +79,7 @@ public class AwsS3SinkTests { @TempDir protected static Path temporaryRemoteFolder; - @Autowired + @MockBean private S3AsyncClient amazonS3; @Autowired @@ -95,8 +96,6 @@ public class AwsS3SinkTests { @BeforeEach public void setupTest() { - S3AsyncClient amazonS3 = spy(this.amazonS3); - willReturn(CompletableFuture.completedFuture(PutObjectResponse.builder().build())) .given(amazonS3) .putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)); @@ -133,7 +132,7 @@ public void testS3SinkWithBinderBasic() throws Exception { AsyncRequestBody asyncRequestBody = asyncRequestBodyArgumentCaptor.getValue(); StepVerifier.create(asyncRequestBody) - .assertNext(buffer -> assertThat(buffer.array()).isEmpty()) + .assertNext(buffer -> assertThat(buffer).isEqualTo(ByteBuffer.allocate(0))) .expectComplete() .verify(); @@ -141,7 +140,7 @@ public void testS3SinkWithBinderBasic() throws Exception { } @SpringBootApplication - @Import({AwsS3ConsumerConfiguration.class, TestChannelBinderConfiguration.class}) + @Import(TestChannelBinderConfiguration.class) public static class SampleConfiguration { @Bean diff --git a/applications/sink/sftp-sink/pom.xml b/applications/sink/sftp-sink/pom.xml index 71ec07dff..9f05bf73b 100644 --- a/applications/sink/sftp-sink/pom.xml +++ b/applications/sink/sftp-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - sftp-consumer + spring-sftp-consumer @@ -43,13 +43,13 @@ sftp sink ${project.version} - org.springframework.cloud.fn.consumer.sftp.SftpConsumerConfiguration.class - + AUTOCONFIGURATION + sftpConsumer org.springframework.cloud.fn - sftp-consumer + spring-sftp-consumer diff --git a/applications/sink/tcp-sink/README.adoc b/applications/sink/tcp-sink/README.adoc index 5f940b246..d0f1c51d3 100644 --- a/applications/sink/tcp-sink/README.adoc +++ b/applications/sink/tcp-sink/README.adoc @@ -23,11 +23,11 @@ $$host$$:: $$The host to which this sink will connect.$$ *($$String$$, default: === tcp -$$nio$$:: $$Whether or not to use NIO.$$ *($$Boolean$$, default: `$$false$$`)* +$$nio$$:: $$Whether to use NIO.$$ *($$Boolean$$, default: `$$false$$`)* $$port$$:: $$The port on which to listen; 0 for the OS to choose a port.$$ *($$Integer$$, default: `$$1234$$`)* $$reverse-lookup$$:: $$Perform a reverse DNS lookup on the remote IP Address; if false, just the IP address is included in the message headers.$$ *($$Boolean$$, default: `$$false$$`)* $$socket-timeout$$:: $$The timeout (ms) before closing the socket when no data is received.$$ *($$Integer$$, default: `$$120000$$`)* -$$use-direct-buffers$$:: $$Whether or not to use direct buffers.$$ *($$Boolean$$, default: `$$false$$`)* +$$use-direct-buffers$$:: $$Whether to use direct buffers.$$ *($$Boolean$$, default: `$$false$$`)* //end::configuration-properties[] == Available Encoders diff --git a/applications/sink/tcp-sink/pom.xml b/applications/sink/tcp-sink/pom.xml index 93955b11d..1a1551a59 100644 --- a/applications/sink/tcp-sink/pom.xml +++ b/applications/sink/tcp-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - tcp-consumer + spring-tcp-consumer @@ -43,12 +43,13 @@ tcp sink ${project.version} - org.springframework.cloud.fn.consumer.tcp.TcpConsumerConfiguration.class + AUTOCONFIGURATION + tcpConsumer org.springframework.cloud.fn - tcp-consumer + spring-tcp-consumer diff --git a/applications/sink/throughput-sink/README.adoc b/applications/sink/throughput-sink/README.adoc index b7589fa8b..1a077a493 100644 --- a/applications/sink/throughput-sink/README.adoc +++ b/applications/sink/throughput-sink/README.adoc @@ -8,7 +8,7 @@ Sink that will count messages and log the observed throughput at a selected inte The **$$throughput$$** $$sink$$ has the following options: //tag::configuration-properties[] -$$throughput.report-every-ms$$:: $$how often to report.$$ *($$Integer$$, default: `$$$$`)* +$$throughput.report-every-ms$$:: $$how often to report.$$ *($$Integer$$, default: `$$1000$$`)* //end::configuration-properties[] //end::ref-doc[] diff --git a/applications/sink/twitter-message-sink/pom.xml b/applications/sink/twitter-message-sink/pom.xml index fbf1f4cf0..1d456076b 100644 --- a/applications/sink/twitter-message-sink/pom.xml +++ b/applications/sink/twitter-message-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - twitter-consumer + spring-twitter-consumer org.awaitility @@ -33,13 +33,6 @@ org.mock-server mockserver-netty - ${mockserver.version} - test - - - org.mock-server - mockserver-client-java - ${mockserver.version} test @@ -66,14 +59,14 @@ twitter-message sink ${project.version} - org.springframework.cloud.fn.consumer.twitter.message.TwitterMessageConsumerConfiguration.class - byteArrayTextToString|sendDirectMessageConsumer + AUTOCONFIGURATION + byteArrayTextToString|twitterSendMessageConsumer org.springframework.cloud.fn - twitter-consumer + spring-twitter-consumer diff --git a/applications/sink/twitter-message-sink/src/test/java/org/springframework/cloud/stream/app/sink/twitter/message/TwitterMessageSinkIntegrationTests.java b/applications/sink/twitter-message-sink/src/test/java/org/springframework/cloud/stream/app/sink/twitter/message/TwitterMessageSinkIntegrationTests.java index 736febcd2..e09e924a0 100644 --- a/applications/sink/twitter-message-sink/src/test/java/org/springframework/cloud/stream/app/sink/twitter/message/TwitterMessageSinkIntegrationTests.java +++ b/applications/sink/twitter-message-sink/src/test/java/org/springframework/cloud/stream/app/sink/twitter/message/TwitterMessageSinkIntegrationTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 the original author or authors. + * Copyright 2020-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,15 +36,12 @@ import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties; import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils; -import org.springframework.cloud.fn.consumer.twitter.message.TwitterMessageConsumerConfiguration; import org.springframework.cloud.stream.binder.test.InputDestination; import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.util.TestSocketUtils; import static org.assertj.core.api.Assertions.assertThat; import static org.mockserver.matchers.Times.unlimited; @@ -54,21 +51,18 @@ /** * @author Christian Tzolov + * @author Artem Bilan */ public class TwitterMessageSinkIntegrationTests { - private static final String MOCK_SERVER_IP = "127.0.0.1"; - - private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort(); - private static ClientAndServer mockServer; private static MockServerClient mockClient; @BeforeEach public void startMockServer() { - mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT); - mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT); + mockServer = ClientAndServer.startClientAndServer(); + mockClient = new MockServerClient("localhost", mockServer.getPort()); mockClient .when( @@ -111,7 +105,7 @@ public void directMessageScreenName() { try (ConfigurableApplicationContext context = new SpringApplicationBuilder( TestChannelBinderConfiguration.getCompleteConfiguration(TestTwitterMessageSinkApplication.class)) .web(WebApplicationType.NONE) - .run("--spring.cloud.function.definition=byteArrayTextToString|sendDirectMessageConsumer", + .run("--spring.cloud.function.definition=byteArrayTextToString|twitterSendMessageConsumer", "--twitter.message.update.screenName='user666'", @@ -149,7 +143,7 @@ public void directMessageUserId() { try (ConfigurableApplicationContext context = new SpringApplicationBuilder( TestChannelBinderConfiguration.getCompleteConfiguration(TestTwitterMessageSinkApplication.class)) .web(WebApplicationType.NONE) - .run("--spring.cloud.function.definition=byteArrayTextToString|sendDirectMessageConsumer", + .run("--spring.cloud.function.definition=byteArrayTextToString|twitterSendMessageConsumer", "--twitter.message.update.userId='1075751718749659136'", @@ -178,7 +172,7 @@ public void directMessageDefaults() { try (ConfigurableApplicationContext context = new SpringApplicationBuilder( TestChannelBinderConfiguration.getCompleteConfiguration(TestTwitterMessageSinkApplication.class)) .web(WebApplicationType.NONE) - .run("--spring.cloud.function.definition=byteArrayTextToString|sendDirectMessageConsumer", + .run("--spring.cloud.function.definition=byteArrayTextToString|twitterSendMessageConsumer", "--twitter.message.update.userId=headers['user']", "--twitter.message.update.text=payload.concat(\" with suffix \")", @@ -192,8 +186,8 @@ public void directMessageDefaults() { InputDestination source = context.getBean(InputDestination.class); assertThat(source).isNotNull(); - Map headers = Collections.singletonMap("user", "1075751718749659136"); - source.send(new GenericMessage("hello".getBytes(StandardCharsets.UTF_8), headers)); + Map headers = Collections.singletonMap("user", "1075751718749659136"); + source.send(new GenericMessage<>("hello".getBytes(StandardCharsets.UTF_8), headers)); mockClient.verify(request() .withMethod("POST") @@ -208,7 +202,6 @@ public void directMessageDefaults() { @SpringBootConfiguration @EnableAutoConfiguration - @Import(TwitterMessageConsumerConfiguration.class) public static class TestTwitterMessageSinkApplication { @Bean @@ -218,10 +211,11 @@ public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionPrope Function mockedConfiguration = toConfigurationBuilder.andThen( - new TwitterTestUtils().mockTwitterUrls( - String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT))); + new TwitterTestUtils().mockTwitterUrls("http://localhost:" + mockServer.getPort())); return mockedConfiguration.apply(properties).build(); } + } + } diff --git a/applications/sink/twitter-update-sink/README.adoc b/applications/sink/twitter-update-sink/README.adoc index 076390d22..22a4cef01 100644 --- a/applications/sink/twitter-update-sink/README.adoc +++ b/applications/sink/twitter-update-sink/README.adoc @@ -21,9 +21,9 @@ Properties grouped by prefix: === twitter.update -$$attachment-url$$:: $$(SpEL expression) In order for a URL to not be counted in the text body of an extended Tweet, provide a URL as a Tweet attachment. This URL must be a Tweet permalink, or Direct Message deep link. Arbitrary, non-Twitter URLs must remain in the text text. URLs passed to the attachment_url parameter not matching either a Tweet permalink or Direct Message deep link will fail at Tweet creation and cause an exception.$$ *($$Expression$$, default: `$$$$`)* -$$display-coordinates$$:: $$(SpEL expression) Whether or not to put a pin on the exact coordinates a Tweet has been sent from.$$ *($$Expression$$, default: `$$$$`)* -$$in-reply-to-status-id$$:: $$(SpEL expression) The ID of an existing text that the update is in reply to. Note: This parameter will be ignored unless the author of the Tweet this parameter references is mentioned within the text text. Therefore, you must include @username, where username is the author of the referenced Tweet, within the update. When inReplyToStatusId is set the auto_populate_reply_metadata is automatically set as well. Later ensures that leading @mentions will be looked up from the original Tweet, and added to the new Tweet from there. This wil append @mentions into the metadata of an extended Tweet as a reply chain grows, until the limit on @mentions is reached. In cases where the original Tweet has been deleted, the reply will fail.$$ *($$Expression$$, default: `$$$$`)* +$$attachment-url$$:: $$(SpEL expression) In order for a URL to not be counted in the text body of an extended Tweet, provide a URL as a Tweet attachment. This URL must be a Tweet permalink, or Direct Message deep link. Arbitrary, non-Twitter URLs must remain in the text. URLs passed to the attachment_url parameter not matching either a Tweet permalink or Direct Message deep link will fail at Tweet creation and cause an exception.$$ *($$Expression$$, default: `$$$$`)* +$$display-coordinates$$:: $$(SpEL expression) Whether to put a pin on the exact coordinates a Tweet has been sent from.$$ *($$Expression$$, default: `$$$$`)* +$$in-reply-to-status-id$$:: $$(SpEL expression) The ID of an existing text that the update is in reply to. Note: This parameter will be ignored unless the author of the Tweet this parameter references is mentioned within the text. Therefore, you must include @username, where username is the author of the referenced Tweet, within the update. When inReplyToStatusId is set the auto_populate_reply_metadata is automatically set as well. Later ensures that leading @mentions will be looked up from the original Tweet, and added to the new Tweet from there. This wil append @mentions into the metadata of an extended Tweet as a reply chain grows, until the limit on @mentions is reached. In cases where the original Tweet has been deleted, the reply will fail.$$ *($$Expression$$, default: `$$$$`)* $$media-ids$$:: $$(SpEL expression) A comma-delimited list of media_ids to associate with the Tweet. You may include up to 4 photos or 1 animated GIF or 1 video in a Tweet. See Uploading Media for further details on uploading media.$$ *($$Expression$$, default: `$$$$`)* $$place-id$$:: $$(SpEL expression) A place in the world.$$ *($$Expression$$, default: `$$$$`)* $$text$$:: $$(SpEL expression) The text of the text update. URL encode as necessary. t.co link wrapping will affect character counts. Defaults to message's payload$$ *($$Expression$$, default: `$$payload$$`)* @@ -31,7 +31,7 @@ $$text$$:: $$(SpEL expression) The text of the text update. URL encode as necess === twitter.update.location $$lat$$:: $$The latitude of the location this Tweet refers to. This parameter will be ignored unless it is inside the range -90.0 to +90.0 (North is positive) inclusive. It will also be ignored if there is no corresponding long parameter.$$ *($$Expression$$, default: `$$$$`)* -$$lon$$:: $$The longitude of the location this Tweet refers to. The valid ranges for longitude are -180.0 to +180.0 (East is positive) inclusive. This parameter will be ignored if outside that range, if it is not a number, if geo_enabled is disabled, or if there no corresponding lat parameter.$$ *($$Expression$$, default: `$$$$`)* +$$lon$$:: $$The longitude of the location this Tweet refers to. The valid ranges for longitude are -180.0 to +180.0 (East is positive) inclusive. This parameter will be ignored if outside that range, if it is not a number, if geo_enabled is disabled, or if there is no corresponding lat parameter.$$ *($$Expression$$, default: `$$$$`)* //end::configuration-properties[] //end::ref-doc[] diff --git a/applications/sink/twitter-update-sink/pom.xml b/applications/sink/twitter-update-sink/pom.xml index b21721fb8..e150c7b40 100644 --- a/applications/sink/twitter-update-sink/pom.xml +++ b/applications/sink/twitter-update-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - twitter-consumer + spring-twitter-consumer org.awaitility @@ -33,13 +33,6 @@ org.mock-server mockserver-netty - ${mockserver.version} - test - - - org.mock-server - mockserver-client-java - ${mockserver.version} test @@ -66,14 +59,14 @@ twitter-update sink ${project.version} - org.springframework.cloud.fn.consumer.twitter.status.update.TwitterUpdateConsumerConfiguration.class + AUTOCONFIGURATION byteArrayTextToString|twitterStatusUpdateConsumer org.springframework.cloud.fn - twitter-consumer + spring-twitter-consumer diff --git a/applications/sink/twitter-update-sink/src/test/java/org/springframework/cloud/stream/app/sink/twitter/update/TwitterUpdateSinkIntegrationTests.java b/applications/sink/twitter-update-sink/src/test/java/org/springframework/cloud/stream/app/sink/twitter/update/TwitterUpdateSinkIntegrationTests.java index 9ed84dd7c..9b5e27a1a 100644 --- a/applications/sink/twitter-update-sink/src/test/java/org/springframework/cloud/stream/app/sink/twitter/update/TwitterUpdateSinkIntegrationTests.java +++ b/applications/sink/twitter-update-sink/src/test/java/org/springframework/cloud/stream/app/sink/twitter/update/TwitterUpdateSinkIntegrationTests.java @@ -17,7 +17,6 @@ package org.springframework.cloud.stream.app.sink.twitter.update; import java.nio.charset.StandardCharsets; -import java.util.concurrent.TimeUnit; import java.util.function.Function; import org.junit.jupiter.api.AfterAll; @@ -34,16 +33,13 @@ import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties; import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils; -import org.springframework.cloud.fn.consumer.twitter.status.update.TwitterUpdateConsumerConfiguration; import org.springframework.cloud.fn.consumer.twitter.status.update.TwitterUpdateConsumerProperties; import org.springframework.cloud.stream.binder.test.InputDestination; import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.util.TestSocketUtils; import static org.assertj.core.api.Assertions.assertThat; import static org.mockserver.matchers.Times.unlimited; @@ -53,21 +49,19 @@ /** * @author Christian Tzolov + * @author Artem Bilan */ public class TwitterUpdateSinkIntegrationTests { - private static final String MOCK_SERVER_IP = "127.0.0.1"; - - private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort(); - private static ClientAndServer mockServer; private static MockServerClient mockClient; @BeforeAll public static void startMockServer() { - mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT); - mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT); + mockServer = ClientAndServer.startClientAndServer(); + mockClient = new MockServerClient("localhost", mockServer.getPort()); + mockClient .when( request().withMethod("POST").withPath("/statuses/update.json"), @@ -75,8 +69,7 @@ public static void startMockServer() { .respond(response() .withStatusCode(200) .withHeader("Content-Type", "application/json; charset=utf-8") - .withBody(TwitterTestUtils.asString("classpath:/response/update_test_1.json")) - .withDelay(TimeUnit.SECONDS, 1)); + .withBody(TwitterTestUtils.asString("classpath:/response/update_test_1.json"))); } @AfterAll @@ -211,7 +204,6 @@ public void updateWithAllParams() { @SpringBootConfiguration @EnableAutoConfiguration - @Import(TwitterUpdateConsumerConfiguration.class) public static class TestTwitterUpdateSinkApplication { @Bean @@ -221,10 +213,11 @@ public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionPrope Function mockedConfiguration = toConfigurationBuilder.andThen( - new TwitterTestUtils().mockTwitterUrls( - String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT))); + new TwitterTestUtils().mockTwitterUrls("http://localhost:" + mockServer.getPort())); return mockedConfiguration.apply(properties).build(); } + } + } diff --git a/applications/sink/wavefront-sink/pom.xml b/applications/sink/wavefront-sink/pom.xml index b862c686f..77686418b 100644 --- a/applications/sink/wavefront-sink/pom.xml +++ b/applications/sink/wavefront-sink/pom.xml @@ -17,11 +17,11 @@ org.springframework.cloud.fn - wavefront-consumer + spring-wavefront-consumer org.springframework.cloud.fn - function-test-support + spring-function-test-support test @@ -48,16 +48,14 @@ wavefront sink ${project.version} - - org.springframework.cloud.fn.consumer.wavefront.WavefrontConsumerConfiguration.class - + AUTOCONFIGURATION wavefrontConsumer org.springframework.cloud.fn - wavefront-consumer + spring-wavefront-consumer diff --git a/applications/sink/websocket-sink/README.adoc b/applications/sink/websocket-sink/README.adoc index 9a99b02ee..7032ae0c2 100644 --- a/applications/sink/websocket-sink/README.adoc +++ b/applications/sink/websocket-sink/README.adoc @@ -7,11 +7,11 @@ A simple Websocket Sink implementation. The following options are supported: //tag::configuration-properties[] -$$websocket.consumer.log-level$$:: $$the logLevel for netty channels. Default is WARN$$ *($$String$$, default: `$$$$`)* -$$websocket.consumer.path$$:: $$the path on which a WebsocketSink consumer needs to connect. Default is /websocket$$ *($$String$$, default: `$$/websocket$$`)* -$$websocket.consumer.port$$:: $$the port on which the Netty server listens. Default is 9292$$ *($$Integer$$, default: `$$9292$$`)* -$$websocket.consumer.ssl$$:: $$whether or not to create a {@link io.netty.handler.ssl.SslContext}.$$ *($$Boolean$$, default: `$$false$$`)* -$$websocket.consumer.threads$$:: $$the number of threads for the Netty {@link io.netty.channel.EventLoopGroup}. Default is 1$$ *($$Integer$$, default: `$$1$$`)* +$$websocket.consumer.log-level$$:: $$The logLevel for netty channels. Default is WARN$$ *($$String$$, default: `$$$$`)* +$$websocket.consumer.path$$:: $$The path on which a WebsocketSink consumer needs to connect. Default is /websocket$$ *($$String$$, default: `$$/websocket$$`)* +$$websocket.consumer.port$$:: $$The port on which the Netty server listens. Default is 9292$$ *($$Integer$$, default: `$$9292$$`)* +$$websocket.consumer.ssl$$:: $$Whether to create a {@link io.netty.handler.ssl.SslContext}.$$ *($$Boolean$$, default: `$$false$$`)* +$$websocket.consumer.threads$$:: $$The number of threads for the Netty {@link io.netty.channel.EventLoopGroup}. Default is 1$$ *($$Integer$$, default: `$$1$$`)* //end::configuration-properties[] == Examples diff --git a/applications/sink/websocket-sink/pom.xml b/applications/sink/websocket-sink/pom.xml index 68b4f59f9..bfaf12984 100644 --- a/applications/sink/websocket-sink/pom.xml +++ b/applications/sink/websocket-sink/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - websocket-consumer + spring-websocket-consumer org.springframework.boot @@ -26,7 +26,7 @@ org.springframework.cloud.fn - function-test-support + spring-function-test-support test @@ -53,13 +53,13 @@ websocket sink ${project.version} - org.springframework.cloud.fn.consumer.websocket.WebsocketConsumerConfiguration.class - + AUTOCONFIGURATION + websocketConsumer org.springframework.cloud.fn - websocket-consumer + spring-websocket-consumer diff --git a/applications/sink/xmpp-sink/README.adoc b/applications/sink/xmpp-sink/README.adoc index 92d0bae36..82de01d05 100644 --- a/applications/sink/xmpp-sink/README.adoc +++ b/applications/sink/xmpp-sink/README.adoc @@ -29,8 +29,8 @@ $$chat-to$$:: $$XMPP handle to send message to.$$ *($$String$$, default: `$$$$`)* $$password$$:: $$The Password for the connected user.$$ *($$String$$, default: `$$$$`)* -$$port$$:: $$Port for connecting to the host. - Default Client Port: 5222$$ *($$Integer$$, default: `$$5222$$`)* -$$resource$$:: $$The Resource to bind to on the XMPP Host. - Can be empty, server will generate one if not set$$ *($$String$$, default: `$$$$`)* +$$port$$:: $$Port for connecting to the host. - Default Client Port: 5222$$ *($$Integer$$, default: `$$5222$$`)* +$$resource$$:: $$The Resource to bind to on the XMPP Host. - Can be empty, server will generate one if not set$$ *($$String$$, default: `$$$$`)* $$security-mode$$:: $$$$ *($$SecurityMode$$, default: `$$$$`, possible values: `required`,`ifpossible`,`disabled`)* $$service-name$$:: $$The Service Name to set for the XMPP Domain.$$ *($$String$$, default: `$$$$`)* $$subscription-mode$$:: $$$$ *($$SubscriptionMode$$, default: `$$$$`, possible values: `accept_all`,`reject_all`,`manual`)* diff --git a/applications/sink/xmpp-sink/pom.xml b/applications/sink/xmpp-sink/pom.xml index 647a1a5a9..a1069495e 100644 --- a/applications/sink/xmpp-sink/pom.xml +++ b/applications/sink/xmpp-sink/pom.xml @@ -20,7 +20,7 @@ org.springframework.cloud.fn - xmpp-consumer + spring-xmpp-consumer @@ -37,7 +37,7 @@ org.springframework.cloud.fn - function-test-support + spring-function-test-support ${project.version} test @@ -66,14 +66,13 @@ xmpp sink ${project.version} - org.springframework.cloud.fn.consumer.xmpp.XmppConsumerConfiguration.class - - + AUTOCONFIGURATION + xmppConsumer org.springframework.cloud.fn - xmpp-consumer + spring-xmpp-consumer org.springframework.cloud.stream.app diff --git a/applications/sink/zeromq-sink/pom.xml b/applications/sink/zeromq-sink/pom.xml index 5ba060ce9..52a49665d 100644 --- a/applications/sink/zeromq-sink/pom.xml +++ b/applications/sink/zeromq-sink/pom.xml @@ -20,7 +20,7 @@ org.springframework.cloud.fn - zeromq-consumer + spring-zeromq-consumer @@ -58,14 +58,13 @@ zeromq sink ${project.version} - org.springframework.cloud.fn.consumer.zeromq.ZeroMqConsumerConfiguration.class - - + AUTOCONFIGURATION + zeromqConsumer org.springframework.cloud.fn - zeromq-consumer + spring-zeromq-consumer org.springframework.cloud.stream.app diff --git a/applications/source/debezium-source/README.adoc b/applications/source/debezium-source/README.adoc index 263450512..4cb6fb3d8 100644 --- a/applications/source/debezium-source/README.adoc +++ b/applications/source/debezium-source/README.adoc @@ -24,14 +24,14 @@ Properties grouped by prefix: === debezium $$debezium-native-configuration$$:: $$$$ *($$Properties$$, default: `$$$$`)* -$$header-format$$:: $${@link ChangeEvent} header format. Defaults to 'JSON'.$$ *($$DebeziumFormat$$, default: `$$$$`, possible values: `JSON`,`AVRO`,`PROTOBUF`)* $$offset-commit-policy$$:: $$The policy that defines when the offsets should be committed to offset storage.$$ *($$DebeziumOffsetCommitPolicy$$, default: `$$$$`, possible values: `ALWAYS`,`PERIODIC`,`DEFAULT`)* -$$payload-format$$:: $${@link ChangeEvent} Key and Payload formats. Defaults to 'JSON'.$$ *($$DebeziumFormat$$, default: `$$$$`, possible values: `JSON`,`AVRO`,`PROTOBUF`)* +$$payload-format$$:: $${@code io.debezium.engine.ChangeEvent} Key and Payload formats. Defaults to 'JSON'.$$ *($$DebeziumFormat$$, default: `$$$$`, possible values: `JSON`,`AVRO`,`PROTOBUF`)* $$properties$$:: $$Spring pass-trough wrapper for debezium configuration properties. All properties with a 'debezium.properties.*' prefix are native Debezium properties.$$ *($$Map$$, default: `$$$$`)* === debezium.supplier -$$copy-headers$$:: $$Copy Change Event headers into Message headers.$$ *($$Boolean$$, default: `$$true$$`)* +$$enable-empty-payload$$:: $$Enable support for tombstone (aka delete) messages.$$ *($$Boolean$$, default: `$$true$$`)* +$$header-names-to-map$$:: $$Patterns for {@code ChangeEvent.headers()} to map.$$ *($$String[]$$, default: `$$[*]$$`)* //end::configuration-properties[] ==== Event flattening configuration diff --git a/applications/source/debezium-source/pom.xml b/applications/source/debezium-source/pom.xml index f3eb9e9f5..aeef35f6b 100644 --- a/applications/source/debezium-source/pom.xml +++ b/applications/source/debezium-source/pom.xml @@ -22,15 +22,13 @@ org.springframework.cloud.fn - debezium-supplier - ${java-functions.version} + spring-debezium-supplier org.springframework.cloud.fn - function-test-support - ${java-functions.version} + spring-function-test-support test @@ -159,7 +157,7 @@ org.springframework.cloud.fn - debezium-supplier + spring-debezium-supplier org.springframework.cloud.stream.app diff --git a/applications/source/debezium-source/src/test/java/org/springframework/cloud/stream/app/source/debezium/integration/DebeziumDeleteHandlingIntegrationTest.java b/applications/source/debezium-source/src/test/java/org/springframework/cloud/stream/app/source/debezium/integration/DebeziumDeleteHandlingIntegrationTest.java index 6b331df84..83499adf5 100644 --- a/applications/source/debezium-source/src/test/java/org/springframework/cloud/stream/app/source/debezium/integration/DebeziumDeleteHandlingIntegrationTest.java +++ b/applications/source/debezium-source/src/test/java/org/springframework/cloud/stream/app/source/debezium/integration/DebeziumDeleteHandlingIntegrationTest.java @@ -18,6 +18,7 @@ import java.time.Duration; import java.util.List; +import java.util.Optional; import org.junit.jupiter.api.Tag; import org.junit.jupiter.params.ParameterizedTest; @@ -26,19 +27,15 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import org.springframework.boot.test.context.FilteredClassLoader; import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.boot.test.context.runner.ContextConsumer; import org.springframework.cloud.fn.common.debezium.DebeziumProperties; -import org.springframework.cloud.fn.supplier.debezium.DebeziumReactiveConsumerConfiguration; import org.springframework.cloud.stream.binder.test.OutputDestination; import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.context.ApplicationContext; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.kafka.support.KafkaNull; import org.springframework.messaging.Message; import org.springframework.test.jdbc.JdbcTestUtils; -import org.springframework.util.ClassUtils; import static org.assertj.core.api.Assertions.assertThat; @@ -106,10 +103,6 @@ public class DebeziumDeleteHandlingIntegrationTest { "debezium.properties.transforms.unwrap.delete.handling.mode=rewrite,debezium.properties.transforms.unwrap.drop.tombstones=false" }) public void handleRecordDeletions(String properties) { - contextRunner.withPropertyValues(properties.split(",")) - .withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the - .run(consumer); - contextRunner.withPropertyValues(properties.split(",")) .run(consumer); } @@ -123,10 +116,6 @@ private String toString(Object object) { JdbcTemplate jdbcTemplate = context.getBean(JdbcTemplate.class); DebeziumProperties props = context.getBean(DebeziumProperties.class); - boolean isKafkaPresent = ClassUtils.isPresent( - DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, - context.getClassLoader()); - String deleteHandlingMode = props.getProperties().get("transforms.unwrap.delete.handling.mode"); String isDropTombstones = props.getProperties().get("transforms.unwrap.drop.tombstones"); @@ -153,15 +142,14 @@ else if (deleteHandlingMode.equals("none")) { else if (deleteHandlingMode.equals("rewrite")) { received = outputDestination.receive(Duration.ofSeconds(10).toMillis(), DebeziumTestUtils.BINDING_NAME); assertThat(received).isNotNull(); - assertThat(toString(received.getPayload()).contains("\"__deleted\":\"true\"")); + assertThat(toString(received.getPayload())).contains("\"__deleted\":\"true\""); } - if (!(isDropTombstones.equals("true")) && isKafkaPresent) { + if (!(isDropTombstones.equals("true"))) { received = outputDestination.receive(Duration.ofSeconds(10).toMillis(), DebeziumTestUtils.BINDING_NAME); assertThat(received).isNotNull(); // Tombstones event should have KafkaNull payload - assertThat(received.getPayload().getClass().getCanonicalName()) - .isEqualTo(DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL); + assertThat(received.getPayload()).isEqualTo(Optional.empty()); Object keyRaw = received.getHeaders().get("debezium_key"); String key = (keyRaw instanceof byte[]) ? new String((byte[]) keyRaw) : "" + keyRaw; diff --git a/applications/source/debezium-source/src/test/java/org/springframework/cloud/stream/app/source/debezium/integration/DebeziumFlatteningIntegrationTest.java b/applications/source/debezium-source/src/test/java/org/springframework/cloud/stream/app/source/debezium/integration/DebeziumFlatteningIntegrationTest.java index 9f95965af..9e392d66b 100644 --- a/applications/source/debezium-source/src/test/java/org/springframework/cloud/stream/app/source/debezium/integration/DebeziumFlatteningIntegrationTest.java +++ b/applications/source/debezium-source/src/test/java/org/springframework/cloud/stream/app/source/debezium/integration/DebeziumFlatteningIntegrationTest.java @@ -18,6 +18,7 @@ import java.time.Duration; import java.util.List; +import java.util.Optional; import net.javacrumbs.jsonunit.JsonAssert; import net.javacrumbs.jsonunit.core.Configuration; @@ -27,19 +28,15 @@ import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import org.springframework.boot.test.context.FilteredClassLoader; import org.springframework.boot.test.context.runner.ApplicationContextRunner; import org.springframework.boot.test.context.runner.ContextConsumer; import org.springframework.cloud.fn.common.debezium.DebeziumProperties; -import org.springframework.cloud.fn.supplier.debezium.DebeziumReactiveConsumerConfiguration; import org.springframework.cloud.stream.binder.test.OutputDestination; import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.context.ApplicationContext; import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.kafka.support.KafkaNull; import org.springframework.messaging.Message; import org.springframework.test.jdbc.JdbcTestUtils; -import org.springframework.util.ClassUtils; import org.springframework.util.StringUtils; import static org.assertj.core.api.Assertions.assertThat; @@ -96,14 +93,7 @@ public class DebeziumFlatteningIntegrationTest { "app.datasource.type=com.zaxxer.hikari.HikariDataSource"); @Test - public void noFlattenedResponseNoKafka() { - contextRunner - .withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath - .run(noFlatteningTest); - } - - @Test - public void noFlattenedResponseWithKafka() { + public void noFlattenedResponse() { contextRunner.run(noFlatteningTest); } @@ -111,15 +101,11 @@ public void noFlattenedResponseWithKafka() { OutputDestination outputDestination = context.getBean(OutputDestination.class); JdbcTemplate jdbcTemplate = context.getBean(JdbcTemplate.class); - boolean isKafkaPresent = ClassUtils.isPresent( - DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, - context.getClassLoader()); - List> messages = DebeziumTestUtils.receiveAll(outputDestination); assertThat(messages).hasSizeGreaterThanOrEqualTo(52); JsonAssert.assertJsonEquals(DebeziumTestUtils.resourceToString( - "classpath:/json/mysql_ddl_drop_inventory_address_table.json"), + "classpath:/json/mysql_ddl_drop_inventory_address_table.json"), toString(messages.get(1).getPayload()), Configuration.empty().whenIgnoringPaths("schemaName", "tableChanges", "source.sequence", "source.ts_ms", "ts_ms")); @@ -143,7 +129,7 @@ public void noFlattenedResponseWithKafka() { messages = DebeziumTestUtils.receiveAll(outputDestination); - assertThat(messages).hasSize(isKafkaPresent ? 4 : 3); + assertThat(messages).hasSize(4); JsonAssert.assertJsonEquals( DebeziumTestUtils.resourceToString("classpath:/json/mysql_update_inventory_customers.json"), @@ -160,30 +146,13 @@ public void noFlattenedResponseWithKafka() { JsonAssert.assertJsonEquals("{\"id\":" + newRecordId + "}", toString(messages.get(1).getHeaders().get("debezium_key"))); - if (isKafkaPresent) { - assertThat(messages.get(3).getPayload().getClass().getCanonicalName()) - .isEqualTo(DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, - "Tombstones event should have KafkaNull payload"); - assertThat(messages.get(3).getHeaders().get("debezium_destination")) - .isEqualTo("my-topic.inventory.customers"); - JsonAssert.assertJsonEquals("{\"id\":" + newRecordId + "}", - toString(messages.get(3).getHeaders().get("debezium_key"))); - } + assertThat(messages.get(3).getPayload()).isEqualTo(Optional.empty()); + assertThat(messages.get(3).getHeaders().get("debezium_destination")) + .isEqualTo("my-topic.inventory.customers"); + JsonAssert.assertJsonEquals("{\"id\":" + newRecordId + "}", + toString(messages.get(3).getHeaders().get("debezium_key"))); }; - @Test - public void flattenedResponseNoKafka() { - contextRunner - .withPropertyValues("debezium.properties.transforms=unwrap", - "debezium.properties.transforms.unwrap.type=io.debezium.transforms.ExtractNewRecordState", - "debezium.properties.transforms.unwrap.add.fields=name,db,op", - "debezium.properties.transforms.unwrap.add.headers=name,op", - "debezium.properties.transforms.unwrap.delete.handling.mode=none", - "debezium.properties.transforms.unwrap.drop.tombstones=false") - .withClassLoader(new FilteredClassLoader(KafkaNull.class)) // Remove Kafka from the classpath - .run(flatteningTest); - } - @Test public void flattenedResponseWithKafka() { contextRunner @@ -212,10 +181,6 @@ public void flattenedResponseWithKafkaDropTombstone() { OutputDestination outputDestination = context.getBean(OutputDestination.class); JdbcTemplate jdbcTemplate = context.getBean(JdbcTemplate.class); - boolean isKafkaPresent = ClassUtils.isPresent( - DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, - context.getClassLoader()); - List> messages = DebeziumTestUtils.receiveAll(outputDestination); assertThat(messages).hasSizeGreaterThanOrEqualTo(52); @@ -225,7 +190,7 @@ public void flattenedResponseWithKafkaDropTombstone() { String isDropTombstones = props.getProperties().get("transforms.unwrap.drop.tombstones"); JsonAssert.assertJsonEquals(DebeziumTestUtils.resourceToString( - "classpath:/json/mysql_ddl_drop_inventory_address_table.json"), + "classpath:/json/mysql_ddl_drop_inventory_address_table.json"), toString(messages.get(1).getPayload()), Configuration.empty().whenIgnoringPaths("schemaName", "tableChanges", "source.sequence", "source.ts_ms", "ts_ms")); @@ -256,7 +221,7 @@ public void flattenedResponseWithKafkaDropTombstone() { messages = DebeziumTestUtils.receiveAll(outputDestination); - assertThat(messages).hasSize((isDropTombstones.equals("false") && isKafkaPresent) ? 4 : 3); + assertThat(messages).hasSize((isDropTombstones.equals("false")) ? 4 : 3); JsonAssert.assertJsonEquals( DebeziumTestUtils.resourceToString("classpath:/json/mysql_flattened_update_inventory_customers.json"), @@ -273,10 +238,8 @@ public void flattenedResponseWithKafkaDropTombstone() { toString(messages.get(1).getHeaders().get("debezium_key"))); } - if (isDropTombstones.equals("false") && isKafkaPresent) { - assertThat(messages.get(3).getPayload().getClass().getCanonicalName()) - .isEqualTo(DebeziumReactiveConsumerConfiguration.ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, - "Tombstones event should have KafkaNull payload"); + if (isDropTombstones.equals("false")) { + assertThat(messages.get(3).getPayload()).isEqualTo(Optional.empty()); assertThat(messages.get(3).getHeaders().get("debezium_destination")) .isEqualTo("my-topic.inventory.customers"); JsonAssert.assertJsonEquals("{\"id\":" + newRecordId + "}", diff --git a/applications/source/file-source/pom.xml b/applications/source/file-source/pom.xml index 4f510888b..914b528a1 100644 --- a/applications/source/file-source/pom.xml +++ b/applications/source/file-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - file-supplier + spring-file-supplier org.springframework.cloud.stream.app @@ -49,13 +49,13 @@ file source ${project.version} - org.springframework.cloud.fn.supplier.file.FileSupplierConfiguration.class - + AUTOCONFIGURATION + fileSupplier org.springframework.cloud.fn - file-supplier + spring-file-supplier org.springframework.cloud.stream.app diff --git a/applications/source/ftp-source/pom.xml b/applications/source/ftp-source/pom.xml index 053d13b62..7b73bc724 100644 --- a/applications/source/ftp-source/pom.xml +++ b/applications/source/ftp-source/pom.xml @@ -17,11 +17,17 @@ org.springframework.cloud.fn - ftp-supplier + spring-ftp-supplier org.springframework.cloud.fn - function-test-support + spring-function-test-support + test + + + org.apache.ftpserver + ftpserver-core + 1.2.0 test @@ -54,13 +60,13 @@ ftp source ${project.version} - org.springframework.cloud.fn.supplier.ftp.FtpSupplierConfiguration.class - + AUTOCONFIGURATION + ftpSupplier org.springframework.cloud.fn - ftp-supplier + spring-ftp-supplier org.springframework.cloud.stream.app diff --git a/applications/source/ftp-source/src/test/java/org/springframework/cloud/stream/app/source/ftp/FtpSourceTests.java b/applications/source/ftp-source/src/test/java/org/springframework/cloud/stream/app/source/ftp/FtpSourceTests.java index 46430cd42..9492d4749 100644 --- a/applications/source/ftp-source/src/test/java/org/springframework/cloud/stream/app/source/ftp/FtpSourceTests.java +++ b/applications/source/ftp-source/src/test/java/org/springframework/cloud/stream/app/source/ftp/FtpSourceTests.java @@ -23,7 +23,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.supplier.ftp.FtpSupplierConfiguration; import org.springframework.cloud.fn.supplier.ftp.FtpSupplierProperties; import org.springframework.cloud.fn.test.support.ftp.FtpTestSupport; import org.springframework.cloud.stream.binder.test.OutputDestination; @@ -54,7 +53,7 @@ public class FtpSourceTests extends FtpTestSupport { FtpSupplierProperties config; @Test - public void testFtpource() { + public void testFtpSource() { Message message = output.receive(10000, "ftpSupplier-out-0"); assertThat(new File(new String(message.getPayload()).replaceAll("\"", ""))).isEqualTo( new File(this.config.getLocalDir(), "ftpSource1.txt")); @@ -64,7 +63,7 @@ public void testFtpource() { } @SpringBootApplication - @Import({TestChannelBinderConfiguration.class, FtpSupplierConfiguration.class}) + @Import(TestChannelBinderConfiguration.class) public static class SampleConfiguration { @Bean @@ -77,4 +76,5 @@ public FtpSupplierProperties ftpSupplierProperties() { } } + } diff --git a/applications/source/http-source/README.adoc b/applications/source/http-source/README.adoc index 06ee12627..786572e30 100644 --- a/applications/source/http-source/README.adoc +++ b/applications/source/http-source/README.adoc @@ -23,13 +23,13 @@ The **$$http$$** $$source$$ supports the following configuration properties: Properties grouped by prefix: -=== http.cors +=== http.supplier.cors $$allow-credentials$$:: $$Whether the browser should include any cookies associated with the domain of the request being annotated.$$ *($$Boolean$$, default: `$$$$`)* $$allowed-headers$$:: $$List of request headers that can be used during the actual request.$$ *($$String[]$$, default: `$$$$`)* $$allowed-origins$$:: $$List of allowed origins, e.g. https://domain1.com.$$ *($$String[]$$, default: `$$$$`)* -=== http +=== http.supplier $$mapped-request-headers$$:: $$Headers that will be mapped.$$ *($$String[]$$, default: `$$$$`)* $$path-pattern$$:: $$HTTP endpoint path mapping.$$ *($$String$$, default: `$$/$$`)* diff --git a/applications/source/http-source/pom.xml b/applications/source/http-source/pom.xml index 3fe7500e0..33ec02423 100644 --- a/applications/source/http-source/pom.xml +++ b/applications/source/http-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - http-supplier + spring-http-supplier org.springframework.boot @@ -53,8 +53,8 @@ http source ${project.version} - org.springframework.cloud.fn.supplier.http.HttpSupplierConfiguration.class - + AUTOCONFIGURATION + httpSupplier reactive> false @@ -65,7 +65,7 @@ org.springframework.cloud.fn - http-supplier + spring-http-supplier org.springframework.cloud.stream.app diff --git a/applications/source/jdbc-source/pom.xml b/applications/source/jdbc-source/pom.xml index 733a66fdb..a97cbaf20 100644 --- a/applications/source/jdbc-source/pom.xml +++ b/applications/source/jdbc-source/pom.xml @@ -17,14 +17,9 @@ org.springframework.cloud.fn - jdbc-supplier + spring-jdbc-supplier - org.springframework.cloud - spring-cloud-stream-test-binder - test - - com.h2database h2 test @@ -58,13 +53,13 @@ jdbc source ${project.version} - org.springframework.cloud.fn.supplier.jdbc.JdbcSupplierConfiguration.class - + AUTOCONFIGURATION + jdbcSupplier org.springframework.cloud.fn - jdbc-supplier + spring-jdbc-supplier org.springframework.cloud.stream.app diff --git a/applications/source/jms-source/README.adoc b/applications/source/jms-source/README.adoc index 6cc0644dc..113c61990 100644 --- a/applications/source/jms-source/README.adoc +++ b/applications/source/jms-source/README.adoc @@ -28,10 +28,11 @@ $$pub-sub-domain$$:: $$Whether the default destination type is topic.$$ *($$Bool === spring.jms.listener -$$acknowledge-mode$$:: $$Acknowledge mode of the container. By default, the listener is transacted with automatic acknowledgment.$$ *($$AcknowledgeMode$$, default: `$$$$`, possible values: `AUTO`,`CLIENT`,`DUPS_OK`)* +$$acknowledge-mode$$:: $$$$ *($$AcknowledgeMode$$, default: `$$$$`)* $$auto-startup$$:: $$Start the container automatically on startup.$$ *($$Boolean$$, default: `$$true$$`)* -$$concurrency$$:: $$Minimum number of concurrent consumers. When max-concurrency is not specified the minimum will also be used as the maximum.$$ *($$Integer$$, default: `$$$$`)* +$$concurrency$$:: $$$$ *($$Integer$$, default: `$$$$`)* $$max-concurrency$$:: $$Maximum number of concurrent consumers.$$ *($$Integer$$, default: `$$$$`)* +$$min-concurrency$$:: $$Minimum number of concurrent consumers. When max-concurrency is not specified the minimum will also be used as the maximum.$$ *($$Integer$$, default: `$$$$`)* $$receive-timeout$$:: $$Timeout to use for receive calls. Use -1 for a no-wait receive or 0 for no timeout at all. The latter is only feasible if not running within a transaction manager and is generally discouraged since it prevents clean shutdown.$$ *($$Duration$$, default: `$$1s$$`)* //end::configuration-properties[] diff --git a/applications/source/jms-source/pom.xml b/applications/source/jms-source/pom.xml index 7f970c16f..1f7b00b0b 100644 --- a/applications/source/jms-source/pom.xml +++ b/applications/source/jms-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - jms-supplier + spring-jms-supplier jakarta.jms @@ -59,13 +59,13 @@ jms source ${project.version} - org.springframework.cloud.fn.supplier.jms.JmsSupplierConfiguration.class - + AUTOCONFIGURATION + jmsSupplier org.springframework.cloud.fn - jms-supplier + spring-jms-supplier org.springframework.cloud.stream.app diff --git a/applications/source/kafka-source/README.adoc b/applications/source/kafka-source/README.adoc index d495cd612..8a425c1a3 100644 --- a/applications/source/kafka-source/README.adoc +++ b/applications/source/kafka-source/README.adoc @@ -50,6 +50,7 @@ $$ack-mode$$:: $$Listener AckMode. See the spring-kafka documentation.$$ *($$Ack $$ack-time$$:: $$Time between offset commits when ackMode is "TIME" or "COUNT_TIME".$$ *($$Duration$$, default: `$$$$`)* $$async-acks$$:: $$Support for asynchronous record acknowledgements. Only applies when spring.kafka.listener.ack-mode is manual or manual-immediate.$$ *($$Boolean$$, default: `$$$$`)* $$auto-startup$$:: $$Whether to auto start the container.$$ *($$Boolean$$, default: `$$true$$`)* +$$change-consumer-thread-name$$:: $$Whether to instruct the container to change the consumer thread name during initialization.$$ *($$Boolean$$, default: `$$$$`)* $$client-id$$:: $$Prefix for the listener's consumer client.id property.$$ *($$String$$, default: `$$$$`)* $$concurrency$$:: $$Number of threads to run in the listener containers.$$ *($$Integer$$, default: `$$$$`)* $$idle-between-polls$$:: $$Sleep interval between Consumer.poll(Duration) calls.$$ *($$Duration$$, default: `$$0$$`)* @@ -60,6 +61,7 @@ $$log-container-config$$:: $$Whether to log the container configuration during i $$missing-topics-fatal$$:: $$Whether the container should fail to start if at least one of the configured topics are not present on the broker.$$ *($$Boolean$$, default: `$$false$$`)* $$monitor-interval$$:: $$Time between checks for non-responsive consumers. If a duration suffix is not specified, seconds will be used.$$ *($$Duration$$, default: `$$$$`)* $$no-poll-threshold$$:: $$Multiplier applied to "pollTimeout" to determine if a consumer is non-responsive.$$ *($$Float$$, default: `$$$$`)* +$$observation-enabled$$:: $$Whether to enable observation.$$ *($$Boolean$$, default: `$$false$$`)* $$poll-timeout$$:: $$Timeout to use when polling the consumer.$$ *($$Duration$$, default: `$$$$`)* $$type$$:: $$Listener type.$$ *($$Type$$, default: `$$single$$`)* //end::configuration-properties[] diff --git a/applications/source/kafka-source/pom.xml b/applications/source/kafka-source/pom.xml index 50ca5b997..4372d8a24 100644 --- a/applications/source/kafka-source/pom.xml +++ b/applications/source/kafka-source/pom.xml @@ -19,7 +19,7 @@ org.springframework.cloud.fn - kafka-supplier + spring-kafka-supplier @@ -58,7 +58,7 @@ org.springframework.cloud.fn - kafka-supplier + spring-kafka-supplier diff --git a/applications/source/load-generator-source/README.adoc b/applications/source/load-generator-source/README.adoc index 1a3fd5722..4ea48e26e 100644 --- a/applications/source/load-generator-source/README.adoc +++ b/applications/source/load-generator-source/README.adoc @@ -8,10 +8,10 @@ A source that sends generated data and dispatches it to the stream. The **$$load-generator$$** $$source$$ has the following options: //tag::configuration-properties[] -$$load-generator.generate-timestamp$$:: $$Whether timestamp generated.$$ *($$Boolean$$, default: `$$$$`)* -$$load-generator.message-count$$:: $$Message count.$$ *($$Integer$$, default: `$$$$`)* -$$load-generator.message-size$$:: $$Message size.$$ *($$Integer$$, default: `$$$$`)* -$$load-generator.producers$$:: $$Number of producers.$$ *($$Integer$$, default: `$$$$`)* +$$load-generator.generate-timestamp$$:: $$Whether timestamp generated.$$ *($$Boolean$$, default: `$$false$$`)* +$$load-generator.message-count$$:: $$Message count.$$ *($$Integer$$, default: `$$1000$$`)* +$$load-generator.message-size$$:: $$Message size.$$ *($$Integer$$, default: `$$1000$$`)* +$$load-generator.producers$$:: $$Number of producers.$$ *($$Integer$$, default: `$$1$$`)* //end::configuration-properties[] //end::ref-doc[] diff --git a/applications/source/mail-source/README.adoc b/applications/source/mail-source/README.adoc index a08737595..d90829cc4 100644 --- a/applications/source/mail-source/README.adoc +++ b/applications/source/mail-source/README.adoc @@ -12,7 +12,7 @@ $$mail.supplier.charset$$:: $$The charset for byte[] mail-to-string transformati $$mail.supplier.delete$$:: $$Set to true to delete email after download.$$ *($$Boolean$$, default: `$$false$$`)* $$mail.supplier.expression$$:: $$Configure a SpEL expression to select messages.$$ *($$String$$, default: `$$true$$`)* $$mail.supplier.idle-imap$$:: $$Set to true to use IdleImap Configuration.$$ *($$Boolean$$, default: `$$false$$`)* -$$mail.supplier.java-mail-properties$$:: $$JavaMail properties as a new line delimited string of name-value pairs, e.g. 'foo=bar\n baz=car'.$$ *($$Properties$$, default: `$$$$`)* +$$mail.supplier.java-mail-properties$$:: $$Java Mail properties as a new line delimited string of name-value pairs, e.g. 'foo=bar\n baz=car'.$$ *($$Properties$$, default: `$$$$`)* $$mail.supplier.mark-as-read$$:: $$Set to true to mark email as read.$$ *($$Boolean$$, default: `$$false$$`)* $$mail.supplier.url$$:: $$Mail connection URL for connection to Mail server e.g. 'imaps://username:password@imap.server.com:993/Inbox'.$$ *($$URLName$$, default: `$$$$`)* $$mail.supplier.user-flag$$:: $$The flag to mark messages when the server does not support \Recent.$$ *($$String$$, default: `$$$$`)* diff --git a/applications/source/mail-source/pom.xml b/applications/source/mail-source/pom.xml index 316d57f06..ee6edb9f4 100644 --- a/applications/source/mail-source/pom.xml +++ b/applications/source/mail-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - mail-supplier + spring-mail-supplier org.springframework.integration @@ -26,19 +26,9 @@ com.icegreen - greenmail-junit5 - 2.0.0-alpha-3 + greenmail + 2.1.0-alpha-4 test - - - com.sun.mail - jakarta.mail - - - jakarta.activation - jakarta.activation-api - - @@ -64,13 +54,13 @@ mail source ${project.version} - org.springframework.cloud.fn.supplier.mail.MailSupplierConfiguration.class - + AUTOCONFIGURATION + mailSupplier org.springframework.cloud.fn - mail-supplier + spring-mail-supplier org.springframework.cloud.stream.app diff --git a/applications/source/mongodb-source/pom.xml b/applications/source/mongodb-source/pom.xml index 573878021..64810a915 100644 --- a/applications/source/mongodb-source/pom.xml +++ b/applications/source/mongodb-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - mongodb-supplier + spring-mongodb-supplier @@ -43,14 +43,13 @@ mongodb source ${project.version} - org.springframework.cloud.fn.supplier.mongo.MongodbSupplierConfiguration.class - - + AUTOCONFIGURATION + mongodbSupplier org.springframework.cloud.fn - mongodb-supplier + spring-mongodb-supplier org.springframework.cloud.stream.app diff --git a/applications/source/mqtt-source/README.adoc b/applications/source/mqtt-source/README.adoc index 14fe33c8d..b0d6b32e3 100644 --- a/applications/source/mqtt-source/README.adoc +++ b/applications/source/mqtt-source/README.adoc @@ -30,11 +30,11 @@ $$username$$:: $$the username to use when connecting to the broker.$$ *($$String === mqtt.supplier -$$binary$$:: $$true to leave the payload as bytes.$$ *($$Boolean$$, default: `$$false$$`)* -$$charset$$:: $$the charset used to convert bytes to String (when binary is false).$$ *($$String$$, default: `$$UTF-8$$`)* -$$client-id$$:: $$identifies the client.$$ *($$String$$, default: `$$stream.client.id.source$$`)* -$$qos$$:: $$the qos; a single value for all topics or a comma-delimited list to match the topics.$$ *($$Integer[]$$, default: `$$[0]$$`)* -$$topics$$:: $$the topic(s) (comma-delimited) to which the source will subscribe.$$ *($$String[]$$, default: `$$[stream.mqtt]$$`)* +$$binary$$:: $$True to leave the payload as bytes.$$ *($$Boolean$$, default: `$$false$$`)* +$$charset$$:: $$The charset used to convert bytes to String (when binary is false).$$ *($$String$$, default: `$$UTF-8$$`)* +$$client-id$$:: $$Identifies the client.$$ *($$String$$, default: `$$stream.client.id.source$$`)* +$$qos$$:: $$The qos; a single value for all topics or a comma-delimited list to match the topics.$$ *($$Integer[]$$, default: `$$[0]$$`)* +$$topics$$:: $$The topic(s) (comma-delimited) to which the source will subscribe.$$ *($$String[]$$, default: `$$[stream.mqtt]$$`)* //end::configuration-properties[] //end::ref-doc[] diff --git a/applications/source/mqtt-source/pom.xml b/applications/source/mqtt-source/pom.xml index 78bc48708..b9171dca7 100644 --- a/applications/source/mqtt-source/pom.xml +++ b/applications/source/mqtt-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - mqtt-supplier + spring-mqtt-supplier org.testcontainers @@ -49,13 +49,13 @@ mqtt source ${project.version} - org.springframework.cloud.fn.supplier.mqtt.MqttSupplierConfiguration.class - + AUTOCONFIGURATION + mqttSupplier org.springframework.cloud.fn - mqtt-supplier + spring-mqtt-supplier org.springframework.cloud.stream.app diff --git a/applications/source/rabbit-source/README.adoc b/applications/source/rabbit-source/README.adoc index 16cf2e66a..409138ded 100644 --- a/applications/source/rabbit-source/README.adoc +++ b/applications/source/rabbit-source/README.adoc @@ -31,7 +31,6 @@ $$initial-retry-interval$$:: $$Initial retry interval when retry is enabled.$$ * $$mapped-request-headers$$:: $$Headers that will be mapped.$$ *($$String[]$$, default: `$$[STANDARD_REQUEST_HEADERS]$$`)* $$max-attempts$$:: $$The maximum delivery attempts when retry is enabled.$$ *($$Integer$$, default: `$$3$$`)* $$max-retry-interval$$:: $$Max retry interval when retry is enabled.$$ *($$Integer$$, default: `$$30000$$`)* -$$own-connection$$:: $$When true, use a separate connection based on the boot properties.$$ *($$Boolean$$, default: `$$false$$`)* $$queues$$:: $$The queues to which the source will listen for messages.$$ *($$String[]$$, default: `$$$$`)* $$requeue$$:: $$Whether rejected messages should be requeued.$$ *($$Boolean$$, default: `$$true$$`)* $$retry-multiplier$$:: $$Retry backoff multiplier when retry is enabled.$$ *($$Double$$, default: `$$2$$`)* @@ -44,6 +43,7 @@ $$addresses$$:: $$Comma-separated list of addresses to which the client should c $$channel-rpc-timeout$$:: $$Continuation timeout for RPC calls in channels. Set it to zero to wait forever.$$ *($$Duration$$, default: `$$10m$$`)* $$connection-timeout$$:: $$Connection timeout. Set it to zero to wait forever.$$ *($$Duration$$, default: `$$$$`)* $$host$$:: $$RabbitMQ host. Ignored if an address is set.$$ *($$String$$, default: `$$localhost$$`)* +$$max-inbound-message-body-size$$:: $$Maximum size of the body of inbound (received) messages.$$ *($$DataSize$$, default: `$$64MB$$`)* $$password$$:: $$Login to authenticate against the broker.$$ *($$String$$, default: `$$guest$$`)* $$port$$:: $$RabbitMQ port. Ignored if an address is set. Default to 5672, or 5671 if SSL is enabled.$$ *($$Integer$$, default: `$$$$`)* $$publisher-confirm-type$$:: $$Type of publisher confirms to use.$$ *($$ConfirmType$$, default: `$$$$`, possible values: `SIMPLE`,`CORRELATED`,`NONE`)* @@ -62,6 +62,7 @@ $$concurrency$$:: $$Minimum number of listener invoker threads.$$ *($$Integer$$, $$consumer-batch-enabled$$:: $$Whether the container creates a batch of messages based on the 'receive-timeout' and 'batch-size'. Coerces 'de-batching-enabled' to true to include the contents of a producer created batch in the batch as discrete records.$$ *($$Boolean$$, default: `$$false$$`)* $$de-batching-enabled$$:: $$Whether the container should present batched messages as discrete messages or call the listener with the batch.$$ *($$Boolean$$, default: `$$true$$`)* $$default-requeue-rejected$$:: $$Whether rejected deliveries are re-queued by default.$$ *($$Boolean$$, default: `$$$$`)* +$$force-stop$$:: $$Whether the container (when stopped) should stop immediately after processing the current message or stop after processing all pre-fetched messages.$$ *($$Boolean$$, default: `$$false$$`)* $$idle-event-interval$$:: $$How often idle container events should be published.$$ *($$Duration$$, default: `$$$$`)* $$max-concurrency$$:: $$Maximum number of listener invoker threads.$$ *($$Integer$$, default: `$$$$`)* $$missing-queues-fatal$$:: $$Whether to fail if the queues declared by the container are not available on the broker and/or whether to stop the container if one or more queues are deleted at runtime.$$ *($$Boolean$$, default: `$$true$$`)* diff --git a/applications/source/rabbit-source/pom.xml b/applications/source/rabbit-source/pom.xml index 7dccd0360..439bf8c86 100644 --- a/applications/source/rabbit-source/pom.xml +++ b/applications/source/rabbit-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - rabbit-supplier + spring-rabbit-supplier org.springframework.boot @@ -65,14 +65,13 @@ rabbit source ${project.version} - org.springframework.cloud.fn.supplier.rabbit.RabbitSupplierConfiguration.class - - + AUTOCONFIGURATION + rabbitSupplier org.springframework.cloud.fn - rabbit-supplier + spring-rabbit-supplier org.springframework.cloud.stream.app diff --git a/applications/source/s3-source/README.adoc b/applications/source/s3-source/README.adoc index 588af9e71..77e63f606 100644 --- a/applications/source/s3-source/README.adoc +++ b/applications/source/s3-source/README.adoc @@ -126,6 +126,7 @@ $$static$$:: $$$$ *($$String$$, default: `$$$$`)* $$accelerate-mode-enabled$$:: $$Option to enable using the accelerate endpoint when accessing S3. Accelerate endpoints allow faster transfer of objects by using Amazon CloudFront's globally distributed edge locations.$$ *($$Boolean$$, default: `$$$$`)* $$checksum-validation-enabled$$:: $$Option to disable doing a validation of the checksum of an object stored in S3.$$ *($$Boolean$$, default: `$$$$`)* $$chunked-encoding-enabled$$:: $$Option to enable using chunked encoding when signing the request payload for {@link software.amazon.awssdk.services.s3.model.PutObjectRequest} and {@link software.amazon.awssdk.services.s3.model.UploadPartRequest}.$$ *($$Boolean$$, default: `$$$$`)* +$$cross-region-enabled$$:: $$Enables cross-region bucket access.$$ *($$Boolean$$, default: `$$$$`)* $$endpoint$$:: $$Overrides the default endpoint.$$ *($$URI$$, default: `$$$$`)* $$path-style-access-enabled$$:: $$Option to enable using path style access for accessing S3 objects instead of DNS style access. DNS style access is preferred as it will result in better load balancing when accessing S3.$$ *($$Boolean$$, default: `$$$$`)* $$region$$:: $$Overrides the default region.$$ *($$String$$, default: `$$$$`)* diff --git a/applications/source/s3-source/pom.xml b/applications/source/s3-source/pom.xml index b19faea4e..fc919056f 100644 --- a/applications/source/s3-source/pom.xml +++ b/applications/source/s3-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - s3-supplier + spring-s3-supplier org.springframework.integration @@ -48,13 +48,13 @@ s3 source ${project.version} - org.springframework.cloud.fn.supplier.s3.AwsS3SupplierConfiguration.class - + AUTOCONFIGURATION + s3Supplier org.springframework.cloud.fn - s3-supplier + spring-s3-supplier org.springframework.cloud.stream.app diff --git a/applications/source/sftp-source/pom.xml b/applications/source/sftp-source/pom.xml index 5b3dc1dba..eee5c1f0e 100644 --- a/applications/source/sftp-source/pom.xml +++ b/applications/source/sftp-source/pom.xml @@ -17,11 +17,11 @@ org.springframework.cloud.fn - sftp-supplier + spring-sftp-supplier org.springframework.cloud.fn - function-test-support + spring-function-test-support test @@ -54,13 +54,13 @@ sftp source ${project.version} - org.springframework.cloud.fn.supplier.sftp.SftpSupplierConfiguration.class - + AUTOCONFIGURATION + sftpSupplier org.springframework.cloud.fn - sftp-supplier + spring-sftp-supplier org.springframework.cloud.stream.app diff --git a/applications/source/syslog-source/README.adoc b/applications/source/syslog-source/README.adoc index 74f6ab2fe..78a690338 100644 --- a/applications/source/syslog-source/README.adoc +++ b/applications/source/syslog-source/README.adoc @@ -7,12 +7,12 @@ The syslog source receives SYSLOG packets over UDP, TCP, or both. RFC3164 (BSD) //tag::configuration-properties[] $$syslog.supplier.buffer-size$$:: $$the buffer size used when decoding messages; larger messages will be rejected.$$ *($$Integer$$, default: `$$2048$$`)* -$$syslog.supplier.nio$$:: $$whether or not to use NIO (when supporting a large number of connections).$$ *($$Boolean$$, default: `$$false$$`)* +$$syslog.supplier.nio$$:: $$Whether to use NIO (when supporting a large number of connections).$$ *($$Boolean$$, default: `$$false$$`)* $$syslog.supplier.port$$:: $$The port to listen on.$$ *($$Integer$$, default: `$$1514$$`)* $$syslog.supplier.protocol$$:: $$Protocol used for SYSLOG (tcp or udp).$$ *($$Protocol$$, default: `$$$$`, possible values: `tcp`,`udp`,`both`)* -$$syslog.supplier.reverse-lookup$$:: $$whether or not to perform a reverse lookup on the incoming socket.$$ *($$Boolean$$, default: `$$false$$`)* -$$syslog.supplier.rfc$$:: $$'5424' or '3164' - the syslog format according to the RFC; 3164 is aka 'BSD' format.$$ *($$String$$, default: `$$3164$$`)* -$$syslog.supplier.socket-timeout$$:: $$the socket timeout.$$ *($$Integer$$, default: `$$0$$`)* +$$syslog.supplier.reverse-lookup$$:: $$Whether to perform a reverse lookup on the incoming socket.$$ *($$Boolean$$, default: `$$false$$`)* +$$syslog.supplier.rfc$$:: $$The '5424' or '3164' - the syslog format according to the RFC; 3164 is aka 'BSD' format.$$ *($$String$$, default: `$$3164$$`)* +$$syslog.supplier.socket-timeout$$:: $$The socket timeout.$$ *($$Integer$$, default: `$$0$$`)* //end::configuration-properties[] //end::ref-doc[] diff --git a/applications/source/syslog-source/pom.xml b/applications/source/syslog-source/pom.xml index 5bdac00e8..b5294542b 100644 --- a/applications/source/syslog-source/pom.xml +++ b/applications/source/syslog-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - syslog-supplier + spring-syslog-supplier @@ -43,13 +43,13 @@ syslog source ${project.version} - org.springframework.cloud.fn.supplier.syslog.SyslogSupplierConfiguration.class - + AUTOCONFIGURATION + syslogSupplier org.springframework.cloud.fn - syslog-supplier + spring-syslog-supplier org.springframework.cloud.stream.app diff --git a/applications/source/tcp-source/README.adoc b/applications/source/tcp-source/README.adoc index 6fb002326..b46b1c64f 100644 --- a/applications/source/tcp-source/README.adoc +++ b/applications/source/tcp-source/README.adoc @@ -15,11 +15,11 @@ Properties grouped by prefix: === tcp -$$nio$$:: $$Whether or not to use NIO.$$ *($$Boolean$$, default: `$$false$$`)* +$$nio$$:: $$Whether to use NIO.$$ *($$Boolean$$, default: `$$false$$`)* $$port$$:: $$The port on which to listen; 0 for the OS to choose a port.$$ *($$Integer$$, default: `$$1234$$`)* $$reverse-lookup$$:: $$Perform a reverse DNS lookup on the remote IP Address; if false, just the IP address is included in the message headers.$$ *($$Boolean$$, default: `$$false$$`)* $$socket-timeout$$:: $$The timeout (ms) before closing the socket when no data is received.$$ *($$Integer$$, default: `$$120000$$`)* -$$use-direct-buffers$$:: $$Whether or not to use direct buffers.$$ *($$Boolean$$, default: `$$false$$`)* +$$use-direct-buffers$$:: $$Whether to use direct buffers.$$ *($$Boolean$$, default: `$$false$$`)* === tcp.supplier diff --git a/applications/source/tcp-source/pom.xml b/applications/source/tcp-source/pom.xml index ea2bc8b79..187af1c41 100644 --- a/applications/source/tcp-source/pom.xml +++ b/applications/source/tcp-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - tcp-supplier + spring-tcp-supplier @@ -43,13 +43,13 @@ tcp source ${project.version} - org.springframework.cloud.fn.supplier.tcp.TcpSupplierConfiguration.class - + AUTOCONFIGURATION + tcpSupplier org.springframework.cloud.fn - tcp-supplier + spring-tcp-supplier org.springframework.cloud.stream.app diff --git a/applications/source/time-source/pom.xml b/applications/source/time-source/pom.xml index 47123a874..6b1f91c7f 100644 --- a/applications/source/time-source/pom.xml +++ b/applications/source/time-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - time-supplier + spring-time-supplier org.springframework.cloud.stream.app @@ -49,13 +49,14 @@ time source ${project.version} - org.springframework.cloud.fn.supplier.time.TimeSupplierConfiguration.class + AUTOCONFIGURATION + timeSupplier org.springframework.cloud.fn - time-supplier - ${java-functions.version} + spring-time-supplier + org.springframework.cloud.stream.app diff --git a/applications/source/twitter-message-source/README.adoc b/applications/source/twitter-message-source/README.adoc index c5e734be1..7e8e16b27 100644 --- a/applications/source/twitter-message-source/README.adoc +++ b/applications/source/twitter-message-source/README.adoc @@ -37,6 +37,7 @@ $$raw-json$$:: $$Enable caching the original (raw) JSON objects as returned by t === twitter.message.source $$count$$:: $$Max number of events to be returned. 20 default. 50 max.$$ *($$Integer$$, default: `$$20$$`)* +$$enabled$$:: $$Whether to enable Twitter message receiving.$$ *($$Boolean$$, default: `$$false$$`)* //end::configuration-properties[] //end::ref-doc[] diff --git a/applications/source/twitter-message-source/pom.xml b/applications/source/twitter-message-source/pom.xml index e990e8202..86055bf88 100644 --- a/applications/source/twitter-message-source/pom.xml +++ b/applications/source/twitter-message-source/pom.xml @@ -17,26 +17,18 @@ org.springframework.cloud.fn - twitter-supplier + spring-twitter-supplier org.springframework.cloud.fn - function-test-support + spring-function-test-support test org.mock-server mockserver-netty - ${mockserver.version} test - - org.mock-server - mockserver-client-java - ${mockserver.version} - test - - @@ -61,16 +53,14 @@ twitter-message source ${project.version} - - org.springframework.cloud.fn.supplier.twitter.message.TwitterMessageSupplierConfiguration.class - - twitterMessageSupplier + AUTOCONFIGURATION + twitterMessagesSupplier org.springframework.cloud.fn - twitter-supplier + spring-twitter-supplier diff --git a/applications/source/twitter-message-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/message/TwitterMessageSourceIntegrationTests.java b/applications/source/twitter-message-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/message/TwitterMessageSourceIntegrationTests.java index 66216d7d5..709bc510e 100644 --- a/applications/source/twitter-message-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/message/TwitterMessageSourceIntegrationTests.java +++ b/applications/source/twitter-message-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/message/TwitterMessageSourceIntegrationTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2021 the original author or authors. + * Copyright 2020-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ import java.time.Duration; import java.util.List; -import java.util.concurrent.TimeUnit; import java.util.function.Function; import com.fasterxml.jackson.core.JsonProcessingException; @@ -38,16 +37,13 @@ import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties; import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils; -import org.springframework.cloud.fn.supplier.twitter.message.TwitterMessageSupplierConfiguration; import org.springframework.cloud.fn.supplier.twitter.message.TwitterMessageSupplierProperties; import org.springframework.cloud.stream.binder.test.OutputDestination; import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; import org.springframework.messaging.Message; -import org.springframework.test.util.TestSocketUtils; import static org.assertj.core.api.Assertions.assertThat; import static org.mockserver.matchers.Times.exactly; @@ -57,23 +53,21 @@ /** * @author Christian Tzolov + * @author Artem Bilan */ public class TwitterMessageSourceIntegrationTests { - private static final String MOCK_SERVER_IP = "127.0.0.1"; - - private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort(); - private static ClientAndServer mockServer; private static MockServerClient mockClient; + private static HttpRequest messageRequest; @BeforeAll public static void startServer() { - mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT); - mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT); + mockServer = ClientAndServer.startClientAndServer(); + mockClient = new MockServerClient("localhost", mockServer.getPort()); messageRequest = setExpectation(request() .withMethod("GET") @@ -93,15 +87,15 @@ public void twitterMessageSourceTests() throws JsonProcessingException { .getCompleteConfiguration(TestTwitterMessageSourceApplication.class)) .web(WebApplicationType.NONE) - .run("--spring.cloud.function.definition=twitterMessageSupplier", + .run("--spring.cloud.function.definition=twitterMessagesSupplier", "--twitter.connection.consumerKey=consumerKey666", "--twitter.connection.consumerSecret=consumerSecret666", "--twitter.connection.accessToken=accessToken666", "--twitter.connection.accessTokenSecret=accessTokenSecret666", - "--twitter.message.source.count=15", - "--spring.cloud.stream.poller.fixed-delay=3000")) { + "--twitter.message.source.enabled=true", + "--twitter.message.source.count=15")) { TwitterConnectionProperties twitterConnectionProperties = context.getBean(TwitterConnectionProperties.class); assertThat(twitterConnectionProperties.getConsumerKey()).isEqualTo("consumerKey666"); @@ -109,19 +103,16 @@ public void twitterMessageSourceTests() throws JsonProcessingException { assertThat(twitterConnectionProperties.getAccessToken()).isEqualTo("accessToken666"); assertThat(twitterConnectionProperties.getAccessTokenSecret()).isEqualTo("accessTokenSecret666"); -// DefaultPollerProperties defaultPollerProperties = context.getBean(DefaultPollerProperties.class); -// assertThat(defaultPollerProperties.getFixedDelay()).isEqualTo(3000); - TwitterMessageSupplierProperties twitterMessageSupplierProperties = context.getBean(TwitterMessageSupplierProperties.class); assertThat(twitterMessageSupplierProperties.getCount()).isEqualTo(15); OutputDestination outputDestination = context.getBean(OutputDestination.class); // Using local region here - Message message = outputDestination.receive(Duration.ofSeconds(300).toMillis(), "twitterMessageSupplier-out-0"); + Message message = outputDestination.receive(Duration.ofSeconds(10).toMillis(), "twitterMessagesSupplier-out-0"); assertThat(message).isNotNull(); String payload = new String(message.getPayload()); - List tweets = new ObjectMapper().readValue(payload, List.class); + List tweets = new ObjectMapper().readValue(payload, List.class); assertThat(tweets).hasSize(4); mockClient.verify(messageRequest, once()); } @@ -135,14 +126,12 @@ private static HttpRequest setExpectation(HttpRequest request) { .withHeaders( new Header("Content-Type", "application/json; charset=utf-8"), new Header("Cache-Control", "public, max-age=86400")) - .withBody(TwitterTestUtils.asString("classpath:/response/messages.json")) - .withDelay(TimeUnit.SECONDS, 10)); + .withBody(TwitterTestUtils.asString("classpath:/response/messages.json"))); return request; } @SpringBootConfiguration @EnableAutoConfiguration - @Import(TwitterMessageSupplierConfiguration.class) public static class TestTwitterMessageSourceApplication { @Bean @@ -152,11 +141,11 @@ public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionPrope Function mockedConfiguration = toConfigurationBuilder.andThen( - new TwitterTestUtils().mockTwitterUrls( - String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT))); + new TwitterTestUtils().mockTwitterUrls("http://localhost:" + mockServer.getPort())); return mockedConfiguration.apply(properties).build(); } + } } diff --git a/applications/source/twitter-search-source/README.adoc b/applications/source/twitter-search-source/README.adoc index 306de4d2e..42a8443bb 100644 --- a/applications/source/twitter-search-source/README.adoc +++ b/applications/source/twitter-search-source/README.adoc @@ -40,11 +40,12 @@ $$raw-json$$:: $$Enable caching the original (raw) JSON objects as returned by t === twitter.search $$count$$:: $$Number of tweets to return per page (e.g. per single request), up to a max of 100.$$ *($$Integer$$, default: `$$100$$`)* -$$lang$$:: $$Restricts searched tweets to the given language, given by an http://en.wikipedia.org/wiki/ISO_639-1 .$$ *($$String$$, default: `$$$$`)* +$$enabled$$:: $$Whether to enable Twitter search supplier.$$ *($$Boolean$$, default: `$$false$$`)* +$$lang$$:: $$Restricts searched tweets to the given language, given by an http://en.wikipedia.org/wiki/ISO_639-1.$$ *($$String$$, default: `$$$$`)* $$page$$:: $$Number of pages (e.g. requests) to search backwards (from most recent to the oldest tweets) before start the search from the most recent tweets again. The total amount of tweets searched backwards is (page * count)$$ *($$Integer$$, default: `$$3$$`)* $$query$$:: $$Search tweets by search query string.$$ *($$String$$, default: `$$$$`)* $$restart-from-most-recent-on-empty-response$$:: $$Restart search from the most recent tweets on empty response. Applied only after the first restart (e.g. when since_id != UNBOUNDED)$$ *($$Boolean$$, default: `$$false$$`)* -$$result-type$$:: $$Specifies what type of search results you would prefer to receive. The current default is "mixed." Valid values include: mixed : Include both popular and real time results in the response. recent : return only the most recent results in the response popular : return only the most popular results in the response$$ *($$ResultType$$, default: `$$$$`, possible values: `popular`,`mixed`,`recent`)* +$$result-type$$:: $$Specifies what type of search results you would prefer to receive. The current default is "mixed." Valid values include: mixed : Include both popular and real time results in the response. recent : return only the most recent results in the response popular : return only the most popular results in the response$$ *($$ResultType$$, default: `$$$$`, possible values: `popular`,`mixed`,`recent`)* $$since$$:: $$If specified, returns tweets with since the given date. Date should be formatted as YYYY-MM-DD.$$ *($$String$$, default: `$$$$`)* === twitter.search.geocode diff --git a/applications/source/twitter-search-source/pom.xml b/applications/source/twitter-search-source/pom.xml index 72912f936..dfc5dd783 100644 --- a/applications/source/twitter-search-source/pom.xml +++ b/applications/source/twitter-search-source/pom.xml @@ -17,23 +17,16 @@ org.springframework.cloud.fn - twitter-supplier + spring-twitter-supplier org.springframework.cloud.fn - function-test-support + spring-function-test-support test org.mock-server mockserver-netty - ${mockserver.version} - test - - - org.mock-server - mockserver-client-java - ${mockserver.version} test @@ -60,14 +53,14 @@ twitter-search source ${project.version} - org.springframework.cloud.fn.supplier.twitter.status.search.TwitterSearchSupplierConfiguration.class + AUTOCONFIGURATION twitterSearchSupplier org.springframework.cloud.fn - twitter-supplier + spring-twitter-supplier diff --git a/applications/source/twitter-search-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/search/TwitterSearchSourceIntegrationTests.java b/applications/source/twitter-search-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/search/TwitterSearchSourceIntegrationTests.java index 2ade8a0b9..ea5ed0a89 100644 --- a/applications/source/twitter-search-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/search/TwitterSearchSourceIntegrationTests.java +++ b/applications/source/twitter-search-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/search/TwitterSearchSourceIntegrationTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2022 the original author or authors. + * Copyright 2020-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ import java.time.Duration; import java.util.List; -import java.util.concurrent.TimeUnit; import java.util.function.Function; import com.fasterxml.jackson.core.JsonProcessingException; @@ -38,16 +37,13 @@ import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties; import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils; -import org.springframework.cloud.fn.supplier.twitter.status.search.TwitterSearchSupplierConfiguration; import org.springframework.cloud.fn.supplier.twitter.status.search.TwitterSearchSupplierProperties; import org.springframework.cloud.stream.binder.test.OutputDestination; import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; import org.springframework.messaging.Message; -import org.springframework.test.util.TestSocketUtils; import static org.assertj.core.api.Assertions.assertThat; import static org.mockserver.matchers.Times.exactly; @@ -57,13 +53,10 @@ /** * @author Christian Tzolov + * @author Artem Bilan */ public class TwitterSearchSourceIntegrationTests { - private static final String MOCK_SERVER_IP = "127.0.0.1"; - - private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort(); - private static ClientAndServer mockServer; private static MockServerClient mockClient; @@ -74,9 +67,8 @@ public class TwitterSearchSourceIntegrationTests { @BeforeAll public static void startServer() { - - mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT); - mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT); + mockServer = ClientAndServer.startClientAndServer(); + mockClient = new MockServerClient("localhost", mockServer.getPort()); searchVratsaRequest = setExpectation(request() .withMethod("GET") @@ -113,6 +105,7 @@ public void twitterSearchTests() throws JsonProcessingException { "--twitter.connection.accessToken=accessToken666", "--twitter.connection.accessTokenSecret=accessTokenSecret666", "--twitter.search.query=Vratsa", + "--twitter.search.enabled=true", "--twitter.search.count=3", "--twitter.search.page=3")) { @@ -122,7 +115,7 @@ public void twitterSearchTests() throws JsonProcessingException { assertThat(message).isNotNull(); String payload = new String(message.getPayload()); - List tweets = new ObjectMapper().readValue(payload, List.class); + List tweets = new ObjectMapper().readValue(payload, List.class); assertThat(tweets).hasSize(3); mockClient.verify(searchVratsaRequest, once()); @@ -144,12 +137,12 @@ public void twitterSearchTestsAmsterdam() throws JsonProcessingException { "--twitter.search.count=3", "--twitter.search.page=3", "--twitter.search.lang=en", + "--twitter.search.enabled=true", "--twitter.search.geocode.latitude=52.1", "--twitter.search.geocode.longitude=4.8", "--twitter.search.geocode.radius=10", "--twitter.search.since=2018-01-01", - "--twitter.search.resultType=popular", - "--spring.cloud.stream.poller.fixed-delay=10000")) { + "--twitter.search.resultType=popular")) { TwitterConnectionProperties twitterConnectionProperties = context.getBean(TwitterConnectionProperties.class); assertThat(twitterConnectionProperties.getConsumerKey()).isEqualTo("consumerKey666"); @@ -157,9 +150,6 @@ public void twitterSearchTestsAmsterdam() throws JsonProcessingException { assertThat(twitterConnectionProperties.getAccessToken()).isEqualTo("accessToken666"); assertThat(twitterConnectionProperties.getAccessTokenSecret()).isEqualTo("accessTokenSecret666"); -// DefaultPollerProperties defaultPollerProperties = context.getBean(DefaultPollerProperties.class); -// assertThat(defaultPollerProperties.getFixedDelay()).isEqualTo(10000); - TwitterSearchSupplierProperties searchSupplierProperties = context.getBean(TwitterSearchSupplierProperties.class); @@ -178,7 +168,7 @@ public void twitterSearchTestsAmsterdam() throws JsonProcessingException { assertThat(message).isNotNull(); String payload = new String(message.getPayload()); - List tweets = new ObjectMapper().readValue(payload, List.class); + List tweets = new ObjectMapper().readValue(payload, List.class); assertThat(tweets).hasSize(3); mockClient.verify(searchAmsterdamRequest, once()); @@ -195,14 +185,12 @@ public static HttpRequest setExpectation(HttpRequest request) { new Header("Content-Type", "application/json; charset=utf-8"), new Header("Cache-Control", "public, max-age=86400")) .withBody(TwitterTestUtils.asString("classpath:/response/search_3.json")) - .withDelay(TimeUnit.SECONDS, 1) ); return request; } @SpringBootConfiguration @EnableAutoConfiguration - @Import(TwitterSearchSupplierConfiguration.class) public static class TestTwitterSearchSourceApplication { @Bean @@ -212,10 +200,11 @@ public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionPrope Function mockedConfiguration = toConfigurationBuilder.andThen( - new TwitterTestUtils().mockTwitterUrls( - String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT))); + new TwitterTestUtils().mockTwitterUrls("http://localhost:" + mockServer.getPort())); return mockedConfiguration.apply(properties).build(); } + } + } diff --git a/applications/source/twitter-stream-source/README.adoc b/applications/source/twitter-stream-source/README.adoc index 6fb96b037..d374f881e 100644 --- a/applications/source/twitter-stream-source/README.adoc +++ b/applications/source/twitter-stream-source/README.adoc @@ -28,18 +28,19 @@ $$consumer-secret$$:: $$Your Twitter secret.$$ *($$String$$, default: `$$$ $$debug-enabled$$:: $$Enables Twitter4J debug mode.$$ *($$Boolean$$, default: `$$false$$`)* $$raw-json$$:: $$Enable caching the original (raw) JSON objects as returned by the Twitter APIs. When set to False the result will use the Twitter4J's json representations. When set to True the result will use the original Twitter APISs json representations.$$ *($$Boolean$$, default: `$$true$$`)* +=== twitter.stream + +$$enabled$$:: $$Whether to enable Twitter streaming supplier.$$ *($$Boolean$$, default: `$$false$$`)* +$$type$$:: $$$$ *($$StreamType$$, default: `$$$$`, possible values: `sample`,`filter`,`firehose`,`link`)* + === twitter.stream.filter $$count$$:: $$Indicates the number of previous statuses to stream before transitioning to the live stream.$$ *($$Integer$$, default: `$$0$$`)* $$filter-level$$:: $$The filter level limits what tweets appear in the stream to those with a minimum filterLevel attribute value. One of either none, low, or medium.$$ *($$FilterLevel$$, default: `$$$$`)* $$follow$$:: $$Specifies the users, by ID, to receive public tweets from.$$ *($$List$$, default: `$$$$`)* $$language$$:: $$Specifies the tweets language of the stream.$$ *($$List$$, default: `$$$$`)* -$$locations$$:: $$Locations to track. Internally represented as 2D array. Bounding box is invalid: 52.38, 4.90, 51.51, -0.12. The first pair must be the SW corner of the box$$ *($$List$$, default: `$$$$`)* +$$locations$$:: $$Locations to track. Internally represented as 2D array. Bounding box is invalid: 52.38, 4.90, 51.51, -0.12. The first pair must be the SW corner of the box$$ *($$List$$, default: `$$$$`)* $$track$$:: $$Specifies keywords to track.$$ *($$List$$, default: `$$$$`)* - -=== twitter.stream - -$$type$$:: $$$$ *($$StreamType$$, default: `$$$$`, possible values: `sample`,`filter`,`firehose`,`link`)* //end::configuration-properties[] //end::ref-doc[] diff --git a/applications/source/twitter-stream-source/pom.xml b/applications/source/twitter-stream-source/pom.xml index c9fffbf1c..efd790bb7 100644 --- a/applications/source/twitter-stream-source/pom.xml +++ b/applications/source/twitter-stream-source/pom.xml @@ -17,23 +17,16 @@ org.springframework.cloud.fn - twitter-supplier + spring-twitter-supplier org.springframework.cloud.fn - function-test-support + spring-function-test-support test org.mock-server mockserver-netty - ${mockserver.version} - test - - - org.mock-server - mockserver-client-java - ${mockserver.version} test @@ -60,14 +53,14 @@ twitter-stream source ${project.version} - org.springframework.cloud.fn.supplier.twitter.status.stream.TwitterStreamSupplierConfiguration.class + AUTOCONFIGURATION twitterStreamSupplier org.springframework.cloud.fn - twitter-supplier + spring-twitter-supplier diff --git a/applications/source/twitter-stream-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/stream/TwitterStreamSourceTests.java b/applications/source/twitter-stream-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/stream/TwitterStreamSourceTests.java index 3fea18778..4450ae030 100644 --- a/applications/source/twitter-stream-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/stream/TwitterStreamSourceTests.java +++ b/applications/source/twitter-stream-source/src/test/java/org/springframework/cloud/stream/app/source/twitter/stream/TwitterStreamSourceTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2020-2020 the original author or authors. + * Copyright 2020-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,12 +16,11 @@ package org.springframework.cloud.stream.app.source.twitter.stream; -import java.util.concurrent.TimeUnit; +import java.time.Duration; import java.util.function.Function; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.mockserver.client.MockServerClient; import org.mockserver.integration.ClientAndServer; @@ -36,14 +35,13 @@ import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties; import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils; -import org.springframework.cloud.fn.supplier.twitter.status.stream.TwitterStreamSupplierConfiguration; import org.springframework.cloud.fn.supplier.twitter.status.stream.TwitterStreamSupplierProperties; +import org.springframework.cloud.stream.binder.test.OutputDestination; import org.springframework.cloud.stream.binder.test.TestChannelBinderConfiguration; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; -import org.springframework.test.util.TestSocketUtils; +import org.springframework.messaging.Message; import static org.assertj.core.api.Assertions.assertThat; import static org.mockserver.matchers.Times.exactly; @@ -53,35 +51,33 @@ public class TwitterStreamSourceTests { - private static final String MOCK_SERVER_IP = "127.0.0.1"; - - private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort(); - private static ClientAndServer mockServer; private static MockServerClient mockClient; + private static HttpRequest streamFilterRequest; + private static HttpRequest streamSampleRequest; + private static HttpRequest streamFirehoseRequest; - private static HttpRequest streamLinknsRequest; + + private static HttpRequest streamLinksRequest; @BeforeAll public static void startServer() { - - mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT); - - mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT); + mockServer = ClientAndServer.startClientAndServer(); + mockClient = new MockServerClient("localhost", mockServer.getPort()); streamFilterRequest = mockClientRecordRequest(request() .withMethod("POST") .withPath("/stream/statuses/filter.json") - .withBody(new StringBody("count=0&track=Java%2CPython&stall_warnings=true"))); + .withBody(new StringBody("count=3&track=Java%2CPython&stall_warnings=true"))); streamSampleRequest = mockClientRecordRequest(request() .withMethod("GET") .withPath("/stream/statuses/sample.json")); - streamLinknsRequest = mockClientRecordRequest(request() + streamLinksRequest = mockClientRecordRequest(request() .withMethod("GET") .withPath("/stream/statuses/links.json")); //.withBody(new StringBody("count=0&stall_warnings=true"))); @@ -98,7 +94,6 @@ public static void stopServer() { } @Test - @Disabled public void testSourceFromSupplier() { try (ConfigurableApplicationContext context = new SpringApplicationBuilder( TestChannelBinderConfiguration.getCompleteConfiguration(TestTwitterStreamSourceApplication.class)) @@ -110,6 +105,7 @@ public void testSourceFromSupplier() { "--twitter.connection.accessToken=accessToken666", "--twitter.connection.accessTokenSecret=accessTokenSecret666", + "--twitter.stream.enabled=true", "--twitter.stream.type=filter", "--twitter.stream.filter.track=Java,Python", "--twitter.stream.filter.count=3")) { @@ -125,9 +121,9 @@ public void testSourceFromSupplier() { assertThat(twitterStreamSupplierProperties.getType()).isEqualTo(TwitterStreamSupplierProperties.StreamType.filter); assertThat(twitterStreamSupplierProperties.getFilter().getTrack()).contains("Java", "Python"); - //OutputDestination target = context.getBean(OutputDestination.class); - //Message sourceMessage = target.receive(10000); - //final String actual = new String(sourceMessage.getPayload()); + OutputDestination outputDestination = context.getBean(OutputDestination.class); + Message message = outputDestination.receive(Duration.ofSeconds(300).toMillis(), "twitterStreamSupplier-out-0"); + assertThat(message).isNotNull(); mockClient.verify(streamFilterRequest, once()); } @@ -140,14 +136,12 @@ private static HttpRequest mockClientRecordRequest(HttpRequest request) { .withHeaders( new Header("Content-Type", "application/json; charset=utf-8"), new Header("Cache-Control", "public, max-age=86400")) - .withBody(TwitterTestUtils.asString("classpath:/response/stream_test_1.json")) - .withDelay(TimeUnit.SECONDS, 10)); + .withBody(TwitterTestUtils.asString("classpath:/response/stream_test_1.json"))); return request; } @SpringBootConfiguration @EnableAutoConfiguration - @Import(TwitterStreamSupplierConfiguration.class) public static class TestTwitterStreamSourceApplication { @Bean @@ -157,10 +151,11 @@ public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionPrope Function mockedConfiguration = toConfigurationBuilder.andThen( - new TwitterTestUtils().mockTwitterUrls( - String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT))); + new TwitterTestUtils().mockTwitterUrls("http://localhost:" + mockServer.getPort())); return mockedConfiguration.apply(properties).build(); } + } + } diff --git a/applications/source/websocket-source/pom.xml b/applications/source/websocket-source/pom.xml index 715835a9d..9c2985921 100644 --- a/applications/source/websocket-source/pom.xml +++ b/applications/source/websocket-source/pom.xml @@ -17,7 +17,7 @@ org.springframework.cloud.fn - websocket-supplier + spring-websocket-supplier org.springframework.boot @@ -53,13 +53,13 @@ websocket source ${project.version} - org.springframework.cloud.fn.supplier.websocket.WebsocketSupplierConfiguration.class - + AUTOCONFIGURATION + websocketSupplier org.springframework.cloud.fn - websocket-supplier + spring-websocket-supplier org.springframework.cloud.stream.app diff --git a/applications/source/xmpp-source/README.adoc b/applications/source/xmpp-source/README.adoc index 6048f9bd9..2c2f47427 100644 --- a/applications/source/xmpp-source/README.adoc +++ b/applications/source/xmpp-source/README.adoc @@ -25,8 +25,8 @@ Properties grouped by prefix: $$host$$:: $$XMPP Host server to connect to.$$ *($$String$$, default: `$$$$`)* $$password$$:: $$The Password for the connected user.$$ *($$String$$, default: `$$$$`)* -$$port$$:: $$Port for connecting to the host. - Default Client Port: 5222$$ *($$Integer$$, default: `$$5222$$`)* -$$resource$$:: $$The Resource to bind to on the XMPP Host. - Can be empty, server will generate one if not set$$ *($$String$$, default: `$$$$`)* +$$port$$:: $$Port for connecting to the host. - Default Client Port: 5222$$ *($$Integer$$, default: `$$5222$$`)* +$$resource$$:: $$The Resource to bind to on the XMPP Host. - Can be empty, server will generate one if not set$$ *($$String$$, default: `$$$$`)* $$security-mode$$:: $$$$ *($$SecurityMode$$, default: `$$$$`, possible values: `required`,`ifpossible`,`disabled`)* $$service-name$$:: $$The Service Name to set for the XMPP Domain.$$ *($$String$$, default: `$$$$`)* $$subscription-mode$$:: $$$$ *($$SubscriptionMode$$, default: `$$$$`, possible values: `accept_all`,`reject_all`,`manual`)* diff --git a/applications/source/xmpp-source/pom.xml b/applications/source/xmpp-source/pom.xml index 9d634eec0..d407932c1 100644 --- a/applications/source/xmpp-source/pom.xml +++ b/applications/source/xmpp-source/pom.xml @@ -20,7 +20,7 @@ org.springframework.cloud.fn - xmpp-supplier + spring-xmpp-supplier @@ -37,7 +37,7 @@ org.springframework.cloud.fn - function-test-support + spring-function-test-support ${project.version} test @@ -66,14 +66,13 @@ xmpp source ${project.version} - org.springframework.cloud.fn.supplier.xmpp.XmppSupplierConfiguration.class - - + AUTOCONFIGURATION + xmppSupplier org.springframework.cloud.fn - xmpp-supplier + spring-xmpp-supplier org.springframework.cloud.stream.app diff --git a/applications/source/zeromq-source/pom.xml b/applications/source/zeromq-source/pom.xml index 5086f2397..f7cf7b53a 100644 --- a/applications/source/zeromq-source/pom.xml +++ b/applications/source/zeromq-source/pom.xml @@ -20,7 +20,7 @@ org.springframework.cloud.fn - zeromq-supplier + spring-zeromq-supplier @@ -54,14 +54,13 @@ zeromq source ${project.version} - org.springframework.cloud.fn.supplier.zeromq.ZeroMqSupplierConfiguration.class - - + AUTOCONFIGURATION + zeromqSupplier org.springframework.cloud.fn - zeromq-supplier + spring-zeromq-supplier org.springframework.cloud.stream.app diff --git a/applications/stream-applications-core/pom.xml b/applications/stream-applications-core/pom.xml index 1e668e665..b3a208299 100644 --- a/applications/stream-applications-core/pom.xml +++ b/applications/stream-applications-core/pom.xml @@ -38,7 +38,7 @@ org.springframework.cloud.fn - function-dependencies + spring-functions-catalog-bom ${java-functions.version} pom import @@ -60,11 +60,6 @@ mockserver-netty ${mockserver.version} - - org.mock-server - mockserver-client-java - ${mockserver.version} - org.junit junit-bom @@ -193,7 +188,7 @@ org.springframework.cloud.fn - function-dependencies + spring-function-dependencies ${java-functions.version} diff --git a/applications/stream-applications-core/stream-applications-composite-function-support/pom.xml b/applications/stream-applications-core/stream-applications-composite-function-support/pom.xml index 85b414de0..8246bf8f8 100644 --- a/applications/stream-applications-core/stream-applications-composite-function-support/pom.xml +++ b/applications/stream-applications-core/stream-applications-composite-function-support/pom.xml @@ -34,28 +34,23 @@ org.springframework.cloud.fn - filter-function - ${java-functions.version} + spring-filter-function org.springframework.cloud.fn - spel-function - ${java-functions.version} + spring-spel-function org.springframework.cloud.fn - header-enricher-function - ${java-functions.version} + spring-header-enricher-function org.springframework.cloud.fn - payload-converter-function - ${java-functions.version} + spring-payload-converter-function org.springframework.cloud.fn - task-launch-request-function - ${java-functions.version} + spring-task-launch-request-function diff --git a/applications/stream-applications-core/stream-applications-security-common/pom.xml b/applications/stream-applications-core/stream-applications-security-common/pom.xml index ea5a09df8..0c391ea17 100644 --- a/applications/stream-applications-core/stream-applications-security-common/pom.xml +++ b/applications/stream-applications-core/stream-applications-security-common/pom.xml @@ -49,11 +49,6 @@ org.springframework.boot spring-boot-starter-security - - org.springframework.cloud - spring-cloud-stream-test-binder - test - diff --git a/applications/stream-applications-core/stream-applications-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebFluxSecurityAutoConfiguration.java b/applications/stream-applications-core/stream-applications-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebFluxSecurityAutoConfiguration.java index adf3eda69..29cf1c930 100644 --- a/applications/stream-applications-core/stream-applications-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebFluxSecurityAutoConfiguration.java +++ b/applications/stream-applications-core/stream-applications-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebFluxSecurityAutoConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 the original author or authors. + * Copyright 2019-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,16 +29,18 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication; import org.springframework.boot.autoconfigure.security.SecurityProperties; import org.springframework.boot.autoconfigure.security.reactive.ReactiveSecurityAutoConfiguration; +import org.springframework.boot.autoconfigure.security.reactive.ReactiveUserDetailsServiceAutoConfiguration; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; import org.springframework.http.HttpMethod; +import org.springframework.security.config.Customizer; import org.springframework.security.config.annotation.web.reactive.EnableWebFluxSecurity; import org.springframework.security.config.web.server.ServerHttpSecurity; import org.springframework.security.core.userdetails.MapReactiveUserDetailsService; +import org.springframework.security.core.userdetails.ReactiveUserDetailsService; import org.springframework.security.core.userdetails.User; import org.springframework.security.core.userdetails.UserDetails; -import org.springframework.security.core.userdetails.UserDetailsService; import org.springframework.security.crypto.factory.PasswordEncoderFactories; import org.springframework.security.web.server.SecurityWebFilterChain; import org.springframework.security.web.server.WebFilterChainProxy; @@ -54,11 +56,12 @@ */ @Conditional(OnHttpCsrfOrSecurityDisabled.class) @AutoConfiguration -@ConditionalOnClass({ Flux.class, EnableWebFluxSecurity.class, WebFilterChainProxy.class, WebFluxConfigurer.class }) +@ConditionalOnClass({Flux.class, EnableWebFluxSecurity.class, WebFilterChainProxy.class, WebFluxConfigurer.class}) @ConditionalOnMissingBean(WebFilterChainProxy.class) @ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.REACTIVE) -@AutoConfigureBefore({ ReactiveManagementWebSecurityAutoConfiguration.class, - ReactiveSecurityAutoConfiguration.class }) +@AutoConfigureBefore({ReactiveManagementWebSecurityAutoConfiguration.class, + ReactiveSecurityAutoConfiguration.class, + ReactiveUserDetailsServiceAutoConfiguration.class}) @EnableConfigurationProperties(AppStarterWebSecurityAutoConfigurationProperties.class) public class AppStarterWebFluxSecurityAutoConfiguration { @@ -66,22 +69,22 @@ public class AppStarterWebFluxSecurityAutoConfiguration { private PathMappedEndpoints pathMappedEndpoints; @Bean - @ConditionalOnMissingBean(UserDetailsService.class) + @ConditionalOnMissingBean(ReactiveUserDetailsService.class) public MapReactiveUserDetailsService userDetailsService(SecurityProperties securityProperties, - AppStarterWebSecurityAutoConfigurationProperties streamAppsecurityProperties) { + AppStarterWebSecurityAutoConfigurationProperties streamAppSecurityProperties) { UserDetails primaryUser = User.builder() .username(securityProperties.getUser().getName()) .password(securityProperties.getUser().getPassword()) .passwordEncoder(PasswordEncoderFactories.createDelegatingPasswordEncoder()::encode) .roles(securityProperties.getUser().getRoles() - .toArray(new String[securityProperties.getUser().getRoles().size()])) + .toArray(new String[0])) .build(); - if (StringUtils.hasText(streamAppsecurityProperties.getAdminPassword()) && - StringUtils.hasText(streamAppsecurityProperties.getAdminUser())) { + if (StringUtils.hasText(streamAppSecurityProperties.getAdminPassword()) && + StringUtils.hasText(streamAppSecurityProperties.getAdminUser())) { UserDetails user = User.builder() - .username(streamAppsecurityProperties.getAdminUser()) - .password(streamAppsecurityProperties.getAdminPassword()) + .username(streamAppSecurityProperties.getAdminUser()) + .password(streamAppSecurityProperties.getAdminPassword()) .roles("ADMIN") .passwordEncoder(PasswordEncoderFactories.createDelegatingPasswordEncoder()::encode) .build(); @@ -96,37 +99,36 @@ public SecurityWebFilterChain springSecurityFilterChain(ServerHttpSecurity http, String managementPath = pathMappedEndpoints.getBasePath() + "/**"; if (!securityProperties.isCsrfEnabled()) { - http.csrf().disable(); + http.csrf(ServerHttpSecurity.CsrfSpec::disable); } else { /* * See https://stackoverflow.com/questions/51079564/spring-security-antmatchers-not-being- * applied-on-post-requests-and-only-works-wi/51088555 */ - http.csrf().requireCsrfProtectionMatcher(exchange -> new AntPathMatcher() - .match(managementPath, - exchange.getRequest().getPath().value()) + http.csrf((csrfSpec) -> + csrfSpec.requireCsrfProtectionMatcher(exchange -> + new AntPathMatcher().match(managementPath, exchange.getRequest().getPath().value()) ? ServerWebExchangeMatcher.MatchResult.notMatch() - : ServerWebExchangeMatcher.MatchResult.match()); + : ServerWebExchangeMatcher.MatchResult.match())); } if (!securityProperties.isEnabled()) { - http.authorizeExchange() - .anyExchange() - .permitAll(); + http.authorizeExchange((authorizeExchangeSpec) -> authorizeExchangeSpec.anyExchange().permitAll()); } else { - http.authorizeExchange() - .pathMatchers(HttpMethod.POST, managementPath).hasRole("ADMIN") - .pathMatchers( - pathMappedEndpoints.getBasePath(), - pathMappedEndpoints.getPath(EndpointId.of("health")), - pathMappedEndpoints.getPath(EndpointId.of("info")), - pathMappedEndpoints.getPath(EndpointId.of("bindings"))) - .permitAll().anyExchange().authenticated(); - http.httpBasic(); - http.formLogin(); + http.authorizeExchange((authorizeExchangeSpec) -> authorizeExchangeSpec + .pathMatchers(HttpMethod.POST, managementPath).hasRole("ADMIN") + .pathMatchers( + pathMappedEndpoints.getBasePath(), + pathMappedEndpoints.getPath(EndpointId.of("health")), + pathMappedEndpoints.getPath(EndpointId.of("info")), + pathMappedEndpoints.getPath(EndpointId.of("bindings"))) + .permitAll().anyExchange().authenticated()) + .formLogin(Customizer.withDefaults()) + .httpBasic(Customizer.withDefaults()); } return http.build(); } + } diff --git a/applications/stream-applications-core/stream-applications-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebSecurityAutoConfiguration.java b/applications/stream-applications-core/stream-applications-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebSecurityAutoConfiguration.java index ba8ea6e23..76dd3c4f0 100644 --- a/applications/stream-applications-core/stream-applications-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebSecurityAutoConfiguration.java +++ b/applications/stream-applications-core/stream-applications-security-common/src/main/java/org/springframework/cloud/stream/app/security/common/AppStarterWebSecurityAutoConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright 2018-2022 the original author or authors. + * Copyright 2018-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,11 +18,8 @@ import java.util.Collections; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.actuate.autoconfigure.endpoint.web.WebEndpointProperties; import org.springframework.boot.actuate.autoconfigure.security.servlet.EndpointRequest; import org.springframework.boot.actuate.autoconfigure.security.servlet.ManagementWebSecurityAutoConfiguration; -import org.springframework.boot.actuate.endpoint.annotation.Endpoint; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.AutoConfigureBefore; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; @@ -35,8 +32,10 @@ import org.springframework.http.HttpMethod; import org.springframework.security.authentication.AuthenticationProvider; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.config.Customizer; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.config.annotation.web.configurers.AbstractHttpConfigurer; import org.springframework.security.core.Authentication; import org.springframework.security.core.AuthenticationException; import org.springframework.security.core.authority.SimpleGrantedAuthority; @@ -56,14 +55,11 @@ @ConditionalOnClass(SecurityFilterChain.class) @ConditionalOnMissingBean(SecurityFilterChain.class) @ConditionalOnWebApplication(type = ConditionalOnWebApplication.Type.SERVLET) -@AutoConfigureBefore({ ManagementWebSecurityAutoConfiguration.class, SecurityAutoConfiguration.class }) +@AutoConfigureBefore({ManagementWebSecurityAutoConfiguration.class, SecurityAutoConfiguration.class}) @EnableConfigurationProperties(AppStarterWebSecurityAutoConfigurationProperties.class) @EnableWebSecurity public class AppStarterWebSecurityAutoConfiguration { - @Autowired - private WebEndpointProperties webEndpointProperties; - /* * Spring security will discover this provider. It grants the ADMIN ROLE to the admin * user, if configured, otherwise it passes through the current authentication. @@ -83,7 +79,7 @@ public Authentication authenticate(Authentication authentication) throws Authent return new UsernamePasswordAuthenticationToken(securityProperties.getAdminUser(), securityProperties.getAdminPassword(), Collections.singletonList( - new SimpleGrantedAuthority("ROLE_ADMIN"))); + new SimpleGrantedAuthority("ROLE_ADMIN"))); } return authentication; } @@ -100,25 +96,30 @@ SecurityFilterChain appStarterWebSecurityFilterChain(HttpSecurity http, AppStarterWebSecurityAutoConfigurationProperties securityProperties) throws Exception { if (!securityProperties.isCsrfEnabled()) { - http.csrf().disable(); + http.csrf(AbstractHttpConfigurer::disable); } else { /* * See https://stackoverflow.com/questions/51079564/spring-security-antmatchers-not-being- * applied-on-post-requests-and-only-works-wi/51088555 */ - http.csrf().ignoringRequestMatchers(MethodAwareEndpointRequest.toAnyEndpoint(HttpMethod.POST)); + http.csrf((csrfConfigurer) -> + csrfConfigurer.ignoringRequestMatchers(MethodAwareEndpointRequest.toAnyEndpoint(HttpMethod.POST))); } if (securityProperties.isEnabled()) { - http.authorizeHttpRequests() - .requestMatchers(MethodAwareEndpointRequest.toAnyEndpoint(HttpMethod.POST)).hasRole("ADMIN") - .requestMatchers(EndpointRequest.toLinks()).permitAll() - .requestMatchers(EndpointRequest.to("health", "info", "bindings")).permitAll() - .requestMatchers(EndpointRequest.toAnyEndpoint()).authenticated() - .and().formLogin().and().httpBasic(); + http.authorizeHttpRequests((authorizeHttpRequests) -> + authorizeHttpRequests + .requestMatchers(MethodAwareEndpointRequest.toAnyEndpoint(HttpMethod.POST)).hasRole("ADMIN") + .requestMatchers(EndpointRequest.toLinks()).permitAll() + .requestMatchers(EndpointRequest.to("health", "info", "bindings")).permitAll() + .requestMatchers(EndpointRequest.toAnyEndpoint()).authenticated() + ) + .formLogin(Customizer.withDefaults()) + .httpBasic(Customizer.withDefaults()); } else { - http.authorizeHttpRequests().anyRequest().permitAll(); + http.authorizeHttpRequests((authorizeHttpRequests) -> + authorizeHttpRequests.anyRequest().permitAll()); } return http.build(); } @@ -128,25 +129,6 @@ SecurityFilterChain appStarterWebSecurityFilterChain(HttpSecurity http, */ static class MethodAwareEndpointRequest { - /** - * Returns a matcher that includes the specified {@link Endpoint actuator endpoints} and http method. - * For example: 
-		 * EndpointRequest.to(HttpMethod.POST, "loggers")
-		 *
- * @param httpMethod the http method to include - * @param endpoints the endpoints to include - * @return the configured {@link RequestMatcher} - */ - static RequestMatcher to(HttpMethod httpMethod, String... endpoints) { - final EndpointRequest.EndpointRequestMatcher matcher = EndpointRequest.to(endpoints); - return (request) -> { - if (!httpMethod.toString().equals(request.getMethod())) { - return false; - } - return matcher.matches(request); - }; - } - static RequestMatcher toAnyEndpoint(HttpMethod httpMethod) { final EndpointRequest.EndpointRequestMatcher matcher = EndpointRequest.toAnyEndpoint(); return (request) -> { @@ -156,5 +138,7 @@ static RequestMatcher toAnyEndpoint(HttpMethod httpMethod) { return matcher.matches(request); }; } + } + } diff --git a/applications/stream-applications-core/stream-applications-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityEnabledTests.java b/applications/stream-applications-core/stream-applications-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityEnabledTests.java index 7ac1e7a13..058b3bb6b 100644 --- a/applications/stream-applications-core/stream-applications-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityEnabledTests.java +++ b/applications/stream-applications-core/stream-applications-security-common/src/test/java/org/springframework/cloud/stream/app/security/common/ReactiveSecurityEnabledManagementSecurityEnabledTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2019-2022 the original author or authors. + * Copyright 2019-2024 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -96,7 +96,7 @@ public void testPostBindingsEndpoint() { Collections.singletonMap("state", "STOPPED"), Void.class); assertThat(response.getStatusCode()).isIn(HttpStatus.NO_CONTENT, HttpStatus.OK); String result = this.restTemplate.getForEntity("/actuator/bindings", String.class).getBody(); - assertThat(result.contains("\"state\":\"stopped\"")).isTrue(); + assertThat(result).contains("\"state\":\"stopped\""); } @Test diff --git a/applications/stream-applications-core/stream-applications-test-support/pom.xml b/applications/stream-applications-core/stream-applications-test-support/pom.xml index 0f764f5e4..288f42a3c 100644 --- a/applications/stream-applications-core/stream-applications-test-support/pom.xml +++ b/applications/stream-applications-core/stream-applications-test-support/pom.xml @@ -38,12 +38,10 @@ org.springframework.kafka spring-kafka - ${spring-kafka.version} org.springframework.amqp spring-rabbit - ${spring-rabbit.version} org.springframework @@ -57,19 +55,16 @@ org.testcontainers junit-jupiter - ${testcontainers.version} compile org.testcontainers kafka - ${testcontainers.version} compile org.testcontainers rabbitmq - ${testcontainers.version} compile diff --git a/applications/stream-applications-integration-tests/pom.xml b/applications/stream-applications-integration-tests/pom.xml index 845c32230..427397123 100644 --- a/applications/stream-applications-integration-tests/pom.xml +++ b/applications/stream-applications-integration-tests/pom.xml @@ -70,12 +70,12 @@ org.springframework.cloud.fn - function-test-support + spring-function-test-support test org.springframework.cloud.fn - s3-supplier + spring-s3-supplier com.squareup.okhttp3 diff --git a/build-core.sh b/build-core.sh index d60b7d370..a9bc42ed2 100755 --- a/build-core.sh +++ b/build-core.sh @@ -20,4 +20,4 @@ else MAVEN_GOAL="$*" fi -$SCDIR/build-folder.sh stream-applications-build,functions,applications/stream-applications-core "$MAVEN_GOAL" +$SCDIR/build-folder.sh stream-applications-build,applications/stream-applications-core "$MAVEN_GOAL" diff --git a/functions/common/aws-s3-common/pom.xml b/functions/common/aws-s3-common/pom.xml deleted file mode 100644 index f8262c88b..000000000 --- a/functions/common/aws-s3-common/pom.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - aws-s3-common - aws-s3-common - aws-s3 common - - - - org.springframework.integration - spring-integration-aws - ${spring-integration-aws.version} - - - io.awspring.cloud - spring-cloud-aws-starter-s3 - - - - software.amazon.awssdk - aws-crt-client - - - org.springframework.integration - spring-integration-file - - - - diff --git a/functions/common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3Configuration.java b/functions/common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3Configuration.java deleted file mode 100644 index 6f9f07e53..000000000 --- a/functions/common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3Configuration.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.aws.s3; - -import java.net.URI; - -import io.awspring.cloud.autoconfigure.s3.S3AutoConfiguration; -import io.awspring.cloud.autoconfigure.s3.S3CrtAsyncClientAutoConfiguration; -import io.awspring.cloud.autoconfigure.s3.properties.S3Properties; -import software.amazon.awssdk.services.s3.S3Client; - -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.AutoConfigureAfter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.context.annotation.Bean; -import org.springframework.integration.aws.support.S3SessionFactory; - -/** - * @author Artem Bilan - */ -@AutoConfiguration -@AutoConfigureAfter({S3AutoConfiguration.class, S3CrtAsyncClientAutoConfiguration.class}) -public class AmazonS3Configuration { - - @Bean - @ConditionalOnMissingBean - public S3SessionFactory s3SessionFactory(S3Client amazonS3, S3Properties s3Properties) { - S3SessionFactory s3SessionFactory = new S3SessionFactory(amazonS3); - URI endpoint = s3Properties.getEndpoint(); - if (endpoint != null) { - s3SessionFactory.setEndpoint(String.join(":", endpoint.getHost(), String.valueOf(endpoint.getPort()))); - } - return s3SessionFactory; - } - -} diff --git a/functions/common/aws-s3-common/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/common/aws-s3-common/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index cf831442d..000000000 --- a/functions/common/aws-s3-common/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.common.aws.s3.AmazonS3Configuration diff --git a/functions/common/aws-s3-common/src/test/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3ConfigurationTests.java b/functions/common/aws-s3-common/src/test/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3ConfigurationTests.java deleted file mode 100644 index ab1cf09c8..000000000 --- a/functions/common/aws-s3-common/src/test/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3ConfigurationTests.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2020-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.aws.s3; - -import io.awspring.cloud.autoconfigure.core.AwsAutoConfiguration; -import io.awspring.cloud.autoconfigure.s3.S3AutoConfiguration; -import io.awspring.cloud.autoconfigure.s3.S3CrtAsyncClientAutoConfiguration; -import io.awspring.cloud.core.region.StaticRegionProvider; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.regions.providers.AwsRegionProvider; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3Utilities; -import software.amazon.awssdk.services.s3.model.GetUrlRequest; - -import org.springframework.boot.autoconfigure.AutoConfigurations; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.context.annotation.Bean; -import org.springframework.integration.test.util.TestUtils; - -/** - * @author Timo Salm - * @author Artem Bilan - */ -public class AmazonS3ConfigurationTests { - - private final ApplicationContextRunner runner = new ApplicationContextRunner() - .withConfiguration( - AutoConfigurations.of( - AwsAutoConfiguration.class, - S3AutoConfiguration.class, - S3CrtAsyncClientAutoConfiguration.class, - AmazonS3Configuration.class)) - .withUserConfiguration(TestConfiguration.class); - - private static final String TEST_REGION_NAME = "eu-central-1"; - - @Test - public void testAmazonS3Configuration() { - runner.withPropertyValues().run(context -> { - S3Client amazonS3 = context.getBean(S3Client.class); - Assertions.assertNotNull(amazonS3); - S3Utilities utilities = amazonS3.utilities(); - Assertions.assertEquals(TEST_REGION_NAME, - TestUtils.getPropertyValue(utilities, "region", Region.class).id()); - Assertions.assertTrue( - utilities.getUrl(GetUrlRequest.builder().bucket("b").key("k").build()).toString() - .startsWith("https://s3.eu-central-1.amazonaws.com")); - }); - } - - @Test - public void testAmazonS3ConfigurationForS3CompatibleStorage() { - runner.withPropertyValues( - "spring.cloud.aws.s3.endpoint=http://localhost:8080" - ).run(context -> { - S3Client amazonS3 = context.getBean(S3Client.class); - Assertions.assertNotNull(amazonS3); - S3Utilities utilities = amazonS3.utilities(); - Assertions.assertTrue(utilities.getUrl(GetUrlRequest.builder().bucket("b").key("k").build()).toString() - .startsWith("http://localhost:8080")); - }); - } - - private static class TestConfiguration { - - @Bean - AwsRegionProvider regionProvider() { - return new StaticRegionProvider(TEST_REGION_NAME); - } - - @Bean - AwsCredentialsProvider awsCredentialsProvider() { - return StaticCredentialsProvider.create(AwsBasicCredentials.create("accessKey", "secretKey")); - } - - } - -} diff --git a/functions/common/config-common/pom.xml b/functions/common/config-common/pom.xml deleted file mode 100644 index c25f4640e..000000000 --- a/functions/common/config-common/pom.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.stream.app - stream-applications-build - 5.0.0-SNAPSHOT - ../../../stream-applications-build/pom.xml - - - org.springframework.cloud.fn - config-common - config-common - Function Common Configuration Components - - - - org.springframework.boot - spring-boot-autoconfigure - - - org.springframework.integration - spring-integration-core - - - - diff --git a/functions/common/config-common/src/main/java/org/springframework/cloud/fn/common/config/ComponentCustomizer.java b/functions/common/config-common/src/main/java/org/springframework/cloud/fn/common/config/ComponentCustomizer.java deleted file mode 100644 index 3af872ae5..000000000 --- a/functions/common/config-common/src/main/java/org/springframework/cloud/fn/common/config/ComponentCustomizer.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2022-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.config; - -/** - * The customizer contract to apply to beans in the application context which - * type is matching to generic type of the instance of this interface. - * - * @param the target component (bean) type in the application context to customize. - * - * @author Artem Bilan - * - * @since 1.2.1 - */ -@FunctionalInterface -public interface ComponentCustomizer { - - void customize(T component); - -} diff --git a/functions/common/config-common/src/main/java/org/springframework/cloud/fn/common/config/SpelExpressionConverterConfiguration.java b/functions/common/config-common/src/main/java/org/springframework/cloud/fn/common/config/SpelExpressionConverterConfiguration.java deleted file mode 100644 index a1f73150b..000000000 --- a/functions/common/config-common/src/main/java/org/springframework/cloud/fn/common/config/SpelExpressionConverterConfiguration.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.config; - -import java.beans.Introspector; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.AutoConfigureAfter; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingClass; -import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Lazy; -import org.springframework.core.convert.converter.Converter; -import org.springframework.expression.EvaluationContext; -import org.springframework.expression.Expression; -import org.springframework.expression.ParseException; -import org.springframework.expression.spel.standard.SpelExpression; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.integration.config.IntegrationConverter; -import org.springframework.integration.expression.SpelPropertyAccessorRegistrar; -import org.springframework.integration.json.JsonPropertyAccessor; - -@AutoConfiguration -@AutoConfigureAfter(name = "org.springframework.cloud.stream.config.SpelExpressionConverterConfiguration") -@ConditionalOnMissingClass("org.springframework.cloud.stream.config.SpelExpressionConverterConfiguration") -public class SpelExpressionConverterConfiguration { - - /** - * Specific Application Context name to be used as Bean qualifier when the {@link EvaluationContext} is injected. - */ - public static final String INTEGRATION_EVALUATION_CONTEXT = "integrationEvaluationContext"; - - @Bean - public static SpelPropertyAccessorRegistrar spelPropertyAccessorRegistrar() { - return (new SpelPropertyAccessorRegistrar()) - .add(Introspector.decapitalize(JsonPropertyAccessor.class.getSimpleName()), new JsonPropertyAccessor()); - } - - @Bean - @ConfigurationPropertiesBinding - @IntegrationConverter - public Converter spelConverter() { - return new SpelExpressionConverterConfiguration.SpelConverter(); - } - - public static class SpelConverter implements Converter { - private SpelExpressionParser parser = new SpelExpressionParser(); - - @Autowired - @Qualifier(INTEGRATION_EVALUATION_CONTEXT) - @Lazy - private EvaluationContext evaluationContext; - - public SpelConverter() { - } - - public Expression convert(String source) { - try { - Expression expression = this.parser.parseExpression(source); - if (expression instanceof SpelExpression) { - ((SpelExpression) expression).setEvaluationContext(this.evaluationContext); - } - - return expression; - } - catch (ParseException var3) { - throw new IllegalArgumentException( - String.format("Could not convert '%s' into a SpEL expression", source), var3); - } - } - } -} diff --git a/functions/common/config-common/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/common/config-common/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index b62b5d1bf..000000000 --- a/functions/common/config-common/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.common.config.SpelExpressionConverterConfiguration diff --git a/functions/common/debezium-autoconfigure/README.adoc b/functions/common/debezium-autoconfigure/README.adoc deleted file mode 100644 index b618e8a65..000000000 --- a/functions/common/debezium-autoconfigure/README.adoc +++ /dev/null @@ -1,353 +0,0 @@ -= Debezium Auto-Configuration - -This module provides a generic https://debezium.io/documentation/reference/development/engine.html[DebeziumEngine.Builder] auto-configuration that can be reused and composed in other applications. - -IMPORTANT: The `DebeziumEngine` does not required Kafka or Kafka Connect as it runs embedded inside your application. -This approach though comes with some delivery guarantee limitations as explained https://debezium.io/documentation/reference/development/engine.html#%5Fhandling_failures[here]. - -The `Debezium Engine` is a https://en.wikipedia.org/wiki/Change_data_capture[Change Data Capture] (CDC) utility, that allows *capturing* database change events and process them with custom `java.util.Consumer` or `io.debezium.engine.ChangeConsumer` event handler implementations. - -The `DebeziumEngine.Builder` auto-configuration is activated only if a https://debezium.io/documentation/reference/stable/connectors/index.html[Debezium Connector] is found on the classpath and the `debezium.properties.connector.class` property is set to point to that connector class. - -== Quick Start - -To use the `DebeziumEngine`, set the required <> and <>, register a custom <> with the engine and submit later for execution. - Those simple steps are illustrated in the following sections. - -[[dependencies]] -=== Dependencies - -To process the incoming database change events you need to include the Debezium `auto-configuration` dependency to your project: - -==== -[source, xml, subs="normal", role="primary"] -.Maven ----- - - org.springframework.cloud.fn - debezium-autoconfigure - ${project-version} - ----- -[source, groovy, subs="normal", role="secondary"] -.Gradle ----- -compile "org.springframework.cloud.fn:debezium-autoconfigure:{project-version}" ----- -==== - -and include the https://debezium.io/documentation/reference/connectors/index.html[debezium connector] dependency for the selected Database. -For example the postgres debezium connector dependency looks like this: - -==== -[source, xml, subs="normal", role="primary"] -.Maven ----- - - io.debezium - debezium-connector-postgres - ${debezium-version} - - ----- -[source, groovy, subs="normal", role="secondary"] -.Gradle ----- -compile "io.debezium:debezium-connector-postgres:{debezium-version}" ----- -==== -[[changeEvent-handler]] -### ChangeEvent Handler - -To process the incoming change events, implement a `java.util.Consumer` (or `ChangeConsumer`) handler and wire it into the `DebeziumEngine.Builder`. -Then build an engine from the builder and run it from within an Executor service: - -[source, java] ----- -public void simple(DebeziumEngine.Builder> builder) { - Executors.newSingleThreadExecutor().execute(builder // <3> - .notifying(changeEvent -> System.out.println(changeEvent.value())) // <1> - .build()); // <2> -} ----- -<1> Implement and register a change events consumer. -<2> Build the Debezium engine. -<3> Submit the engine to a single thread executor. - -For a real life applications you should manage the lifecycles of the engine and its executor. -The Spring `Lifecycle` and `SmartLifecycle` interfaces can help you to manage this. -Here is a more realistic example snipped: - -[source, java] ----- -@Bean -public Consumer> customConsumer() { // <1> - return new Consumer>() { - @Override - public void accept(ChangeEvent changeEvent) { - if (changeEvent != null) { // ignore null records - System.out.println("Key:" + changeEvent.key() + ", Value: " changeEvent.value()); - } - } - } -} - -@Bean -public DebeziumEngine> debeziumEngine( // <2> - Consumer> consumer, - DebeziumEngine.Builder> builder) { - - return new builder.notifying(consumer).build(); -} - -@Bean -public EmbeddedEngineExecutorService embeddedEngine( // <3> - DebeziumEngine> debeziumEngine) { - return new EmbeddedEngineExecutorService(debeziumEngine); -} ----- -<1> Create a custom change event consumer. -<2> Use the consumer from step (1) and the debezium builder provided from the auto-configuration to create a new DebeziumEngine instance. -<3> The DebeziumEngine is designed to be submitted to an `Executor` or `ExecutorService` for execution. -The `EmbeddedEngineExecutorService` is a convenient `ExecutorService` implementation, aligned with the Spring lifecycle. - -NOTE: The `EmbeddedEngineExecutorService` is optional. -Some application such as `DebeziumReactiveConsumerConfiguration` implement the ExecutorService as part of their -`Supplier>>` configuration. - -NOTE: Neither the `Consumer`/`ChangeConsumer` nor the `DebeziumEngine` need to be `@Bean`! -It is enough to set some consumer into it the `DebeziumEngine.Builder`, build an engine and run it from within some `@Service` implementation. - -[[configuration-options]] -== Configuration Options - -$$debezium.header-format$$:: `ChangeEvent` header format. -*(default and only option at the moment: `JSON`)* -$$debezium.payload-format$$:: `ChangeEvent` Key and Payload formats. -*($$DebeziumFormat$$, default: `JSON`, possible values: `JSON`,`AVRO`,`PROTOBUF`)* -$$debezium.offset-commit-policy$$:: The policy that defines when the offsets should be committed to offset storage. -*($$DebeziumOffsetCommitPolicy$$, default: `PERIODIC`, possible values: `ALWAYS`,`PERIODIC`,`DEFAULT`)* -$$debezium.properties$$:: $$Spring pass-trough wrapper for debezium configuration properties. -All properties with a `debezium.properties.*` prefix are native Debezium properties.$$ *($$Map$$, default: `$$$$`)*. -For example the `debezium.properties.connector.class` property is converted into `connector.class` before provided to the DebeziumEngine. - -Here is an example configuration for the sample snipped above: - -[source, bash] ----- -debezium.properties.connector.class=io.debezium.connector.mysql.MySqlConnector # <1> - -debezium.properties.database.user=debezium # <2> -debezium.properties.database.password=dbz # <2> -debezium.properties.database.hostname=localhost # <2> -debezium.properties.database.port=3306 # <2> - -debezium.properties.name=my-sql-connector # <3> -debezium.properties.database.server.id=85744 # <3> -debezium.properties.topic.prefix=my-topic # <3> - -debezium.properties.key.converter.schemas.enable=true # <4> -debezium.properties.value.converter.schemas.enable=true # <4> - -debezium.properties.offset.flush.interval.ms=60000 - -debezium.properties.schema.history.internal=io.debezium.relational.history.MemorySchemaHistory # <5> -debezium.properties.offset.storage=org.apache.kafka.connect.storage.MemoryOffsetBackingStore # <5> - -debezium.header-format=JSON # <6> -debezium.payload-format=JSON # <6> - ----- -<1> Configures the Debezium Engine to use https://debezium.io/docs/connectors/mysql/[MySqlConnector]. -<2> Configure the connection to a MySQL server running on `localhost:3306` as `debezium` user. -<3> Metadata used to identify and dispatch the incoming events. -* `debezium.properties.topic.prefix` - provides a namespace for the particular database server/cluster in which Debezium is capturing changes. -The topic prefix **should be unique** across all other connectors. -Only alphanumeric characters, hyphens, dots and underscores must be used. -* `debezium.properties.database.server.id` - a numeric identifier of this database client, which **must be unique across all currently-running database processes**. -<4> Includes the https://debezium.io/docs/connectors/mysql/#change-events-value[Change Event Value] schema in the `ChangeEvent` message. -<5> Metadata stores to preserver the debezium state between multiple starts. -<6> Sets, explicitly, the ChangeEvent header and payload (e.g. key and value) serialization formats. -Defaults to JSON with binary encoding. - -=== Connectors properties - -The table below lists all available Debezium properties for each connecter. - -.Table of the native Debezium configuration properties for every connector. -|=== -| Connector | Connector properties - -|https://debezium.io/documentation/reference/reference/connectors/mysql.html[MySQL] -|https://debezium.io/documentation/reference/connectors/mysql.html#mysql-connector-properties - -|https://debezium.io/documentation/reference/connectors/mongodb.html[MongoDB] -|https://debezium.io/documentation/reference/connectors/mongodb.html#mongodb-connector-properties - -|https://debezium.io/documentation/reference/connectors/postgresql.html[PostgreSQL] -|https://debezium.io/documentation/reference/connectors/postgresql.html#postgresql-connector-properties - -|https://debezium.io/documentation/reference/connectors/oracle.html[Oracle] -|https://debezium.io/documentation/reference/connectors/oracle.html#oracle-connector-properties - -|https://debezium.io/documentation/reference/connectors/sqlserver.html[SQL Server] -|https://debezium.io/documentation/reference/connectors/sqlserver.html#sqlserver-connector-properties - -|https://debezium.io/documentation/reference/connectors/db2.html[DB2] -|https://debezium.io/documentation/reference/connectors/db2.html#db2-connector-properties - -|https://debezium.io/documentation/reference/connectors/vitess.html[Vitess] -|https://debezium.io/documentation/reference/connectors/vitess.html#vitess-connector-properties - -|https://debezium.io/documentation/reference/connectors/spanner.html[Spanner] -|https://debezium.io/documentation/reference/connectors/spanner.html#spanner-connector-properties - -|=== - -=== Streaming vs Batching - -If you register a `java.util.Consumer` with the `DebeziumEngine.Builder` then the incoming events are processed element-wise, one by one in the order of their occurrence in the source database. -Opting for the `io.debezium.engineChangeConsumer` provides an https://debezium.io/documentation/reference/stable/development/engine.html#advanced-consuming[advanced event consumption] that can process batch of events in one go, acknowledging their processing once that's done. -This snippet illustrates how to implement a batch handler: - -[source, java] ----- -@Bean -public EmbeddedEngineExecutorService batch(DebeziumEngine.Builder> builder) { - - return new EmbeddedEngineExecutorService( // <3> - - builder.notifying(new ChangeConsumer<>() { // <1> - - @Override - public void handleBatch( - List> changeEventBatch, - RecordCommitter> committer) - throws InterruptedException { - - for (ChangeEvent event : changeEventBatch) { - System.out.println(event.value()); - committer.markProcessed(event); - } - committer.markBatchFinished(); - } - - }).build()); // <2> -} ----- -<1> Implement and register a `ChangeConsumer` batch handler. -The `committer.markProcessed(event)` and `committer.markBatchFinished()` are used to mark the event and batch completion. -<2> Build the engine. -<3> Crate and return an `EmbeddedEngineExecutorService` - a Spring lifecycle manageable `ExecutorService`. - -=== Additional Configuration Components - -The Debezium builder auto-configuration provides an opinionated implementation for the following configurable components: - - - `OffsetCommitPolicy` - Commit policy type. - The default is a periodic commit policy based upon time intervals. - - `Clock` - Clock needing to determine the current time. - Defaults to the `Clock#systemDefaultZone()` system clock. -- `CompletionCallback` - callback called by the engine on `DebeziumEngine#run()` method completes with the results. -By default logs the completion status. -- `ConnectorCallback` - During the engine run, provides feedback about the the completion state of each component running within the engine (connectors, tasks etc). -By default logs the connector state. - -You can override any of the above components. -Just provide your `@Bean` implementation to the application context. - -=== Event Flattening - -Debezium provides a comprehensive message format, that accurately details information about changes that happen in the system. -Sometime this format, though, might not be suitable for the downstream consumers, that might require messages that are formatted so that field names and values are presented in a simplified, `flattened` structure. - -To simplify the format of the event records that the Debezium connectors produce, you can use the https://debezium.io/documentation/reference/stable/transformations/event-flattening.html[Debezium event flattening] message transformation: - -[source, bash] ----- -debezium.properties.transforms=flattening # <1> -debezium.properties.transforms.flattening.type=io.debezium.transforms.ExtractNewRecordState # <2> -debezium.properties.transforms.flattening.drop.tombstones=false # <3> -debezium.properties.transforms.flattening.delete.handling.mode=rewrite # <4> -debezium.properties.transforms.flattening.add.headers=op # <5> -debezium.properties.transforms.flattening.add.fields=name,db # <5> ----- -<1> flattening transformation name. -<2> Class that implements the flatting transformation. -<3> Debezium generates a tombstone record for each DELETE operation. -The default behavior is that event flattening removes tombstone records from the stream. -To keep tombstone records in the stream, specify drop.tombstones=false. -<4> Debezium generates a change event event for each DELETE operation. -The `rewrite` mode keeps those events, which a dropped otherwise. -<5> Comma-separated list of metadata fields to add to the header and the value of the simplified event value. - -Follow the https://debezium.io/documentation/reference/stable/transformations/event-flattening.html#_configuration[configuration documentation] for further details. - -=== Offset Storages - -When a Debezium source runs, it reads information from the source and periodically records `offsets` that define how much of that information it has processed. -Should the source be restarted, it will use the last recorded offset to know where in the source information it should resume reading. -Out of the box, the following https://debezium.io/documentation/reference/development/engine.html#engine-properties[offset storage configuration] options are provided: - -==== In-Memory - -Doesn't persist the offset data but keeps it in memory. -Therefore all offsets are lost on debezium source restart. - -===== -[source, bash] ----- -debezium.properties.offset.storage=org.apache.kafka.connect.storage.MemoryOffsetBackingStore ----- -===== - -==== Local Filesystem - -Store the offsets in a file on the local file system (the file can be named anything and stored anywhere). -Additionally, although the connector records the offsets with every source record it produces, the engine flushes the offsets to the backing store periodically (in the example below, once each minute). - -===== -[source, bash] ----- -debezium.properties.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore -debezium.properties.offset.storage.file.filename=/tmp/offsets.dat # <1> -debezium.properties.offset.flush.interval.ms=60000 # <2> ----- -<1> Path to file where offsets are to be stored. -Required when `offset.storage`` is set to the `FileOffsetBackingStore`. -<2> Interval at which to try committing offsets. -The default is 1 minute. -===== - -==== Kafka topic - -Uses a Kafka topic to store offset data. - -===== -[source, bash] ----- -debezium.properties.offset.storage=org.apache.kafka.connect.storage.KafkaOffsetBackingStore -debezium.properties.offset.storage.topic=my-kafka-offset-topic # <1> -debezium.properties.offset.storage.partitions=2 # <2> -debezium.properties.offset.storage.replication.factor=1 # <3> -debezium.properties.offset.flush.interval.ms=60000 # <4> ----- -<1> The name of the Kafka topic where offsets are to be stored. -Required when `offset.storage` is set to the `KafkaOffsetBackingStore`. -<2> The number of partitions used when creating the offset storage topic. -<3> Replication factor used when creating the offset storage topic. -<4> Interval at which to try committing offsets. -The default is 1 minute. -===== - -One can implement the `org.apache.kafka.connect.storage.OffsetBackingStore` interface in to provide a offset storage bound to a custom backend key-value store. - -== Tests - -See this link:org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationIntegrationTest.java[test suite] for how to use the auto-configuration with custom Consumer. - -== Other usage - -- See the https://github.com/spring-cloud/stream-applications/blob/master/functions/supplier/debezium-source/debezium-supplier[debezium-supplier] implementation about how to implement reactive consumer on top of the debezium auto-configuration. -- See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/debezium-source/README.adoc[debezium-source] about how the debezium auto-configuration and supplier are used to create a Spring Cloud Stream applications. -- See the https://docs.spring.io/spring-integration/docs/6.2.0-SNAPSHOT/reference/html/debezium.html#debezium[Spring Integration Debezium support] about how to initialize Inbound Debezium Channel Adapter with `DebeziumEngine.Builder>` provided by the auto-configuration. diff --git a/functions/common/debezium-autoconfigure/pom.xml b/functions/common/debezium-autoconfigure/pom.xml deleted file mode 100644 index 5231847de..000000000 --- a/functions/common/debezium-autoconfigure/pom.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - debezium-autoconfigure - debezium-autoconfigure - Debezium Spring Boot auto-configuration - - - 2.3.3.Final - 2.4.2.Final - - - - - io.debezium - debezium-embedded - ${version.debezium} - - - org.slf4j - slf4j-reload4j - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-api - - - - - - - io.apicurio - apicurio-registry-utils-converter - ${apicurio.version} - - - io.apicurio - apicurio-registry-client - ${apicurio.version} - - - org.slf4j - slf4j-api - - - - - - - io.debezium - debezium-connector-mysql - ${version.debezium} - - - slf4j-log4j12 - org.slf4j - - - test - - - com.zaxxer - HikariCP - 4.0.3 - test - - - org.springframework - spring-jdbc - test - - - ch.qos.logback - logback-classic - 1.4.8 - test - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0 - - 1 - 1 - integration - - - - - diff --git a/functions/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfiguration.java b/functions/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfiguration.java deleted file mode 100644 index 6e6c6656f..000000000 --- a/functions/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfiguration.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.debezium; - -import java.time.Clock; -import java.time.Duration; -import java.util.Objects; - -import io.debezium.engine.ChangeEvent; -import io.debezium.engine.DebeziumEngine; -import io.debezium.engine.DebeziumEngine.CompletionCallback; -import io.debezium.engine.DebeziumEngine.ConnectorCallback; -import io.debezium.engine.format.KeyValueHeaderChangeEventFormat; -import io.debezium.engine.format.SerializationFormat; -import io.debezium.engine.spi.OffsetCommitPolicy; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.condition.AnyNestedCondition; -import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.debezium.DebeziumProperties.DebeziumFormat; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Conditional; -import org.springframework.core.Ordered; -import org.springframework.core.annotation.Order; - -/** - * {@link EnableAutoConfiguration Auto-configuration} for {@link DebeziumEngine.Builder}. - *

- * The builder provides a standalone engine configuration that talks with the source data system. - *

- * The application that runs the debezium engine assumes all responsibility for fault tolerance, scalability, and - * durability. Additionally, applications must specify how the engine can store its relational database schema history - * and offsets. By default, this information will be stored in memory and will thus be lost upon application restart. - *

- * The {@link DebeziumEngine.Builder} auto-configuration is activated only if a Debezium Connector is available on the - * classpath and the debezium.properties.connector.class property is set. - *

- * Properties prefixed with debezium.properties are passed through as native Debezium properties. - * - * @author Christian Tzolov - * @author Corneil du Plessis - */ -@AutoConfiguration -@EnableConfigurationProperties(DebeziumProperties.class) -@Conditional(DebeziumEngineBuilderAutoConfiguration.OnDebeziumConnectorCondition.class) -@ConditionalOnProperty(prefix = "debezium", name = "properties.connector.class") -public class DebeziumEngineBuilderAutoConfiguration { - - private static final Log logger = LogFactory.getLog(DebeziumEngineBuilderAutoConfiguration.class); - - /** - * The fully-qualified class name of the commit policy type. The default is a periodic commit policy based upon time - * intervals. - * @param properties The 'debezium.properties.offset.flush.interval.ms' configuration is compulsory for the Periodic - * policy type. The ALWAYS and DEFAULT doesn't require additional configuration. - */ - @Bean - @ConditionalOnMissingBean - public OffsetCommitPolicy offsetCommitPolicy(DebeziumProperties properties) { - - switch (properties.getOffsetCommitPolicy()) { - case PERIODIC: - return OffsetCommitPolicy.periodic(properties.getDebeziumNativeConfiguration()); - case ALWAYS: - return OffsetCommitPolicy.always(); - case DEFAULT: - default: - return NULL_OFFSET_COMMIT_POLICY; - } - } - - /** - * Use the specified clock when needing to determine the current time. Defaults to {@link Clock#systemDefaultZone() - * system clock}, but you can override the Bean in your configuration with you {@link Clock implementation}. Returns - * @return Clock for the system default zone. - */ - @Bean - @ConditionalOnMissingBean - public Clock debeziumClock() { - return Clock.systemDefaultZone(); - } - - /** - * When the engine's {@link DebeziumEngine#run()} method completes, call the supplied function with the results. - * @return Default completion callback that logs the completion status. The bean can be overridden in custom - * implementation. - */ - @Bean - @ConditionalOnMissingBean - public CompletionCallback completionCallback() { - return DEFAULT_COMPLETION_CALLBACK; - } - - /** - * During the engine run, provides feedback about the different stages according to the completion state of each - * component running within the engine (connectors, tasks etc). The bean can be overridden in custom implementation. - */ - @Bean - @ConditionalOnMissingBean - public ConnectorCallback connectorCallback() { - return DEFAULT_CONNECTOR_CALLBACK; - } - - @Bean - @ConditionalOnMissingBean - public DebeziumEngine.Builder> debeziumEngineBuilder( - OffsetCommitPolicy offsetCommitPolicy, CompletionCallback completionCallback, - ConnectorCallback connectorCallback, DebeziumProperties properties, Clock debeziumClock) { - - Class> payloadFormat = Objects.requireNonNull( - serializationFormatClass(properties.getPayloadFormat()), - "Cannot find payload format for " + properties.getProperties()); - - Class> headerFormat = Objects.requireNonNull( - serializationFormatClass(properties.getHeaderFormat()), - "Cannot find header format for " + properties.getProperties()); - - return DebeziumEngine - .create(KeyValueHeaderChangeEventFormat.of(payloadFormat, payloadFormat, headerFormat)) - .using(properties.getDebeziumNativeConfiguration()) - .using(debeziumClock) - .using(completionCallback) - .using(connectorCallback) - .using((offsetCommitPolicy != NULL_OFFSET_COMMIT_POLICY) ? offsetCommitPolicy : null); - } - - /** - * Converts the {@link DebeziumFormat} enum into Debezium {@link SerializationFormat} class. - * @param debeziumFormat debezium format property. - */ - private Class> serializationFormatClass(DebeziumFormat debeziumFormat) { - switch (debeziumFormat) { - case JSON: - return io.debezium.engine.format.JsonByteArray.class; - case AVRO: - return io.debezium.engine.format.Avro.class; - case PROTOBUF: - return io.debezium.engine.format.Protobuf.class; - default: - throw new IllegalArgumentException("Unknown debezium format: " + debeziumFormat); - } - } - - /** - * A callback function to be notified when the connector completes. - */ - private static final CompletionCallback DEFAULT_COMPLETION_CALLBACK = new CompletionCallback() { - @Override - public void handle(boolean success, String message, Throwable error) { - logger.info(String.format("Debezium Engine completed with success:%s, message:%s ", success, message), - error); - } - }; - - /** - * Callback function which informs users about the various stages a connector goes through during startup. - */ - private static final ConnectorCallback DEFAULT_CONNECTOR_CALLBACK = new ConnectorCallback() { - - /** - * Called after a connector has been successfully started by the engine. - */ - public void connectorStarted() { - logger.info("Connector Started!"); - }; - - /** - * Called after a connector has been successfully stopped by the engine. - */ - public void connectorStopped() { - logger.info("Connector Stopped!"); - } - - /** - * Called after a connector task has been successfully started by the engine. - */ - public void taskStarted() { - logger.info("Connector Task Started!"); - } - - /** - * Called after a connector task has been successfully stopped by the engine. - */ - public void taskStopped() { - logger.info("Connector Task Stopped!"); - } - - }; - - /** - * The policy that defines when the offsets should be committed to offset storage. - */ - private static final OffsetCommitPolicy NULL_OFFSET_COMMIT_POLICY = new OffsetCommitPolicy() { - @Override - public boolean performCommit(long numberOfMessagesSinceLastCommit, Duration timeSinceLastCommit) { - throw new UnsupportedOperationException("Unimplemented method 'performCommit'"); - } - }; - - /** - * Determine if Debezium connector is available. This either kicks in if any debezium connector is available. - */ - @Order(Ordered.LOWEST_PRECEDENCE) - static class OnDebeziumConnectorCondition extends AnyNestedCondition { - - OnDebeziumConnectorCondition() { - super(ConfigurationPhase.REGISTER_BEAN); - } - - @ConditionalOnClass(name = { "io.debezium.connector.mysql.MySqlConnector" }) - static class HasMySqlConnector { - - } - - @ConditionalOnClass(name = "io.debezium.connector.postgresql.PostgresConnector") - static class HasPostgreSqlConnector { - - } - - @ConditionalOnClass(name = "io.debezium.connector.db2.Db2Connector") - static class HasDb2Connector { - - } - - @ConditionalOnClass(name = "io.debezium.connector.oracle.OracleConnector") - static class HasOracleConnector { - - } - - @ConditionalOnClass(name = "io.debezium.connector.sqlserver.SqlServerConnector") - static class HasSqlServerConnector { - - } - - @ConditionalOnClass(name = "io.debezium.connector.mongodb.MongoDbConnector") - static class HasMongoDbConnector { - - } - - @ConditionalOnClass(name = "io.debezium.connector.vitess.VitessConnector") - static class HasVitessConnector { - - } - - @ConditionalOnClass(name = "io.debezium.connector.spanner.SpannerConnector") - static class HasSpannerConnector { - - } - - } - -} diff --git a/functions/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumProperties.java b/functions/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumProperties.java deleted file mode 100644 index 23a2cf9bf..000000000 --- a/functions/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/DebeziumProperties.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.debezium; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import org.springframework.boot.context.properties.ConfigurationProperties; - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("debezium") -public class DebeziumProperties { - - public enum DebeziumFormat { - /** - * JSON change event format. - */ - JSON("application/json"), - /** - * AVRO change event format. - */ - AVRO("application/avro"), - /** - * ProtoBuf change event format. - */ - PROTOBUF("application/x-protobuf"),; - - private final String contentType; - - DebeziumFormat(String contentType) { - this.contentType = contentType; - } - - public final String contentType() { - return contentType; - } - }; - - /** - * Spring pass-trough wrapper for debezium configuration properties. All properties with a 'debezium.properties.*' - * prefix are native Debezium properties. - */ - private Map properties = new HashMap<>(); - - /** - * {@link ChangeEvent} Key and Payload formats. Defaults to 'JSON'. - */ - private DebeziumFormat payloadFormat = DebeziumFormat.JSON; - - /** - * {@link ChangeEvent} header format. Defaults to 'JSON'. - */ - private DebeziumFormat headerFormat = DebeziumFormat.JSON; - - /** - * The policy that defines when the offsets should be committed to offset storage. - */ - private DebeziumOffsetCommitPolicy offsetCommitPolicy = DebeziumOffsetCommitPolicy.DEFAULT; - - public Map getProperties() { - return properties; - } - - public DebeziumFormat getPayloadFormat() { - return payloadFormat; - } - - public void setPayloadFormat(DebeziumFormat format) { - this.payloadFormat = format; - } - - public DebeziumFormat getHeaderFormat() { - return headerFormat; - } - - public void setHeaderFormat(DebeziumFormat headerFormat) { - this.headerFormat = headerFormat; - } - - public enum DebeziumOffsetCommitPolicy { - /** - * Commits offsets as frequently as possible. This may result in reduced performance, but it has the least - * potential for seeing source records more than once upon restart. - */ - ALWAYS, - /** - * Commits offsets no more than the specified time period. If the specified time is less than {@code 0} then the - * policy will behave as ALWAYS policy. Requires the 'debezium.properties.offset.flush.interval.ms' native - * property to be set. - */ - PERIODIC, - /** - * Uses the default Debezium engine policy (PERIODIC). - */ - DEFAULT; - } - - public DebeziumOffsetCommitPolicy getOffsetCommitPolicy() { - return offsetCommitPolicy; - } - - public void setOffsetCommitPolicy(DebeziumOffsetCommitPolicy offsetCommitPolicy) { - this.offsetCommitPolicy = offsetCommitPolicy; - } - - /** - * Converts the Spring Framework "debezium.properties.*" properties into native Debezium configuration. - */ - public Properties getDebeziumNativeConfiguration() { - Properties outProps = new java.util.Properties(); - outProps.putAll(this.getProperties()); - return outProps; - } -} diff --git a/functions/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/EmbeddedEngineExecutorService.java b/functions/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/EmbeddedEngineExecutorService.java deleted file mode 100644 index 971e6cf0e..000000000 --- a/functions/common/debezium-autoconfigure/src/main/java/org/springframework/cloud/fn/common/debezium/EmbeddedEngineExecutorService.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.debezium; - -import java.io.IOException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicBoolean; - -import io.debezium.engine.DebeziumEngine; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.context.SmartLifecycle; - -/** - * The Debezium Engine is designed to be submitted to an {@link Executor} or {@link ExecutorService} for execution by a - * single thread, and a running connector can be stopped either by calling {@link #stop()} from another thread or by - * interrupting the running thread (e.g., as is the case with {@link ExecutorService#shutdownNow()}). - * - * The EmbeddedEngineExecutorService provides a sample ExecutorService implementation aligned with the Spring lifecycle. - * - * Note that the DebeziumReactiveConsumerConfiguration embeds an ExecutorService as part of the - * Supplier<Flux<Message<?>>> configuration. - * - * @author Christian Tzolov - */ -public class EmbeddedEngineExecutorService implements SmartLifecycle, AutoCloseable { - - private static final Log logger = LogFactory.getLog(EmbeddedEngineExecutorService.class); - - private final DebeziumEngine engine; - private final ExecutorService executor; - private final AtomicBoolean running = new AtomicBoolean(false); - - public EmbeddedEngineExecutorService(DebeziumEngine engine) { - this.engine = engine; - this.executor = Executors.newSingleThreadExecutor(); - } - - @Override - public void start() { - logger.info("Start Embedded Engine"); - this.executor.execute(this.engine); - this.running.set(true); - } - - @Override - public void stop() { - this.close(); - } - - @Override - public void close() { - logger.info("Stop Embedded Engine"); - try { - this.engine.close(); - this.running.set(false); - } - catch (IOException e) { - logger.warn("Failed to close the Debezium Engine:", e); - } - this.executor.shutdown(); - } - - @Override - public boolean isRunning() { - return this.running.get(); - } -} diff --git a/functions/common/debezium-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/common/debezium-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index a5f9a35b7..000000000 --- a/functions/common/debezium-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.common.debezium.DebeziumEngineBuilderAutoConfiguration diff --git a/functions/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationIntegrationTest.java b/functions/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationIntegrationTest.java deleted file mode 100644 index fd39520f2..000000000 --- a/functions/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationIntegrationTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.debezium; - -import java.io.File; -import java.time.Duration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.function.Consumer; - -import javax.sql.DataSource; - -import com.zaxxer.hikari.HikariDataSource; -import io.debezium.engine.ChangeEvent; -import io.debezium.engine.DebeziumEngine; -import io.debezium.engine.DebeziumEngine.Builder; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; -import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; -import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.test.jdbc.JdbcTestUtils; - -import static org.awaitility.Awaitility.await; - -/** - * This test illustrate how to leverage the DebeziumEngineAutoConfiguration to build a consumer function with a custom - * change event Consumer. - * - * @author Christian Tzolov - */ -@Tag("integration") -@Testcontainers -public class DebeziumEngineBuilderAutoConfigurationIntegrationTest { - private static final Log logger = LogFactory.getLog(DebeziumEngineBuilderAutoConfigurationIntegrationTest.class); - - private static final String DATABASE_NAME = "inventory"; - public static final String IMAGE_TAG = "2.3.0.Final"; - public static final String DEBEZIUM_EXAMPLE_MYSQL_IMAGE = "debezium/example-mysql:" + IMAGE_TAG; - - @TempDir - static File anotherTempDir; - - @Container - static GenericContainer debeziumMySQL = new GenericContainer<>(DEBEZIUM_EXAMPLE_MYSQL_IMAGE) - .withEnv("MYSQL_ROOT_PASSWORD", "debezium") - .withEnv("MYSQL_USER", "mysqluser") - .withEnv("MYSQL_PASSWORD", "mysqlpw") - .withExposedPorts(3306); - - private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() - .withUserConfiguration(DebeziumCustomConsumerApplication.class) - .withPropertyValues( - "spring.datasource.type=com.zaxxer.hikari.HikariDataSource", - - "debezium.properties.offset.storage=org.apache.kafka.connect.storage.FileOffsetBackingStore", - "debezium.properties.offset.storage.file.filename=" + anotherTempDir.getAbsolutePath() - + "offsets.dat", - "debezium.properties.offset.flush.interval.ms=60000", - - "debezium.properties.schema.history.internal=io.debezium.storage.file.history.FileSchemaHistory", // new - "debezium.properties.schema.history.internal.file.filename=" + anotherTempDir.getAbsolutePath() - + "schemahistory.dat", - - "debezium.properties.topic.prefix=my-topic", - - "debezium.properties.name=my-sql-connector", - "debezium.properties.connector.class=io.debezium.connector.mysql.MySqlConnector", - - "debezium.properties.database.user=debezium", - "debezium.properties.database.password=dbz", - "debezium.properties.database.hostname=localhost", - "debezium.properties.database.port=" + debeziumMySQL.getMappedPort(3306), - "debezium.properties.database.server.id=85744", - - // JdbcTemplate configuration - String.format("app.datasource.url=jdbc:mysql://localhost:%d/%s?enabledTLSProtocols=TLSv1.2", - debeziumMySQL.getMappedPort(3306), DATABASE_NAME), - "app.datasource.username=root", - "app.datasource.password=debezium", - "app.datasource.driver-class-name=com.mysql.cj.jdbc.Driver", - "app.datasource.type=com.zaxxer.hikari.HikariDataSource"); - - @Test - public void consumerTest() { - - logger.info("Temp dir: " + anotherTempDir.getAbsolutePath()); - - contextRunner - .withPropertyValues( - // Flattering: - // https://debezium.io/documentation/reference/stable/transformations/event-flattening.html - "debezium.properties.transforms=unwrap", - "debezium.properties.transforms.unwrap.type=io.debezium.transforms.ExtractNewRecordState", - "debezium.properties.transforms.unwrap.drop.tombstones=false", - "debezium.properties.transforms.unwrap.delete.handling.mode=rewrite", - "debezium.properties.transforms.unwrap.add.fields=name,db") - .run(context -> { - JdbcTemplate jdbcTemplate = context.getBean(JdbcTemplate.class); - - DebeziumCustomConsumerApplication.TestDebeziumConsumer testConsumer = context - .getBean(DebeziumCustomConsumerApplication.TestDebeziumConsumer.class); - jdbcTemplate.update( - "insert into `customers`(`first_name`,`last_name`,`email`) " + - "VALUES('Test666', 'Test666', 'Test666@spring.org')"); - JdbcTestUtils.deleteFromTableWhere(jdbcTemplate, "customers", "first_name = ?", "Test666"); - - await().atMost(Duration.ofSeconds(30)).until(() -> (testConsumer.recordList.size() >= 52)); - }); - } - - @SpringBootConfiguration - @EnableAutoConfiguration(exclude = { DataSourceAutoConfiguration.class, MongoAutoConfiguration.class }) - public static class DebeziumCustomConsumerApplication { - - @Bean - public JdbcTemplate myJdbcTemplate(DataSource dataSource) { - return new JdbcTemplate(dataSource); - } - - @Bean - @Primary - @ConfigurationProperties("app.datasource") - public DataSourceProperties dataSourceProperties() { - return new DataSourceProperties(); - } - - @Bean - public HikariDataSource dataSource(DataSourceProperties dataSourceProperties) { - return dataSourceProperties.initializeDataSourceBuilder() - .type(HikariDataSource.class) - .build(); - } - - @Bean - public EmbeddedEngineExecutorService embeddedEngine(Consumer> changeEventConsumer, - Builder> debeziumEngineBuilder) { - DebeziumEngine> b = debeziumEngineBuilder.notifying(changeEventConsumer) - .build(); - return new EmbeddedEngineExecutorService(debeziumEngineBuilder.notifying(changeEventConsumer).build()); - } - - @Bean - public Consumer> customConsumer() { - return new TestDebeziumConsumer(); - } - - /** - * Custom, test, change event Consumer. - */ - public static class TestDebeziumConsumer implements Consumer> { - - public Map keyValue = new HashMap<>(); - - public List> recordList = new CopyOnWriteArrayList<>(); - - public TestDebeziumConsumer() { - } - - @Override - public void accept(ChangeEvent changeEvent) { - if (changeEvent != null) { // ignore null records - recordList.add(changeEvent); - keyValue.put(changeEvent.key(), changeEvent.value()); - System.out.println("Key: " + recordList.size()); - System.out.println("[Debezium Event]: " + changeEvent.toString()); - } - } - } - } - -} diff --git a/functions/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationTests.java b/functions/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationTests.java deleted file mode 100644 index 50840e8c5..000000000 --- a/functions/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumEngineBuilderAutoConfigurationTests.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.debezium; - -import io.debezium.engine.DebeziumEngine; -import org.junit.jupiter.api.Test; - -import org.springframework.boot.autoconfigure.AutoConfigurations; -import org.springframework.boot.test.context.FilteredClassLoader; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Tests for {@link DebeziumEngineBuilderAutoConfiguration}. - * - * @author Christian Tzolov - */ -public class DebeziumEngineBuilderAutoConfigurationTests { - - private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() - .withConfiguration(AutoConfigurations.of(DebeziumEngineBuilderAutoConfiguration.class)); - - // We have the debezium connectors on the classpath by default. - - @Test - void noConnectorNoProperty() { - this.contextRunner.run((context) -> { - assertThat(context).doesNotHaveBean(DebeziumEngine.Builder.class); - }); - } - - @Test - void noConnectorWithProperty() { - this.contextRunner.withPropertyValues("debezium.properties.connector.class=Dummy") - .withClassLoader(new FilteredClassLoader("io.debezium.connector")) - .run((context) -> { - assertThat(context).doesNotHaveBean(DebeziumEngine.Builder.class); - }); - } - - @Test - void withConnectorWithProperty() { - this.contextRunner.withPropertyValues("debezium.properties.connector.class=Dummy").run((context) -> { - assertThat(context).hasSingleBean(DebeziumEngine.Builder.class); - }); - } - -} diff --git a/functions/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumPropertiesTests.java b/functions/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumPropertiesTests.java deleted file mode 100644 index 6c9638951..000000000 --- a/functions/common/debezium-autoconfigure/src/test/java/org/springframework/cloud/fn/common/debezium/DebeziumPropertiesTests.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.debezium; - -import org.junit.jupiter.api.Test; - -import org.springframework.cloud.fn.common.debezium.DebeziumProperties.DebeziumFormat; -import org.springframework.cloud.fn.common.debezium.DebeziumProperties.DebeziumOffsetCommitPolicy; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Tests for {@link DebeziumProperties}. - * - * @author Christian Tzolov - */ -public class DebeziumPropertiesTests { - - DebeziumProperties properties = new DebeziumProperties(); - - @Test - public void defaultPropertiesTest() { - assertThat(this.properties.getPayloadFormat()).isEqualTo(DebeziumFormat.JSON); - assertThat(this.properties.getHeaderFormat()).isEqualTo(DebeziumFormat.JSON); - assertThat(this.properties.getOffsetCommitPolicy()).isEqualTo(DebeziumOffsetCommitPolicy.DEFAULT); - assertThat(this.properties.getProperties()).isNotNull(); - assertThat(this.properties.getProperties()).isEmpty(); - } - - @Test - public void debeziumFormatTest() { - this.properties.setPayloadFormat(DebeziumFormat.AVRO); - assertThat(this.properties.getPayloadFormat()).isEqualTo(DebeziumFormat.AVRO); - assertThat(this.properties.getPayloadFormat().contentType()).isEqualTo("application/avro"); - - this.properties.setPayloadFormat(DebeziumFormat.JSON); - assertThat(this.properties.getPayloadFormat()).isEqualTo(DebeziumFormat.JSON); - assertThat(this.properties.getPayloadFormat().contentType()).isEqualTo("application/json"); - - this.properties.setPayloadFormat(DebeziumFormat.PROTOBUF); - assertThat(this.properties.getPayloadFormat()).isEqualTo(DebeziumFormat.PROTOBUF); - assertThat(this.properties.getPayloadFormat().contentType()).isEqualTo("application/x-protobuf"); - } - -} diff --git a/functions/common/file-common/pom.xml b/functions/common/file-common/pom.xml deleted file mode 100644 index 1609e5509..000000000 --- a/functions/common/file-common/pom.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - file-common - file-common - file common - - - - org.springframework.integration - spring-integration-file - - - - diff --git a/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileConsumerProperties.java b/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileConsumerProperties.java deleted file mode 100644 index 7d3cc19dd..000000000 --- a/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileConsumerProperties.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.file; - -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * - * @author David Turanski - * @author Artem Bilan - */ -@ConfigurationProperties("file.consumer") -@Validated -public class FileConsumerProperties { - - /** - * The FileReadingMode to use for file reading sources. - * Values are 'ref' - The File object, - * 'lines' - a message per line, or - * 'contents' - the contents as bytes. - */ - private FileReadingMode mode = FileReadingMode.contents; - - /** - * Set to true to emit start of file/end of file marker messages before/after the data. - * Only valid with FileReadingMode 'lines'. - */ - private Boolean withMarkers = null; - - /** - * When 'fileMarkers == true', specify if they should be produced - * as FileSplitter.FileMarker objects or JSON. - */ - private boolean markersJson = true; - - @NotNull - public FileReadingMode getMode() { - return this.mode; - } - - public void setMode(FileReadingMode mode) { - this.mode = mode; - } - - public Boolean getWithMarkers() { - return this.withMarkers; - } - - public void setWithMarkers(Boolean withMarkers) { - this.withMarkers = withMarkers; - } - - public boolean getMarkersJson() { - return this.markersJson; - } - - public void setMarkersJson(boolean markersJson) { - this.markersJson = markersJson; - } - - @AssertTrue(message = "withMarkers can only be supplied when FileReadingMode is 'lines'") - public boolean isWithMarkersValid() { - return this.withMarkers == null || FileReadingMode.lines == this.mode; - } -} diff --git a/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileReadingMode.java b/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileReadingMode.java deleted file mode 100644 index c6db428e5..000000000 --- a/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileReadingMode.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.file; - -/** - * Defines the supported modes of reading and processing files. - * - * @author Gunnar Hillert - * @author David Turanski - */ -public enum FileReadingMode { - /** - * ref mode. - */ - ref, - /** - * lines mode. - */ - lines, - /** - * contents mode. - */ - contents; -} diff --git a/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileUtils.java b/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileUtils.java deleted file mode 100644 index 9a56618bf..000000000 --- a/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileUtils.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.file; - -import java.util.Collections; - -import org.springframework.integration.dsl.IntegrationFlowBuilder; -import org.springframework.integration.file.splitter.FileSplitter; -import org.springframework.integration.file.transformer.FileToByteArrayTransformer; -import org.springframework.integration.transformer.StreamTransformer; -import org.springframework.messaging.MessageHeaders; -import org.springframework.util.MimeTypeUtils; - -/** - * @author Gary Russell - * @author Artem Bilan - * @author Christian Tzolov - */ -public final class FileUtils { - - private FileUtils() { - - } - - /** - * Enhance an {@link IntegrationFlowBuilder} to add flow snippets, depending on - * {@link FileConsumerProperties}. - * - * @param flowBuilder the flow builder. - * @param fileConsumerProperties the properties. - * @return the updated flow builder. - */ - public static IntegrationFlowBuilder enhanceFlowForReadingMode(IntegrationFlowBuilder flowBuilder, - FileConsumerProperties fileConsumerProperties) { - switch (fileConsumerProperties.getMode()) { - case contents: - flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, - MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE)) - .transform(new FileToByteArrayTransformer()); - break; - case lines: - Boolean withMarkers = fileConsumerProperties.getWithMarkers(); - if (withMarkers == null) { - withMarkers = false; - } - flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, - MimeTypeUtils.TEXT_PLAIN_VALUE)) - .split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson())); - break; - case ref: - flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, - MimeTypeUtils.APPLICATION_JSON_VALUE)); - break; - default: - throw new IllegalArgumentException(fileConsumerProperties.getMode().name() + - " is not a supported file reading mode."); - } - return flowBuilder; - } - - /** - * Enhance an {@link IntegrationFlowBuilder} to add flow snippets, depending on - * {@link FileConsumerProperties}; used for streaming sources. - * - * @param flowBuilder the flow builder. - * @param fileConsumerProperties the properties. - * @return the updated flow builder. - */ - public static IntegrationFlowBuilder enhanceStreamFlowForReadingMode(IntegrationFlowBuilder flowBuilder, - FileConsumerProperties fileConsumerProperties) { - switch (fileConsumerProperties.getMode()) { - case contents: - flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, - MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE)) - .transform(new StreamTransformer()); - break; - case lines: - Boolean withMarkers = fileConsumerProperties.getWithMarkers(); - if (withMarkers == null) { - withMarkers = false; - } - flowBuilder.enrichHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, - MimeTypeUtils.TEXT_PLAIN_VALUE)) - .split(new FileSplitter(true, withMarkers, fileConsumerProperties.getMarkersJson())); - break; - case ref: - default: - throw new IllegalArgumentException(fileConsumerProperties.getMode().name() + - " is not a supported file reading mode when streaming."); - } - return flowBuilder; - } - -} diff --git a/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/remote/RemoteFileDeletingAdvice.java b/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/remote/RemoteFileDeletingAdvice.java deleted file mode 100644 index 67fd7f81b..000000000 --- a/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/remote/RemoteFileDeletingAdvice.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2020-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.file.remote; - -import org.springframework.integration.aop.MessageSourceMutator; -import org.springframework.integration.core.MessageSource; -import org.springframework.integration.file.FileHeaders; -import org.springframework.integration.file.remote.RemoteFileTemplate; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; - -/** - * A {@link MessageSourceMutator} that deletes a remote file on success. - * - * @author David Turanski - * @author Artem Bilan - * - */ -public class RemoteFileDeletingAdvice implements MessageSourceMutator { - - private final RemoteFileTemplate template; - - private final String remoteFileSeparator; - - /** - * Construct an instance with the provided template and separator. - * @param template the template. - * @param remoteFileSeparator the separator. - */ - public RemoteFileDeletingAdvice(RemoteFileTemplate template, - String remoteFileSeparator) { - this.template = template; - this.remoteFileSeparator = remoteFileSeparator; - } - - @Nullable - @Override - public Message afterReceive(@Nullable Message result, MessageSource source) { - if (result != null) { - String remoteDir = (String) result.getHeaders().get(FileHeaders.REMOTE_DIRECTORY); - String remoteFile = (String) result.getHeaders().get(FileHeaders.REMOTE_FILE); - this.template.remove(remoteDir + this.remoteFileSeparator + remoteFile); - } - return result; - } -} diff --git a/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/remote/RemoteFileRenamingAdvice.java b/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/remote/RemoteFileRenamingAdvice.java deleted file mode 100644 index 061816832..000000000 --- a/functions/common/file-common/src/main/java/org/springframework/cloud/fn/common/file/remote/RemoteFileRenamingAdvice.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2020-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.file.remote; - -import org.springframework.expression.Expression; -import org.springframework.integration.aop.MessageSourceMutator; -import org.springframework.integration.core.MessageSource; -import org.springframework.integration.file.FileHeaders; -import org.springframework.integration.file.remote.RemoteFileTemplate; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; - - -/** - * A {@link MessageSourceMutator} that renames a remote file on success. - * - * @author Andrea Montemaggio - * - */ -public class RemoteFileRenamingAdvice implements MessageSourceMutator { - - private final RemoteFileTemplate template; - - private final String remoteFileSeparator; - - private final Expression newName; - - /** - * Construct an instance with the provided template and separator. - * @param template the template. - * @param remoteFileSeparator the separator. - * @param newNameExp the SpEl expression for the new name. - */ - public RemoteFileRenamingAdvice(RemoteFileTemplate template, - String remoteFileSeparator, - Expression newNameExp) { - this.template = template; - this.remoteFileSeparator = remoteFileSeparator; - this.newName = newNameExp; - } - - @Nullable - @Override - public Message afterReceive(@Nullable Message result, MessageSource source) { - if (result != null) { - String remoteDir = (String) result.getHeaders().get(FileHeaders.REMOTE_DIRECTORY); - String remoteFile = (String) result.getHeaders().get(FileHeaders.REMOTE_FILE); - String newNameValue = this.newName.getValue(result, String.class); - if (newNameValue != null && !newNameValue.isEmpty()) { - this.template.rename(remoteDir + this.remoteFileSeparator + remoteFile, newNameValue); - } - } - return result; - } -} diff --git a/functions/common/ftp-common/pom.xml b/functions/common/ftp-common/pom.xml deleted file mode 100644 index 01ae1a393..000000000 --- a/functions/common/ftp-common/pom.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - ftp-common - ftp-common - ftp common - - - - - org.springframework.integration - spring-integration-ftp - - - - - diff --git a/functions/common/ftp-common/src/main/java/org/springframework/cloud/fn/common/ftp/FtpSessionFactoryConfiguration.java b/functions/common/ftp-common/src/main/java/org/springframework/cloud/fn/common/ftp/FtpSessionFactoryConfiguration.java deleted file mode 100644 index cc2c0e17a..000000000 --- a/functions/common/ftp-common/src/main/java/org/springframework/cloud/fn/common/ftp/FtpSessionFactoryConfiguration.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.ftp; - -import org.apache.commons.net.ftp.FTPFile; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.file.remote.session.CachingSessionFactory; -import org.springframework.integration.file.remote.session.SessionFactory; -import org.springframework.integration.ftp.session.DefaultFtpSessionFactory; - -@Configuration -@EnableConfigurationProperties(FtpSessionFactoryProperties.class) -public class FtpSessionFactoryConfiguration { - - @Bean - @ConditionalOnMissingBean - public SessionFactory ftpSessionFactory(FtpSessionFactoryProperties properties) { - DefaultFtpSessionFactory ftpSessionFactory = new DefaultFtpSessionFactory(); - ftpSessionFactory.setHost(properties.getHost()); - ftpSessionFactory.setPort(properties.getPort()); - ftpSessionFactory.setUsername(properties.getUsername()); - ftpSessionFactory.setPassword(properties.getPassword()); - ftpSessionFactory.setClientMode(properties.getClientMode().getMode()); - if (properties.getCacheSessions() != null) { - CachingSessionFactory csf = new CachingSessionFactory<>(ftpSessionFactory); - return csf; - } - else { - return ftpSessionFactory; - } - } - -} diff --git a/functions/common/ftp-common/src/main/java/org/springframework/cloud/fn/common/ftp/FtpSessionFactoryProperties.java b/functions/common/ftp-common/src/main/java/org/springframework/cloud/fn/common/ftp/FtpSessionFactoryProperties.java deleted file mode 100644 index 5dc145589..000000000 --- a/functions/common/ftp-common/src/main/java/org/springframework/cloud/fn/common/ftp/FtpSessionFactoryProperties.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.ftp; - -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import org.apache.commons.net.ftp.FTPClient; -import org.hibernate.validator.constraints.Range; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -@ConfigurationProperties("ftp.factory") -@Validated -public class FtpSessionFactoryProperties { - - /** - * The port of the server. - */ - private int port = 21; - - /** - * The client mode to use for the FTP session. - */ - private ClientMode clientMode = ClientMode.PASSIVE; - - /** - * The host name of the server. - */ - private String host = "localhost"; - - /** - * The username to use to connect to the server. - */ - - private String username; - /** - * The password to use to connect to the server. - */ - private String password; - - /** - * Cache sessions. - */ - private Boolean cacheSessions; - - @Range(min = 0, max = 65535) - public int getPort() { - return this.port; - } - - public void setPort(int port) { - this.port = port; - } - - @NotNull - public ClientMode getClientMode() { - return this.clientMode; - } - - public void setClientMode(ClientMode clientMode) { - this.clientMode = clientMode; - } - - public enum ClientMode { - - /** - * Active client mode. - */ - ACTIVE(FTPClient.ACTIVE_LOCAL_DATA_CONNECTION_MODE), - /** - * Passive client mode. - */ - PASSIVE(FTPClient.PASSIVE_LOCAL_DATA_CONNECTION_MODE); - - private final int mode; - - ClientMode(int mode) { - this.mode = mode; - } - - public int getMode() { - return mode; - } - - } - - @NotBlank - public String getHost() { - return this.host; - } - - public void setHost(String host) { - this.host = host; - } - - @NotBlank - public String getUsername() { - return this.username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return this.password; - } - - public void setPassword(String password) { - this.password = password; - } - - public Boolean getCacheSessions() { - return this.cacheSessions; - } - - public void setCacheSessions(Boolean cacheSessions) { - this.cacheSessions = cacheSessions; - } - -} diff --git a/functions/common/function-test-support/pom.xml b/functions/common/function-test-support/pom.xml deleted file mode 100644 index 4d5efcbc3..000000000 --- a/functions/common/function-test-support/pom.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - function-test-support - function-test-support - file consumer - - - 2.7.0 - 1.2.0 - - - - - org.springframework.boot - spring-boot-starter-test - compile - - - org.springframework.integration - spring-integration-ftp - true - - - org.springframework - spring-websocket - - - org.apache.ftpserver - ftpserver-core - ${apache-ftpserver.version} - compile - - - org.apache.sshd - sshd-sftp - ${sshd-sftp.version} - compile - - - org.junit.jupiter - junit-jupiter - compile - - - org.testcontainers - testcontainers - ${testcontainers.version} - compile - - - org.testcontainers - junit-jupiter - ${testcontainers.version} - compile - - - org.awaitility - awaitility - compile - - - - diff --git a/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/file/remote/RemoteFileTestSupport.java b/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/file/remote/RemoteFileTestSupport.java deleted file mode 100644 index adf735c6c..000000000 --- a/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/file/remote/RemoteFileTestSupport.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2015-2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.test.support.file.remote; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.nio.file.Path; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.io.TempDir; - -/** - * Abstract base class for tests requiring remote file servers, e.g. (S)FTP. - * - * @author Gary Russell - * - */ -public abstract class RemoteFileTestSupport { - - protected static final int port = 0; - - @TempDir - protected static Path remoteTemporaryFolder; - - @TempDir - protected static Path localTemporaryFolder; - - protected volatile File sourceRemoteDirectory; - - protected volatile File targetRemoteDirectory; - - protected volatile File sourceLocalDirectory; - - protected volatile File targetLocalDirectory; - - public File getSourceRemoteDirectory() { - return sourceRemoteDirectory; - } - - public File getTargetRemoteDirectory() { - return targetRemoteDirectory; - } - - public File getSourceLocalDirectory() { - return sourceLocalDirectory; - } - - public File getTargetLocalDirectory() { - return targetLocalDirectory; - } - - /** - * Default implementation creates the following folder structures: - * - * 

-	 *  $ tree remoteSource/
-	 *  remoteSource/
-	 *  ├── remoteSource1.txt - contains 'source1'
-	 *  ├── remoteSource2.txt - contains 'source2'
-	 *  remoteTarget/
-	 *  $ tree localSource/
-	 *  localSource/
-	 *  ├── localSource1.txt - contains 'local1'
-	 *  ├── localSource2.txt - contains 'local2'
-	 *  localTarget/
-	 *
- * - * The intent is tests retrieve from remoteSource and verify arrival in localTarget or send from localSource and verify - * arrival in remoteTarget. - *

- * Subclasses can change 'remote' in these names by overriding {@link #prefix()} or override this method completely to - * create a different structure. - *

- * While a single server exists for all tests, the directory structure is rebuilt for each test. - * @throws IOException IO Exception. - */ - @BeforeEach - public void setupFolders() throws IOException { - String prefix = prefix(); - recursiveDelete(new File(remoteTemporaryFolder.toFile(), prefix + "Source")); - - sourceRemoteDirectory = new File(remoteTemporaryFolder.toFile(), prefix + "Source"); - sourceRemoteDirectory.mkdirs(); - recursiveDelete(new File(remoteTemporaryFolder.toFile(), prefix + "Target")); - targetRemoteDirectory = new File(remoteTemporaryFolder.toFile(), prefix + "Target"); - targetRemoteDirectory.mkdirs(); - recursiveDelete(new File(localTemporaryFolder.toFile(), "localSource")); - sourceLocalDirectory = new File(localTemporaryFolder.toFile(), "localSource"); - sourceLocalDirectory.mkdirs(); - recursiveDelete(new File(localTemporaryFolder.toFile(), "localTarget")); - targetLocalDirectory = new File(localTemporaryFolder.toFile(), "localTarget"); - targetLocalDirectory.mkdirs(); - File file = new File(sourceRemoteDirectory, prefix + "Source1.txt"); - file.createNewFile(); - FileOutputStream fos = new FileOutputStream(file); - fos.write("source1".getBytes()); - fos.close(); - file = new File(sourceRemoteDirectory, prefix + "Source2.txt"); - file.createNewFile(); - fos = new FileOutputStream(file); - fos.write("source2".getBytes()); - fos.close(); - file = new File(sourceLocalDirectory, "localSource1.txt"); - file.createNewFile(); - fos = new FileOutputStream(file); - fos.write("local1".getBytes()); - fos.close(); - file = new File(sourceLocalDirectory, "localSource2.txt"); - file.createNewFile(); - fos = new FileOutputStream(file); - fos.write("local2".getBytes()); - fos.close(); - } - - public static void recursiveDelete(File file) { - if (file != null && file.exists()) { - File[] files = file.listFiles(); - if (files != null) { - for (File fyle : files) { - if (fyle.isDirectory()) { - recursiveDelete(fyle); - } - else { - fyle.delete(); - } - } - } - file.delete(); - } - } - - /** - * Prefix for directory/file structure; default 'remote'. - * @return the prefix. - */ - protected String prefix() { - return "remote"; - } -} diff --git a/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/ftp/FtpTestSupport.java b/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/ftp/FtpTestSupport.java deleted file mode 100644 index 0788f664e..000000000 --- a/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/ftp/FtpTestSupport.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.test.support.ftp; - -import java.io.File; -import java.util.Arrays; - -import org.apache.ftpserver.FtpServer; -import org.apache.ftpserver.FtpServerFactory; -import org.apache.ftpserver.ftplet.Authentication; -import org.apache.ftpserver.ftplet.AuthenticationFailedException; -import org.apache.ftpserver.ftplet.FtpException; -import org.apache.ftpserver.ftplet.User; -import org.apache.ftpserver.ftplet.UserManager; -import org.apache.ftpserver.listener.ListenerFactory; -import org.apache.ftpserver.usermanager.impl.BaseUser; -import org.apache.ftpserver.usermanager.impl.ConcurrentLoginPermission; -import org.apache.ftpserver.usermanager.impl.TransferRatePermission; -import org.apache.ftpserver.usermanager.impl.WritePermission; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; - -import org.springframework.cloud.fn.test.support.file.remote.RemoteFileTestSupport; - -public class FtpTestSupport extends RemoteFileTestSupport { - - private static final FtpServerFactory serverFactory = new FtpServerFactory(); - - private static volatile FtpServer server; - - public String getTargetLocalDirectoryName() { - return targetLocalDirectory.getAbsolutePath() + File.separator; - } - - @BeforeAll - public static void createServer() throws Exception { - serverFactory.setUserManager(new TestUserManager(remoteTemporaryFolder.toFile().getAbsolutePath())); - - ListenerFactory factory = new ListenerFactory(); - factory.setPort(0); - serverFactory.addListener("default", factory.createListener()); - - server = serverFactory.createServer(); - server.start(); - System.setProperty("ftp.factory.port", String.valueOf(serverFactory.getListener("default").getPort())); - System.setProperty("ftp.localDir", - localTemporaryFolder.toFile().getAbsolutePath() + File.separator + "localTarget"); - } - - @AfterAll - public static void stopServer() throws Exception { - server.stop(); - System.clearProperty("ftp.factory.port"); - System.clearProperty("ftp.localDir"); - } - - @Override - protected String prefix() { - return "ftp"; - } - - private static final class TestUserManager implements UserManager { - - private final BaseUser testUser; - - private TestUserManager(String homeDirectory) { - this.testUser = new BaseUser(); - this.testUser.setAuthorities(Arrays.asList(new ConcurrentLoginPermission(1024, 1024), - new WritePermission(), - new TransferRatePermission(1024, 1024))); - this.testUser.setHomeDirectory(homeDirectory); - this.testUser.setName("TEST_USER"); - } - - - @Override - public User getUserByName(String s) throws FtpException { - return this.testUser; - } - - @Override - public String[] getAllUserNames() throws FtpException { - return new String[] { "TEST_USER" }; - } - - @Override - public void delete(String s) throws FtpException { - } - - @Override - public void save(User user) throws FtpException { - } - - @Override - public boolean doesExist(String s) throws FtpException { - return true; - } - - @Override - public User authenticate(Authentication authentication) throws AuthenticationFailedException { - return this.testUser; - } - - @Override - public String getAdminName() throws FtpException { - return "admin"; - } - - @Override - public boolean isAdmin(String s) throws FtpException { - return s.equals("admin"); - } - - } -} diff --git a/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/sftp/SftpTestSupport.java b/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/sftp/SftpTestSupport.java deleted file mode 100644 index ead7a8924..000000000 --- a/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/sftp/SftpTestSupport.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.test.support.sftp; - -import java.io.File; -import java.io.InputStream; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.security.KeyFactory; -import java.security.PublicKey; -import java.security.spec.RSAPublicKeySpec; -import java.util.Arrays; -import java.util.Collections; - -import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory; -import org.apache.sshd.server.SshServer; -import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider; -import org.apache.sshd.sftp.server.SftpSubsystemFactory; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; - -import org.springframework.cloud.fn.test.support.file.remote.RemoteFileTestSupport; -import org.springframework.core.io.ClassPathResource; -import org.springframework.util.Base64Utils; -import org.springframework.util.FileCopyUtils; -import org.springframework.util.StringUtils; - -/** - * Provides an embedded SFTP Server for test cases. - * - * @author David Turanski - * @author Gary Russell - * @author Artem Bilan - */ -public class SftpTestSupport extends RemoteFileTestSupport { - - private static SshServer server; - - @Override - public String prefix() { - return "sftp"; - } - - @BeforeAll - public static void createServer() throws Exception { - server = SshServer.setUpDefaultServer(); - server.setPasswordAuthenticator((username, password, session) -> - StringUtils.hasText(password) && !"badPassword".equals(password)); // fail if pub key validation failed - server.setPublickeyAuthenticator((username, key, session) -> key.equals(decodePublicKey("id_rsa_pp.pub"))); - server.setPort(0); - server.setKeyPairProvider(new SimpleGeneratorHostKeyProvider(new File("hostkey.ser").toPath())); - server.setSubsystemFactories(Collections.singletonList(new SftpSubsystemFactory())); - server.setFileSystemFactory(new VirtualFileSystemFactory(remoteTemporaryFolder)); - server.start(); - System.setProperty("sftp.factory.port", String.valueOf(server.getPort())); - System.setProperty("sftp.consumer.localDir", - localTemporaryFolder + File.separator + "localTarget"); - } - - @AfterAll - public static void stopServer() throws Exception { - server.stop(); - File hostkey = new File("hostkey.ser"); - if (hostkey.exists()) { - hostkey.delete(); - } - System.clearProperty("sftp.factory.port"); - System.clearProperty("sftp.consumer.localDir"); - } - - private static PublicKey decodePublicKey(String key) { - try { - InputStream stream = new ClassPathResource(key).getInputStream(); - byte[] keyBytes = FileCopyUtils.copyToByteArray(stream); - // strip any newline chars - while (keyBytes[keyBytes.length - 1] == 0x0a || keyBytes[keyBytes.length - 1] == 0x0d) { - keyBytes = Arrays.copyOf(keyBytes, keyBytes.length - 1); - } - byte[] decodeBuffer = Base64Utils.decode(keyBytes); - ByteBuffer bb = ByteBuffer.wrap(decodeBuffer); - int len = bb.getInt(); - byte[] type = new byte[len]; - bb.get(type); - if ("ssh-rsa".equals(new String(type))) { - BigInteger e = decodeBigInt(bb); - BigInteger m = decodeBigInt(bb); - RSAPublicKeySpec spec = new RSAPublicKeySpec(m, e); - return KeyFactory.getInstance("RSA").generatePublic(spec); - - } - else { - throw new IllegalArgumentException("Only supports RSA"); - } - } - catch (Exception e) { - throw new IllegalStateException("Failed to determine the test public key", e); - } - } - - private static BigInteger decodeBigInt(ByteBuffer bb) { - int len = bb.getInt(); - byte[] bytes = new byte[len]; - bb.get(bytes); - return new BigInteger(bytes); - } -} diff --git a/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/websocket/WebsocketConsumerClientHandler.java b/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/websocket/WebsocketConsumerClientHandler.java deleted file mode 100644 index 8a857d0fb..000000000 --- a/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/websocket/WebsocketConsumerClientHandler.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2014-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.test.support.websocket; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.springframework.web.socket.TextMessage; -import org.springframework.web.socket.WebSocketSession; -import org.springframework.web.socket.handler.AbstractWebSocketHandler; - -public class WebsocketConsumerClientHandler extends AbstractWebSocketHandler { - - final List receivedMessages = new ArrayList<>(); - - final int waitMessageCount; - - final CountDownLatch latch; - - final long timeout; - - final String id; - - public WebsocketConsumerClientHandler(String id, int waitMessageCount, long timeout) { - this.id = id; - this.waitMessageCount = waitMessageCount; - this.latch = new CountDownLatch(waitMessageCount); - this.timeout = timeout; - } - - @Override - public void handleTextMessage(WebSocketSession session, TextMessage message) { - receivedMessages.add(message.getPayload()); - latch.countDown(); - } - - public void await() throws InterruptedException { - latch.await(timeout, TimeUnit.MILLISECONDS); - } - - public List getReceivedMessages() { - return receivedMessages; - } -} diff --git a/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/xmpp/XmppTestContainerSupport.java b/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/xmpp/XmppTestContainerSupport.java deleted file mode 100644 index e426765f8..000000000 --- a/functions/common/function-test-support/src/main/java/org/springframework/cloud/fn/test/support/xmpp/XmppTestContainerSupport.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2014-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.test.support.xmpp; - -import java.time.Duration; - -import org.junit.jupiter.api.BeforeAll; -import org.testcontainers.containers.BindMode; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * @author Chris Bono - */ -@Testcontainers(disabledWithoutDocker = true) -public interface XmppTestContainerSupport { - - /** - * Default XMPP Host. - */ - String XMPP_HOST = "localhost"; - - /** - * Sample John User setup by config. - */ - String JOHN_USER = "john"; - - /** - * Sample Jane User setup by config. - */ - String JANE_USER = "jane"; - - /** - * Password for sample users. - */ - String USER_PW = "secret"; - - /** - * Default Service Name. - */ - String SERVICE_NAME = "localhost"; - - /** - * The container. - */ - GenericContainer XMPP_CONTAINER = new GenericContainer<>("fishbowler/openfire:v4.7.0") - .withExposedPorts(5222) - .withClasspathResourceMapping("xmpp/conf", "/var/lib/openfire/conf", BindMode.READ_ONLY) - .withCommand("-demoboot") - .withStartupTimeout(Duration.ofSeconds(120)) - .withStartupAttempts(3); - - @BeforeAll - static void startContainer() { - XMPP_CONTAINER.start(); - } - - static String getXmppHost() { - return XMPP_HOST; - } - - static Integer getXmppMappedPort() { - return XMPP_CONTAINER.getFirstMappedPort(); - } - -} diff --git a/functions/common/function-test-support/src/main/resources/id_rsa_pp b/functions/common/function-test-support/src/main/resources/id_rsa_pp deleted file mode 100644 index 7c7cba165..000000000 --- a/functions/common/function-test-support/src/main/resources/id_rsa_pp +++ /dev/null @@ -1,30 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-128-CBC,26D1755B05980BA01B1E8D3B65EF98D3 - -J5fMQDf0HrwcnfYujq+q05GEOSEMVWMU0vr0hBtz2WvUeaFBBVAvUWbJo7PHTdDV -vEdSv+k8FazOkIZpeOW26pKNaLSuFfN+lgtAW4p4yqGQhbL8byh+Ka5uGaPH1xQj -o4exMFqbwFaHG10LJoPp5NnK+T64w8McJEihBPxv/qwtsz0YhhDVl/1eSwKsGa5y -Usfdv0QbjNDtpV7+sy7OunpaaKjb8iQ/PbFDsX0TSiy6jJflPwCYVUoh3jCEJze3 -OUKwoQu7AiHmUrsnLtDuL49Q5hV+f9+IPJlzSqU5Fu8PlfCowH+e8WWLVQkh7Hht -iwOQs5UIWH+Nzbguu3Gbph5lqMtgkQwK6/PSFLQXuJmf9l6eo7E+Wh5mFWwoIENT -mKUlgVU0ymajPHcA7uOVXl5XX+Mt0DiMdGiB4N6RJ1OjWgqlRSJbVmGPrnZWmjdW -VltNux6JrO1bOgwtApCouDZhnP7/JDhM2PCa/F2+XCc6s8Hnmmt+gBw6IM4fb+YH -2gf1jx5Onp1yuBGe6tvbrfPAdXYU1R3kd2+Yf5mdsu2+xiC4wBKSfjleoHN4xJ/b -wopjQvGV5dj/VjdUt99lMHGWPr+p4NmdEBoriCgjYuRjRDvCiTV2qKZe2MU+0CVE -/jCA8iMRQnx3LMjnmJXP/96j2fyXHMowClTF43Cvdc5jh622WITOOFAIq9RfIXdS -47G6FAN54V+Qt0pXEgIOmvG+B2C24A041fo3jUPZxFRSYYuv9vG6QJby3kCsaqqt -ngYs2JSKx9CfOfGfyPbNt2/CO/bBsXgYzLR7REx5My1Mp9YsDmeIgbcd2V5hw/Me -rlSXrG4Eqs9gRDrBUvsydUOJFC1PlnIXH3VSBc6X9o4n2H6XECOsJQSeXLCaeMav -deKb0r1HvdbAYrdqw6mRM8Ok3fpSoD3mUsZQ8fp3luO3tHL1lddxHb7EsKzt/ubh -B9lEzDTcILlINlCl41X0OZKr/c+Ec8EdaSvITYJj0fvaZmDF7Wcs/dDWNft2XUaD -VcptOKbQVE1ufbrc2s1BdKOriAC6dSVKVDrAUQD/MlhT3p/YwbjSY2MqwRKztWRv -sA1Kqjg1IdUQzQizRKuHa322qnduLjHy0rb0ElrMpFe3B+OcIPs1E7Gvo4BVQ5jU -5GqHm83iaFIQaXmEsrVtCOBygVf00+WNRV3WFTOP9UEWFgtxsaHAU8UZhsDbKcZ5 -l/w6kQdNElQuJA+1n1OxRZ5wIpfrRIMaxBQg2plUTVb9Tgz1qTZxHIEYPAAePfvX -tDn9SBiktj/8dhy1T0ko89yXCekpGxkU9rbmAd4Lrp6Wbc6+bKt4KzAqw319qq9U -Pslq7EKNM0Zq5pfwn1MRjzvmyHxz2sYHeij0CmuZY7xuOa4NzXt1vr6nSlIpX77W -ng/Rnd7qpyG2IWYihi6ztFHyj9h3FEcBHMk4JINLcdYOxXICz3KsRMfntrwQ4E6O -NFJ3fPpVNkk4GZcxy6idNkUBz1s8ixVWC8yi36byxE+TTRtcvqXQnJVgs63vMUlC -HVwaGau4YeUp4Nj2ZO44Srd/kQRy8yuCOPDdlJEiO6eDD4+XKedJQg1LuGeoXMVn ------END RSA PRIVATE KEY----- diff --git a/functions/common/function-test-support/src/main/resources/id_rsa_pp.pub b/functions/common/function-test-support/src/main/resources/id_rsa_pp.pub deleted file mode 100644 index 50cf189af..000000000 --- a/functions/common/function-test-support/src/main/resources/id_rsa_pp.pub +++ /dev/null @@ -1 +0,0 @@ -AAAAB3NzaC1yc2EAAAADAQABAAABAQC6MIzgyVi8G1+HRhFHPWRH+3w/8/uxtiuIfb4puVPjHI53Lvf5odzfhv0T6Z2/jSXmI3I6dpjbsgiptdCTX4kqUFLXxkuJR4LHatNtgO1w32aVIdAvfj7KtrL3SmP2XWqQGVcUWHEn2H1RHFHKdC6ArYFb1X8p5N/BHSQjuttaeVi9FsDxvC5euIbtDEEJmmvjjfWlI1m/6qCqMYxDWA9i9APU/rB0QwFNUQ6HuZ2QzEaU/hQMGmqgW5o1I/W8JR0bqis8wZQDLv1fwCkXpWG5BAuiJH+FJMxRAkfEMBpVwO7Sl0ufePVuSM2BMAAe+4a75sVp8ahbOId6y0GUTeJl diff --git a/functions/common/function-test-support/src/main/resources/logback-test.xml b/functions/common/function-test-support/src/main/resources/logback-test.xml deleted file mode 100644 index 554cd1bd9..000000000 --- a/functions/common/function-test-support/src/main/resources/logback-test.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n - - - - - - - - diff --git a/functions/common/function-test-support/src/main/resources/xmpp/conf/available-plugins.xml b/functions/common/function-test-support/src/main/resources/xmpp/conf/available-plugins.xml deleted file mode 100755 index b070cec25..000000000 --- a/functions/common/function-test-support/src/main/resources/xmpp/conf/available-plugins.xml +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/functions/common/function-test-support/src/main/resources/xmpp/conf/crowd.properties b/functions/common/function-test-support/src/main/resources/xmpp/conf/crowd.properties deleted file mode 100755 index d43a8ba45..000000000 --- a/functions/common/function-test-support/src/main/resources/xmpp/conf/crowd.properties +++ /dev/null @@ -1,42 +0,0 @@ -# -# This file defines the configuration properties required -# when using the Atlassian Crowd integration for Openfire. -# -# https://confluence.atlassian.com/display/CROWD/The+crowd.properties+file -# -# To activate the Crowd integration for Openfire, you must define -# the following Openfire system properties: -# -# provider.admin.className org.jivesoftware.openfire.crowd.CrowdAdminProvider -# provider.auth.className org.jivesoftware.openfire.crowd.CrowdAuthProvider -# provider.group.className org.jivesoftware.openfire.crowd.CrowdGroupProvider -# provider.user.className org.jivesoftware.openfire.crowd.CrowdUserProvider -# provider.vcard.className org.jivesoftware.openfire.crowd.CrowdVCardProvider -# -# In addition, you may customize the Crowd provider using the following Openfire -# system properties: -# -# admin.authorizedGroups -# crowd.groups.cache.ttl.seconds 3600 -# crowd.users.cache.ttl.seconds 3600 -# - -# The REST URL for your Crowd server. -crowd.server.url=https://YOUR-CROWD-SERVER:8095/crowd/ - -# These properties are required to authenticate with the Crowd server. -# They must match the values specified in the Crowd configuration. -application.name=openfire -application.password= - -# Other optional configuration properties. - -#http.proxy.host= -#http.proxy.port= -#http.proxy.username= -#http.proxy.password= - -# These properties can be used to tune the Crowd integration. -#http.max.connections=20 -#http.timeout=5000 -#http.socket.timeout=20000 diff --git a/functions/common/function-test-support/src/main/resources/xmpp/conf/openfire-demoboot.xml b/functions/common/function-test-support/src/main/resources/xmpp/conf/openfire-demoboot.xml deleted file mode 100755 index 8855c30b7..000000000 --- a/functions/common/function-test-support/src/main/resources/xmpp/conf/openfire-demoboot.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - 9090 - 9091 - - - org.jivesoftware.database.EmbeddedConnectionProvider - - - true - en - - - true - - localhost - localhost - - - embedded - - - admin@example.com - admin - - - - john - secret - John Doe - john.doe@example.com - - - jane@localhost - Jane - - - - - jane - secret - Jane Doe - jane.doe@example.com - - - john@localhost - John - - - - - - diff --git a/functions/common/function-test-support/src/main/resources/xmpp/conf/openfire.xml b/functions/common/function-test-support/src/main/resources/xmpp/conf/openfire.xml deleted file mode 100644 index 2c9f7e3dc..000000000 --- a/functions/common/function-test-support/src/main/resources/xmpp/conf/openfire.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - 9090 - 9091 - - - org.jivesoftware.database.DefaultConnectionProvider - - true - en - localhost - diff --git a/functions/common/function-test-support/src/main/resources/xmpp/conf/security.xml b/functions/common/function-test-support/src/main/resources/xmpp/conf/security.xml deleted file mode 100644 index 418502161..000000000 --- a/functions/common/function-test-support/src/main/resources/xmpp/conf/security.xml +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - - Blowfish - - - - - - - database.defaultProvider.username - database.defaultProvider.password - - - - diff --git a/functions/common/function-test-support/src/main/resources/xmpp/conf/security/archive/readme.txt b/functions/common/function-test-support/src/main/resources/xmpp/conf/security/archive/readme.txt deleted file mode 100755 index 62573a55e..000000000 --- a/functions/common/function-test-support/src/main/resources/xmpp/conf/security/archive/readme.txt +++ /dev/null @@ -1 +0,0 @@ -This directory is used as a default location in which Openfire stores backups of keystore files. diff --git a/functions/common/function-test-support/src/main/resources/xmpp/conf/security/client.truststore b/functions/common/function-test-support/src/main/resources/xmpp/conf/security/client.truststore deleted file mode 100755 index c40846550..000000000 Binary files a/functions/common/function-test-support/src/main/resources/xmpp/conf/security/client.truststore and /dev/null differ diff --git a/functions/common/function-test-support/src/main/resources/xmpp/conf/security/keystore b/functions/common/function-test-support/src/main/resources/xmpp/conf/security/keystore deleted file mode 100755 index 1405b0f0a..000000000 Binary files a/functions/common/function-test-support/src/main/resources/xmpp/conf/security/keystore and /dev/null differ diff --git a/functions/common/function-test-support/src/main/resources/xmpp/conf/security/truststore b/functions/common/function-test-support/src/main/resources/xmpp/conf/security/truststore deleted file mode 100755 index cc04a6534..000000000 Binary files a/functions/common/function-test-support/src/main/resources/xmpp/conf/security/truststore and /dev/null differ diff --git a/functions/common/function-test-support/src/main/resources/xmpp/conf/server-update.xml b/functions/common/function-test-support/src/main/resources/xmpp/conf/server-update.xml deleted file mode 100755 index c8b656346..000000000 --- a/functions/common/function-test-support/src/main/resources/xmpp/conf/server-update.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/functions/common/metadata-store-common/README.adoc b/functions/common/metadata-store-common/README.adoc deleted file mode 100644 index 249ae33e6..000000000 --- a/functions/common/metadata-store-common/README.adoc +++ /dev/null @@ -1,139 +0,0 @@ -=== `MetadataStore` Common Module - -This artifact contains a Spring Boot auto-configuration for the `MetadataStore`which can be used in various Spring Integration scenarios, like file polling, idempotent receiver, offset management etc. -See Spring Integration "`https://docs.spring.io/spring-integration/docs/5.0.6.RELEASE/reference/html/system-management-chapter.html#metadata-store[Reference Manual]`" for more information. - -In addition to the standard Spring Boot configuration properties this module exposes a `MetadataStoreProperties` with the `metadata.store` prefix. - -To auto-configure particular `MetadataStore` you need to set `metadata.store.type` and include the respective dependencies into the target app starter: - -==== Redis - -The `RedisMetadataStore` requires regular Spring Boot auto-configuration for https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#boot-features-redis[Spring Data Redis] and minimal set of dependencies is like this: - -[source,xml] ----- - - org.springframework.integration - spring-integration-redis - - - org.springframework.boot - spring-boot-starter-data-redis - ----- - -Additional configuration property for `RedisMetadataStore` is: - -$$metadata.store.redis.key$$:: $$Redis key for metadata.$$ *($$String$$, default: `$$MetaData$$`)* - -==== MongoDb - -The `MongoDbMetadataStore` requires regular Spring Boot auto-configuration for https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#boot-features-mongodb[Spring Data MongoDB] and minimal set of dependencies is like this: - -[source,xml] ----- - - org.springframework.integration - spring-integration-mongodb - - - org.springframework.boot - spring-boot-starter-data-mongodb - ----- - -Additional configuration property for `MongoDbMetadataStore` is: - -$$metadata.store.mongo-db.collection$$:: $$MongoDB collection name for metadata.$$ *($$String$$, default: `$$metadataStore$$`)* - -==== Hazelcast - -The `HazelcastMetadataStore` requires regular Spring Boot auto-configuration for https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#boot-features-caching-provider-hazelcast[Hazelcast] and minimal set of dependencies is like this: - -[source,xml] ----- - - org.springframework.integration - spring-integration-hazelcast - ----- - -There are no additional configuration properties for the `HazelcastMetadataStore`, however a `MetadataStoreListener` bean can be configured in the application context to react to the `MetadataStore` events. - -==== Zookeeper - -The `ZookeeperMetadataStore` requires this dependency for auto-configuration: - -[source,xml] ----- - - org.springframework.integration - spring-integration-zookeeper - ----- - -The configuration properties for `ZookeeperMetadataStore` are: - -$$metadata.store.zookeeper.connect-string$$:: $$Zookeeper connect string in form HOST:PORT.$$ *($$String$$, default: `$$127.0.0.1:2181$$`)* -$$metadata.store.zookeeper.retry-interval$$:: $$Retry interval for Zookeeper operations in milliseconds.$$ *($$int$$, default: `$$1000$$`)* -$$metadata.store.zookeeper.encoding$$:: $$Encoding to use when storing data in Zookeeper.$$ *($$Charset$$, default: `$$UTF-8$$`)* -$$metadata.store.zookeeper.root$$:: $$Root node - store entries are children of this node.$$ *($$String$$, default: `$$/SpringIntegration-MetadataStore$$`)* - -In addition, for the `ZookeeperMetadataStore`, a `MetadataStoreListener` bean can be configured in the application context to react to the `MetadataStore` events. -Also a `CuratorFramework` bean can be provided to override a default auto-configured one. - -==== AWS DymanoDb - -The `DynamoDbMetadataStore` requires regular Spring Cloud AWS auto-configuration for https://cloud.spring.io/spring-cloud-static/spring-cloud-aws/2.0.0.RELEASE/single/spring-cloud-aws.html#_spring_boot_auto_configuration[Spring Boot] and minimal set of dependencies is like this: - -[source,xml] ----- - - org.springframework.integration - spring-integration-aws - - - com.amazonaws - aws-java-sdk-dynamodb - ----- - -Additional configuration properties for `DynamoDbMetadataStore` are: - -$$metadata.store.dynamo-db.table:: $$Table name for metadata.$$ *($$String$$, default: `$$SpringIntegrationMetadataStore$$`)* -$$metadata.store.dynamo-db.read-capacity:: $$Read capacity on the table.$$ *($$long$$, default: `$$1$$`)* -$$metadata.store.dynamo-db.write-capacity:: $$Write capacity on the table.$$ *($$long$$, default: `$$1$$`)* -$$metadata.store.dynamo-db.create-delay:: $$Delay between create table retries.$$ *($$int$$, default: `$$1$$`)* -$$metadata.store.dynamo-db.create-retries:: $$Retry number for create table request.$$ *($$int$$, default: `$$25$$`)* -$$metadata.store.dynamo-db.time-to-live:: $$TTL for table entries.$$ *($$Integer$$, default: `$$$$`)* - -A default, auto-configured `AmazonDynamoDBAsync` bean can be overridden in the target application. - -==== JDBC - -The `JdbcMetadataStore` requires regular Spring Boot auto-configuration for https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#boot-features-sql[JDBC DataSource] and minimal set of dependencies is like this: - -[source,xml] ----- - - org.springframework.integration - spring-integration-jdbc - - - org.springframework.boot - spring-boot-starter-jdbc - ----- - -Plus vendor-specific JDBC driver artifact(s). - -Additional configuration properties for `JdbcMetadataStore` are: - -$$metadata.store.jdbc.table-prefix:: $$Prefix for the custom table name.$$ *($$String$$, default: `$$INT_$$`)* -$$metadata.store.jdbc.region:: $$Unique grouping identifier for messages persisted with this store.$$ *($$String$$, default: `$$DEFAULT$$`)* - - - -When no any of those technologies dependencies are preset, an in-memory `SimpleMetadataStore` is auto-configured. -The target application can also provide its own `MetadataStore` bean to override any auto-configuration hooks. diff --git a/functions/common/metadata-store-common/pom.xml b/functions/common/metadata-store-common/pom.xml deleted file mode 100644 index 34aae63db..000000000 --- a/functions/common/metadata-store-common/pom.xml +++ /dev/null @@ -1,112 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - metadata-store-common - metadata-store-common - metadata-store common - - - - - - org.springframework.integration - spring-integration-redis - true - - - org.springframework.boot - spring-boot-starter-data-redis - true - - - - - org.springframework.integration - spring-integration-mongodb - true - - - org.springframework.boot - spring-boot-starter-data-mongodb - true - - - - io.awspring.cloud - spring-cloud-aws-starter - true - - - - org.springframework.boot - spring-boot-starter-logging - true - - - org.apache.logging.log4j - log4j-to-slf4j - - - - - - - org.springframework.integration - spring-integration-jdbc - true - - - org.springframework.boot - spring-boot-starter-jdbc - true - - - org.hsqldb - hsqldb - test - - - - - org.springframework.integration - spring-integration-zookeeper - true - - - org.apache.curator - curator-test - ${curator.version} - test - - - - - org.springframework.integration - spring-integration-hazelcast - true - - - - - org.springframework.integration - spring-integration-aws - ${spring-integration-aws.version} - true - - - software.amazon.awssdk - dynamodb - true - - - - - diff --git a/functions/common/metadata-store-common/src/main/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreAutoConfiguration.java b/functions/common/metadata-store-common/src/main/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreAutoConfiguration.java deleted file mode 100644 index fa2b9fa68..000000000 --- a/functions/common/metadata-store-common/src/main/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreAutoConfiguration.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright 2018-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.metadata.store; - -import com.hazelcast.core.HazelcastInstance; -import io.awspring.cloud.autoconfigure.core.AwsClientBuilderConfigurer; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.RetryForever; -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; - -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.integration.aws.metadata.DynamoDbMetadataStore; -import org.springframework.integration.hazelcast.metadata.HazelcastMetadataStore; -import org.springframework.integration.jdbc.metadata.JdbcMetadataStore; -import org.springframework.integration.metadata.ConcurrentMetadataStore; -import org.springframework.integration.metadata.MetadataStoreListener; -import org.springframework.integration.metadata.SimpleMetadataStore; -import org.springframework.integration.mongodb.metadata.MongoDbMetadataStore; -import org.springframework.integration.redis.metadata.RedisMetadataStore; -import org.springframework.integration.zookeeper.metadata.ZookeeperMetadataStore; -import org.springframework.jdbc.core.JdbcTemplate; - -/** - * @author Artem Bilan - * @author David Turanski - * @author Corneil du Plessis - * - * @since 2.0.2 - */ -@AutoConfiguration -@ConditionalOnClass(ConcurrentMetadataStore.class) -@EnableConfigurationProperties(MetadataStoreProperties.class) -public class MetadataStoreAutoConfiguration { - - @Bean - @ConditionalOnProperty(prefix = "metadata.store", name = "type", havingValue = "memory", matchIfMissing = true) - @ConditionalOnMissingBean - public ConcurrentMetadataStore simpleMetadataStore() { - return new SimpleMetadataStore(); - } - - @ConditionalOnProperty(prefix = "metadata.store", name = "type", havingValue = "redis") - static class Redis { - - @Bean - @ConditionalOnMissingBean - public ConcurrentMetadataStore redisMetadataStore(RedisTemplate redisTemplate, - MetadataStoreProperties metadataStoreProperties) { - - return new RedisMetadataStore(redisTemplate, metadataStoreProperties.getRedis().getKey()); - } - - } - - @ConditionalOnProperty(prefix = "metadata.store", name = "type", havingValue = "mongodb") - static class Mongo { - - @Bean - @ConditionalOnMissingBean - public ConcurrentMetadataStore mongoDbMetadataStore(MongoTemplate mongoTemplate, - MetadataStoreProperties metadataStoreProperties) { - - return new MongoDbMetadataStore(mongoTemplate, metadataStoreProperties.getMongoDb().getCollection()); - } - - } - - @ConditionalOnProperty(prefix = "metadata.store", name = "type", havingValue = "hazelcast") - static class Hazelcast { - - @Bean - @ConditionalOnMissingBean - public HazelcastInstance hazelcastInstance() { - return com.hazelcast.core.Hazelcast.newHazelcastInstance(); - } - - @Bean - @ConditionalOnMissingBean - public ConcurrentMetadataStore hazelcastMetadataStore(HazelcastInstance hazelcastInstance, - ObjectProvider metadataStoreListenerObjectProvider) { - - HazelcastMetadataStore hazelcastMetadataStore = new HazelcastMetadataStore(hazelcastInstance); - metadataStoreListenerObjectProvider.ifAvailable(hazelcastMetadataStore::addListener); - return hazelcastMetadataStore; - } - - } - - @ConditionalOnProperty(prefix = "metadata.store", name = "type", havingValue = "zookeeper") - static class Zookeeper { - - @Bean(initMethod = "start") - @ConditionalOnMissingBean - public CuratorFramework curatorFramework(MetadataStoreProperties metadataStoreProperties) { - MetadataStoreProperties.Zookeeper zookeeperProperties = metadataStoreProperties.getZookeeper(); - return CuratorFrameworkFactory.newClient(zookeeperProperties.getConnectString(), - new RetryForever(zookeeperProperties.getRetryInterval())); - } - - @Bean - @ConditionalOnMissingBean - public ConcurrentMetadataStore zookeeperMetadataStore(CuratorFramework curatorFramework, - MetadataStoreProperties metadataStoreProperties, - ObjectProvider metadataStoreListenerObjectProvider) { - - MetadataStoreProperties.Zookeeper zookeeperProperties = metadataStoreProperties.getZookeeper(); - ZookeeperMetadataStore zookeeperMetadataStore = new ZookeeperMetadataStore(curatorFramework); - zookeeperMetadataStore.setEncoding(zookeeperProperties.getEncoding().name()); - zookeeperMetadataStore.setRoot(zookeeperProperties.getRoot()); - metadataStoreListenerObjectProvider.ifAvailable(zookeeperMetadataStore::addListener); - return zookeeperMetadataStore; - } - - } - - @ConditionalOnProperty(prefix = "metadata.store", name = "type", havingValue = "dynamodb") - static class DynamoDb { - - @Bean - @ConditionalOnMissingBean - public DynamoDbAsyncClient dynamoDB(AwsClientBuilderConfigurer awsClientBuilderConfigurer) { - return awsClientBuilderConfigurer.configure(DynamoDbAsyncClient.builder()).build(); - } - - @Bean - @ConditionalOnMissingBean - public ConcurrentMetadataStore dynamoDbMetadataStore(DynamoDbAsyncClient dynamoDB, - MetadataStoreProperties metadataStoreProperties) { - - MetadataStoreProperties.DynamoDb dynamoDbProperties = metadataStoreProperties.getDynamoDb(); - - DynamoDbMetadataStore dynamoDbMetadataStore = new DynamoDbMetadataStore(dynamoDB, - dynamoDbProperties.getTable()); - - dynamoDbMetadataStore.setReadCapacity(dynamoDbProperties.getReadCapacity()); - dynamoDbMetadataStore.setWriteCapacity(dynamoDbProperties.getWriteCapacity()); - dynamoDbMetadataStore.setCreateTableDelay(dynamoDbProperties.getCreateDelay()); - dynamoDbMetadataStore.setCreateTableRetries(dynamoDbProperties.getCreateRetries()); - if (dynamoDbProperties.getTimeToLive() != null) { - dynamoDbMetadataStore.setTimeToLive(dynamoDbProperties.getTimeToLive()); - } - - return dynamoDbMetadataStore; - } - - } - - @ConditionalOnProperty(prefix = "metadata.store", name = "type", havingValue = "jdbc") - static class Jdbc { - @Bean - @ConditionalOnMissingBean - public ConcurrentMetadataStore jdbcMetadataStore(JdbcTemplate jdbcTemplate, - MetadataStoreProperties metadataStoreProperties) { - - MetadataStoreProperties.Jdbc jdbcProperties = metadataStoreProperties.getJdbc(); - - JdbcMetadataStore jdbcMetadataStore = new JdbcMetadataStore(jdbcTemplate); - jdbcMetadataStore.setTablePrefix(jdbcProperties.getTablePrefix()); - jdbcMetadataStore.setRegion(jdbcProperties.getRegion()); - - return jdbcMetadataStore; - } - - } - -} diff --git a/functions/common/metadata-store-common/src/main/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreProperties.java b/functions/common/metadata-store-common/src/main/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreProperties.java deleted file mode 100644 index 85b0c4d93..000000000 --- a/functions/common/metadata-store-common/src/main/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreProperties.java +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Copyright 2018-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.metadata.store; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.integration.aws.metadata.DynamoDbMetadataStore; -import org.springframework.integration.jdbc.metadata.JdbcMetadataStore; -import org.springframework.integration.redis.metadata.RedisMetadataStore; - -/** - * @author Artem Bilan - * @author David Turanski - * @author Corneil du Plessis - * @since 2.0.2 - */ -@ConfigurationProperties("metadata.store") -public class MetadataStoreProperties { - enum StoreType { - mongodb, - redis, - dynamodb, - jdbc, - zookeeper, - hazelcast, - memory - } - - /** - * Indicates the type of metadata store to configure (default is 'memory'). - * You must include the corresponding Spring Integration dependency to use a persistent store. - */ - private StoreType type = StoreType.memory; - - private final Mongo mongoDb = new Mongo(); - - private final Redis redis = new Redis(); - - private final DynamoDb dynamoDb = new DynamoDb(); - - private final Jdbc jdbc = new Jdbc(); - - private final Zookeeper zookeeper = new Zookeeper(); - - public StoreType getType() { - return this.type; - } - - public void setType(StoreType type) { - this.type = type; - } - - public Mongo getMongoDb() { - return this.mongoDb; - } - - public Redis getRedis() { - return this.redis; - } - - public DynamoDb getDynamoDb() { - return this.dynamoDb; - } - - public Jdbc getJdbc() { - return this.jdbc; - } - - public Zookeeper getZookeeper() { - return this.zookeeper; - } - - public static class Mongo { - - /** - * MongoDB collection name for metadata. - */ - private String collection = "metadataStore"; - - public String getCollection() { - return this.collection; - } - - public void setCollection(String collection) { - this.collection = collection; - } - - } - - public static class Redis { - - /** - * Redis key for metadata. - */ - private String key = RedisMetadataStore.KEY; - - public String getKey() { - return this.key; - } - - public void setKey(String key) { - this.key = key; - } - - } - - public static class DynamoDb { - - /** - * Table name for metadata. - */ - private String table = DynamoDbMetadataStore.DEFAULT_TABLE_NAME; - - /** - * Read capacity on the table. - */ - private long readCapacity = 1L; - - /** - * Write capacity on the table. - */ - private long writeCapacity = 1L; - - /** - * Delay between create table retries. - */ - private int createDelay = 1; - - /** - * Retry number for create table request. - */ - private int createRetries = 25; - - /** - * TTL for table entries. - */ - private Integer timeToLive; - - public String getTable() { - return this.table; - } - - public void setTable(String table) { - this.table = table; - } - - public long getReadCapacity() { - return this.readCapacity; - } - - public void setReadCapacity(long readCapacity) { - this.readCapacity = readCapacity; - } - - public long getWriteCapacity() { - return this.writeCapacity; - } - - public void setWriteCapacity(long writeCapacity) { - this.writeCapacity = writeCapacity; - } - - public int getCreateDelay() { - return this.createDelay; - } - - public void setCreateDelay(int createDelay) { - this.createDelay = createDelay; - } - - public int getCreateRetries() { - return this.createRetries; - } - - public void setCreateRetries(int createRetries) { - this.createRetries = createRetries; - } - - public Integer getTimeToLive() { - return this.timeToLive; - } - - public void setTimeToLive(Integer timeToLive) { - this.timeToLive = timeToLive; - } - - } - - public static class Jdbc { - - /** - * Prefix for the custom table name. - */ - private String tablePrefix = JdbcMetadataStore.DEFAULT_TABLE_PREFIX; - - /** - * Unique grouping identifier for messages persisted with this store. - */ - private String region = "DEFAULT"; - - public String getTablePrefix() { - return this.tablePrefix; - } - - public void setTablePrefix(String tablePrefix) { - this.tablePrefix = tablePrefix; - } - - public String getRegion() { - return this.region; - } - - public void setRegion(String region) { - this.region = region; - } - - } - - public static class Zookeeper { - - /** - * Zookeeper connect string in form HOST:PORT. - */ - private String connectString = "127.0.0.1:2181"; - - /** - * Retry interval for Zookeeper operations in milliseconds. - */ - private int retryInterval = 1000; - - /** - * Encoding to use when storing data in Zookeeper. - */ - private Charset encoding = StandardCharsets.UTF_8; - - /** - * Root node - store entries are children of this node. - */ - private String root = "/SpringIntegration-MetadataStore"; - - public String getConnectString() { - return connectString; - } - - public void setConnectString(String connectString) { - this.connectString = connectString; - } - - public int getRetryInterval() { - return this.retryInterval; - } - - public void setRetryInterval(int retryInterval) { - this.retryInterval = retryInterval; - } - - public Charset getEncoding() { - return this.encoding; - } - - public void setEncoding(Charset encoding) { - this.encoding = encoding; - } - - public String getRoot() { - return this.root; - } - - public void setRoot(String root) { - this.root = root; - } - - } - -} diff --git a/functions/common/metadata-store-common/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/common/metadata-store-common/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index 77a75edd2..000000000 --- a/functions/common/metadata-store-common/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.common.metadata.store.MetadataStoreAutoConfiguration diff --git a/functions/common/metadata-store-common/src/test/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreAutoConfigurationTests.java b/functions/common/metadata-store-common/src/test/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreAutoConfigurationTests.java deleted file mode 100644 index 107bdde34..000000000 --- a/functions/common/metadata-store-common/src/test/java/org/springframework/cloud/fn/common/metadata/store/MetadataStoreAutoConfigurationTests.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2018-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.metadata.store; - -import java.beans.Introspector; -import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.function.Consumer; -import java.util.function.Predicate; - -import org.apache.curator.test.TestingServer; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.ArgumentMatchers; -import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; -import software.amazon.awssdk.regions.providers.AwsRegionProvider; -import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient; -import software.amazon.awssdk.services.dynamodb.model.DescribeTableRequest; -import software.amazon.awssdk.services.dynamodb.model.DescribeTableResponse; - -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; -import org.springframework.boot.test.context.FilteredClassLoader; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.aws.metadata.DynamoDbMetadataStore; -import org.springframework.integration.hazelcast.metadata.HazelcastMetadataStore; -import org.springframework.integration.jdbc.metadata.JdbcMetadataStore; -import org.springframework.integration.metadata.ConcurrentMetadataStore; -import org.springframework.integration.metadata.MetadataStore; -import org.springframework.integration.metadata.SimpleMetadataStore; -import org.springframework.integration.mongodb.metadata.MongoDbMetadataStore; -import org.springframework.integration.redis.metadata.RedisMetadataStore; -import org.springframework.integration.zookeeper.metadata.ZookeeperMetadataStore; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.BDDMockito.willReturn; -import static org.mockito.Mockito.mock; - -/** - * @author Artem Bilan - * @author Corneil du Plessis - * - * @since 2.0.2 - */ -public class MetadataStoreAutoConfigurationTests { - - private final static List> METADATA_STORE_CLASSES = - List.of( - RedisMetadataStore.class, - MongoDbMetadataStore.class, - JdbcMetadataStore.class, - ZookeeperMetadataStore.class, - HazelcastMetadataStore.class, - DynamoDbMetadataStore.class, - SimpleMetadataStore.class - ); - - @ParameterizedTest - @MethodSource - public void testMetadataStore(Class classToInclude) { - ApplicationContextRunner contextRunner = - new ApplicationContextRunner() - .withUserConfiguration(TestConfiguration.class) - .withPropertyValues("metadata.store.type=" + - classToInclude.getSimpleName() - .replaceFirst("MetadataStore", "") - .toLowerCase() - .replaceFirst("simple", "memory")) - .withClassLoader(filteredClassLoaderBut(classToInclude)); - contextRunner - .run(context -> { - assertThat(context.getBeansOfType(MetadataStore.class)).hasSize(1); - - assertThat(context.getBeanNamesForType(classToInclude)) - .containsOnlyOnce(Introspector.decapitalize(classToInclude.getSimpleName())); - }); - } - - static List> testMetadataStore() { - return METADATA_STORE_CLASSES; - } - - private static FilteredClassLoader filteredClassLoaderBut(Class classToInclude) { - return new FilteredClassLoader( - METADATA_STORE_CLASSES.stream() - .filter(Predicate.isEqual(classToInclude).negate()) - .toArray(Class[]::new)); - } - - @Configuration - @EnableAutoConfiguration - public static class TestConfiguration { - - @Bean(destroyMethod = "stop") - @ConditionalOnClass(ZookeeperMetadataStore.class) - public static TestingServer zookeeperTestingServer() throws Exception { - TestingServer testingServer = new TestingServer(true); - - System.setProperty("metadata.store.zookeeper.connect-string", testingServer.getConnectString()); - System.setProperty("metadata.store.zookeeper.encoding", StandardCharsets.US_ASCII.name()); - - return testingServer; - } - - @Configuration - @ConditionalOnClass(DynamoDbMetadataStore.class) - protected static class DynamoDbMockConfig { - - @Bean - public static DynamoDbAsyncClient dynamoDB() { - DynamoDbAsyncClient dynamoDb = mock(DynamoDbAsyncClient.class); - willReturn(CompletableFuture.completedFuture(DescribeTableResponse.builder().build())) - .given(dynamoDb) - .describeTable(ArgumentMatchers.>any()); - - return dynamoDb; - } - - @Bean - public static AwsCredentialsProvider awsCredentialsProvider() { - return mock(AwsCredentialsProvider.class); - } - - @Bean - public static AwsRegionProvider regionProvider() { - return mock(AwsRegionProvider.class); - } - - } - - } - -} diff --git a/functions/common/mongodb-common/pom.xml b/functions/common/mongodb-common/pom.xml deleted file mode 100644 index baa33f34f..000000000 --- a/functions/common/mongodb-common/pom.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - mongodb-common - mongodb-common - MongoDB Common - - - - org.testcontainers - mongodb - ${testcontainers.version} - test - - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.1.2 - - - Jar Tests Package - package - - test-jar - - - - - - - diff --git a/functions/common/mongodb-common/src/main/resources/application.properties b/functions/common/mongodb-common/src/main/resources/application.properties deleted file mode 100644 index 1bb8bf6d7..000000000 --- a/functions/common/mongodb-common/src/main/resources/application.properties +++ /dev/null @@ -1 +0,0 @@ -# empty diff --git a/functions/common/mongodb-common/src/test/java/org/springframework/cloud/fn/common/mongo/MongoDbTestContainerSupport.java b/functions/common/mongodb-common/src/test/java/org/springframework/cloud/fn/common/mongo/MongoDbTestContainerSupport.java deleted file mode 100644 index 8848a342d..000000000 --- a/functions/common/mongodb-common/src/test/java/org/springframework/cloud/fn/common/mongo/MongoDbTestContainerSupport.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2022-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.mongo; - -import java.time.Duration; - -import org.junit.jupiter.api.BeforeAll; -import org.testcontainers.containers.MongoDBContainer; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * Provides a static {@link MongoDBContainer} that can be shared across test classes. - * - * @author Chris Bono - */ -@Testcontainers(disabledWithoutDocker = true) -public interface MongoDbTestContainerSupport { - - MongoDBContainer MONGO_CONTAINER = new MongoDBContainer("mongo:6.0.6") - .withStartupTimeout(Duration.ofSeconds(120)) - .withStartupAttempts(3); - @BeforeAll - static void startContainer() { - MONGO_CONTAINER.start(); - } - - static String mongoDbUri() { - return "mongodb://localhost:" + MONGO_CONTAINER.getFirstMappedPort(); - } - -} diff --git a/functions/common/mqtt-common/pom.xml b/functions/common/mqtt-common/pom.xml deleted file mode 100644 index f2cc6ab10..000000000 --- a/functions/common/mqtt-common/pom.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - mqtt-common - mqtt-common - MQTT Common - - - - org.springframework.integration - spring-integration-mqtt - - - - diff --git a/functions/common/mqtt-common/src/main/java/org/springframework/cloud/fn/common/mqtt/MqttConfiguration.java b/functions/common/mqtt-common/src/main/java/org/springframework/cloud/fn/common/mqtt/MqttConfiguration.java deleted file mode 100644 index 7cdad1510..000000000 --- a/functions/common/mqtt-common/src/main/java/org/springframework/cloud/fn/common/mqtt/MqttConfiguration.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2017-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.mqtt; - -import java.util.Map; -import java.util.Properties; - -import org.eclipse.paho.client.mqttv3.MqttConnectOptions; -import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence; -import org.eclipse.paho.client.mqttv3.persist.MqttDefaultFilePersistence; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.mqtt.core.DefaultMqttPahoClientFactory; -import org.springframework.integration.mqtt.core.MqttPahoClientFactory; -import org.springframework.util.ObjectUtils; - -/** - * Generic mqtt configuration. - * - * @author Janne Valkealahti - * @author Artem Bilan - */ -@Configuration -public class MqttConfiguration { - - @Autowired - private MqttProperties mqttProperties; - - @Bean - public MqttPahoClientFactory mqttClientFactory() { - - MqttConnectOptions mqttConnectOptions = new MqttConnectOptions(); - mqttConnectOptions.setServerURIs(mqttProperties.getUrl()); - mqttConnectOptions.setUserName(mqttProperties.getUsername()); - mqttConnectOptions.setPassword(mqttProperties.getPassword().toCharArray()); - mqttConnectOptions.setCleanSession(mqttProperties.isCleanSession()); - mqttConnectOptions.setConnectionTimeout(mqttProperties.getConnectionTimeout()); - mqttConnectOptions.setKeepAliveInterval(mqttProperties.getKeepAliveInterval()); - - Map sslProperties = mqttProperties.getSslProperties(); - - if (!sslProperties.isEmpty()) { - Properties sslProps = new Properties(); - sslProps.putAll(sslProperties); - mqttConnectOptions.setSSLProperties(sslProps); - } - - DefaultMqttPahoClientFactory factory = new DefaultMqttPahoClientFactory(); - factory.setConnectionOptions(mqttConnectOptions); - - if (ObjectUtils.nullSafeEquals(mqttProperties.getPersistence(), "file")) { - factory.setPersistence(new MqttDefaultFilePersistence(mqttProperties.getPersistenceDirectory())); - } - else if (ObjectUtils.nullSafeEquals(mqttProperties.getPersistence(), "memory")) { - factory.setPersistence(new MemoryPersistence()); - } - return factory; - } - -} diff --git a/functions/common/mqtt-common/src/main/java/org/springframework/cloud/fn/common/mqtt/MqttProperties.java b/functions/common/mqtt-common/src/main/java/org/springframework/cloud/fn/common/mqtt/MqttProperties.java deleted file mode 100644 index ea70f75d4..000000000 --- a/functions/common/mqtt-common/src/main/java/org/springframework/cloud/fn/common/mqtt/MqttProperties.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2017-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.mqtt; - -import java.util.HashMap; -import java.util.Map; - -import jakarta.validation.constraints.Size; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * Generic mqtt connection properties. - * - * @author Janne Valkealahti - * @author Artem Bilan - */ -@Validated -@ConfigurationProperties("mqtt") -public class MqttProperties { - - /** - * location of the mqtt broker(s) (comma-delimited list). - */ - private String[] url = new String[] { "tcp://localhost:1883" }; - - /** - * the username to use when connecting to the broker. - */ - private String username = "guest"; - - /** - * the password to use when connecting to the broker. - */ - private String password = "guest"; - - /** - * whether the client and server should remember state across restarts and reconnects. - */ - private boolean cleanSession = true; - - /** - * the connection timeout in seconds. - */ - private int connectionTimeout = 30; - - /** - * the ping interval in seconds. - */ - private int keepAliveInterval = 60; - - /** - * 'memory' or 'file'. - */ - private String persistence = "memory"; - - /** - * Persistence directory. - */ - private String persistenceDirectory = "/tmp/paho"; - - /** - * MQTT Client SSL properties. - */ - private final Map sslProperties = new HashMap<>(); - - @Size(min = 1) - public String[] getUrl() { - return url; - } - - public void setUrl(String[] url) { - this.url = url; - } - - public String getUsername() { - return username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return password; - } - - public void setPassword(String password) { - this.password = password; - } - - public boolean isCleanSession() { - return cleanSession; - } - - public void setCleanSession(boolean cleanSession) { - this.cleanSession = cleanSession; - } - - public int getKeepAliveInterval() { - return keepAliveInterval; - } - - public void setKeepAliveInterval(int keepAliveInterval) { - this.keepAliveInterval = keepAliveInterval; - } - - public int getConnectionTimeout() { - return connectionTimeout; - } - - public void setConnectionTimeout(int connectionTimeout) { - this.connectionTimeout = connectionTimeout; - } - - public String getPersistence() { - return persistence; - } - - public void setPersistence(String persistence) { - this.persistence = persistence; - } - - public String getPersistenceDirectory() { - return persistenceDirectory; - } - - public void setPersistenceDirectory(String persistenceDirectory) { - this.persistenceDirectory = persistenceDirectory; - } - - public Map getSslProperties() { - return this.sslProperties; - } - -} diff --git a/functions/common/pom.xml b/functions/common/pom.xml deleted file mode 100644 index d07b314d2..000000000 --- a/functions/common/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - 4.0.0 - - org.springframework.cloud.fn - functions-common - 5.0.0-SNAPSHOT - java-functions-common - Java Functions Common - pom - - - config-common - debezium-autoconfigure - aws-s3-common - file-common - ftp-common - function-test-support - metadata-store-common - mongodb-common - mqtt-common - redis-common - tcp-common - twitter-common - tensorflow-common - xmpp-common - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.0.0 - - true - - - - - diff --git a/functions/common/redis-common/pom.xml b/functions/common/redis-common/pom.xml deleted file mode 100644 index f469d5d35..000000000 --- a/functions/common/redis-common/pom.xml +++ /dev/null @@ -1,35 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - redis-common - redis-common - Redis Common - - - - - org.apache.maven.plugins - maven-jar-plugin - 3.1.2 - - - Jar Tests Package - package - - test-jar - - - - - - - diff --git a/functions/common/redis-common/src/main/resources/application.properties b/functions/common/redis-common/src/main/resources/application.properties deleted file mode 100644 index 1bb8bf6d7..000000000 --- a/functions/common/redis-common/src/main/resources/application.properties +++ /dev/null @@ -1 +0,0 @@ -# empty diff --git a/functions/common/redis-common/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisTestContainerSupport.java b/functions/common/redis-common/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisTestContainerSupport.java deleted file mode 100644 index 8f12bba96..000000000 --- a/functions/common/redis-common/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisTestContainerSupport.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2022-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.redis; - -import java.time.Duration; - -import org.junit.jupiter.api.BeforeAll; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * Provides a static Redis Container that can be shared across test classes. - * - * @author Corneil du Plessis - */ -@Testcontainers(disabledWithoutDocker = true) -public interface RedisTestContainerSupport { - GenericContainer REDIS_CONTAINER = new GenericContainer<>("redis:7") - .withExposedPorts(6379) - .withStartupTimeout(Duration.ofSeconds(120)) - .withStartupAttempts(3); - - @BeforeAll - static void startContainer() { - REDIS_CONTAINER.start(); - } - - static String getUri() { - return "redis://localhost:" + REDIS_CONTAINER.getFirstMappedPort(); - } -} diff --git a/functions/common/tcp-common/pom.xml b/functions/common/tcp-common/pom.xml deleted file mode 100644 index 9c8827390..000000000 --- a/functions/common/tcp-common/pom.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - tcp-common - tcp-common - tcp common - - - - - org.springframework.integration - spring-integration-ip - - - - - diff --git a/functions/common/tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/EncoderDecoderFactoryBean.java b/functions/common/tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/EncoderDecoderFactoryBean.java deleted file mode 100644 index 7cfe763c1..000000000 --- a/functions/common/tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/EncoderDecoderFactoryBean.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tcp; - -import org.springframework.beans.factory.config.AbstractFactoryBean; -import org.springframework.context.ApplicationEventPublisher; -import org.springframework.context.ApplicationEventPublisherAware; -import org.springframework.integration.ip.tcp.serializer.AbstractByteArraySerializer; -import org.springframework.integration.ip.tcp.serializer.ByteArrayCrLfSerializer; -import org.springframework.integration.ip.tcp.serializer.ByteArrayLengthHeaderSerializer; -import org.springframework.integration.ip.tcp.serializer.ByteArrayLfSerializer; -import org.springframework.integration.ip.tcp.serializer.ByteArrayRawSerializer; -import org.springframework.integration.ip.tcp.serializer.ByteArraySingleTerminatorSerializer; -import org.springframework.integration.ip.tcp.serializer.ByteArrayStxEtxSerializer; -import org.springframework.util.Assert; - -/** - * Factory bean for an encoder/decoder based on - * {@link Encoding}. - * - * @author Gary Russell - * @author Christian Tzolov - */ -public class EncoderDecoderFactoryBean extends AbstractFactoryBean - implements ApplicationEventPublisherAware { - - private final Encoding encoding; - - private ApplicationEventPublisher applicationEventPublisher; - - private Integer maxMessageSize; - - public EncoderDecoderFactoryBean(Encoding encoding) { - Assert.notNull(encoding, "'encoding' cannot be null"); - this.encoding = encoding; - } - - @Override - public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) { - this.applicationEventPublisher = applicationEventPublisher; - } - - /** - * The maximum message size allowed when decoding. - * @param maxMessageSize the maximum message size. - */ - public void setMaxMessageSize(int maxMessageSize) { - this.maxMessageSize = maxMessageSize; - } - - @Override - protected AbstractByteArraySerializer createInstance() throws Exception { - AbstractByteArraySerializer codec; - switch (this.encoding) { - case CRLF: - codec = new ByteArrayCrLfSerializer(); - break; - case LF: - codec = new ByteArrayLfSerializer(); - break; - case NULL: - codec = new ByteArraySingleTerminatorSerializer((byte) 0); - break; - case STXETX: - codec = new ByteArrayStxEtxSerializer(); - break; - case L1: - codec = new ByteArrayLengthHeaderSerializer(1); - break; - case L2: - codec = new ByteArrayLengthHeaderSerializer(2); - break; - case L4: - codec = new ByteArrayLengthHeaderSerializer(4); - break; - case RAW: - codec = new ByteArrayRawSerializer(); - break; - default: - throw new IllegalArgumentException("Invalid encoding: " + this.encoding); - } - codec.setApplicationEventPublisher(this.applicationEventPublisher); - if (this.maxMessageSize != null) { - codec.setMaxMessageSize(this.maxMessageSize); - } - return codec; - } - - @Override - public Class getObjectType() { - return AbstractByteArraySerializer.class; - } - -} diff --git a/functions/common/tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/Encoding.java b/functions/common/tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/Encoding.java deleted file mode 100644 index d434ce548..000000000 --- a/functions/common/tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/Encoding.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tcp; - -/** - * @author Gary Russell - * @author Christian Tzolov - */ -public enum Encoding { - /** - * CRLF encoding. - */ - CRLF, - /** - * LF encoding. - */ - LF, - /** - * Null encoding. - */ - NULL, - /** - * STXETX encoding. - */ - STXETX, - /** - * Raw encoding. - */ - RAW, - /** - * L1 encoding. - */ - L1, - /** - * L2 encoding. - */ - L2, - /** - * L4 encoding. - */ - L4; -} diff --git a/functions/common/tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/TcpConnectionFactoryProperties.java b/functions/common/tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/TcpConnectionFactoryProperties.java deleted file mode 100644 index 6d011dc51..000000000 --- a/functions/common/tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/TcpConnectionFactoryProperties.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tcp; - -import org.springframework.boot.context.properties.ConfigurationProperties; - -/** - * Base class for TCP connection factory properties. - * - * @author Eric Bottard - * @author Gary Russell - * @author Christian Tzolov - */ -@ConfigurationProperties("tcp") -public class TcpConnectionFactoryProperties { - - /** - * The port on which to listen; 0 for the OS to choose a port. - */ - private int port = 1234; - - /** - * Perform a reverse DNS lookup on the remote IP Address; if false, - * just the IP address is included in the message headers. - */ - private boolean reverseLookup = false; - - /** - * The timeout (ms) before closing the socket when no data is received. - */ - private int socketTimeout = 120000; - - /** - * Whether or not to use NIO. - */ - private boolean nio = false; - - /** - * Whether or not to use direct buffers. - */ - private boolean useDirectBuffers = false; - - public int getPort() { - return this.port; - } - - public void setPort(int port) { - this.port = port; - } - - public boolean isUseDirectBuffers() { - return this.useDirectBuffers; - } - - public void setUseDirectBuffers(boolean useDirectBuffers) { - this.useDirectBuffers = useDirectBuffers; - } - - public boolean isNio() { - return this.nio; - } - - public void setNio(boolean nio) { - this.nio = nio; - } - - public int getSocketTimeout() { - return this.socketTimeout; - } - - public void setSocketTimeout(int socketTimeout) { - this.socketTimeout = socketTimeout; - } - - public boolean isReverseLookup() { - return this.reverseLookup; - } - - public void setReverseLookup(boolean reverseLookup) { - this.reverseLookup = reverseLookup; - } -} diff --git a/functions/common/tensorflow-common/README.adoc b/functions/common/tensorflow-common/README.adoc deleted file mode 100644 index 158646673..000000000 --- a/functions/common/tensorflow-common/README.adoc +++ /dev/null @@ -1,344 +0,0 @@ -:images-asciidoc: https://raw.githubusercontent.com/spring-cloud/stream-applications/master/functions/common/tensorflow-common/src/main/resources/images/ - -= Programming Model for TensorFlow Inference - -Programming model builds on the https://docs.oracle.com/javase/8/docs/api/java/util/function/package-summary.html[Java Function API], the `TF Java Ops API` and few basic data structure that together help to unify and streamline the building of TensorFlow inference pipelines. - -Quick Start: just add the following dependency: - -[source,XML] ----- - - org.springframework.cloud.fn - tensorflow-common - ${revision} - ----- - -== Programming Model - -Implementing a real-time TensorFlow Inference in Java, typically leverages the TF Java API for loading and scoring the pre-trained models. But the logic used to convert the upstream data into model input Tensors (e.g. pre-processor) and in turn to convert the inferred Tensors back into application data (e.g. post-processor) is commonly implemented in plain Java: - -image::{images-asciidoc}/programming_model.png[TF Architecture, scaledwidth="70%"] - -The Pre and Post processing steps could become very complex (check https://github.com/ildoonet/tf-pose-estimation[Pose Estimation] or https://github.com/davidsandberg/facenet[Face Recognition]) and computationally intensive. E.g. tons of math and image operations that are better fit for optimized TF utilities rather the plain Java math or AWT/2D/Canvas such. - -Additionally, the unnecessary shuffling of data between the JVM and the native TF impacts the overall performance of the flow. The issue is apparent when multiple pre-trained TF models are combined (composed) or the same TF models are evaluated iteratively. In those cases the processed data is repeatedly being moved between the JVM Heap and the TF native memory. - -The `Java Ops API` exposes the native https://www.tensorflow.org/versions/r1.9/api_docs/cc?hl=en[TF C++ Core API] offering comprehensive, native math, image, io and alike utilities. Later is useful for implementing the computational intensive logic by using the same TF tools and infrastructure used for running the pre-trained models. - -Proposed programming model builds on the https://docs.oracle.com/javase/8/docs/api/java/util/function/package-summary.html[Java Function API], the Java Ops API and a basic data structure that together help to unify and streamline the building of TensorFlow inference pipelines. While focused on model-inference, this programming model is likely to be useful for building model-training pipelines as well. - -The programming model leverages the following functional definition: - -[source,Java] ----- -Function>, Map>> ----- - -and the corresponding method expression - -[source,Java] ----- -Map> apply( Map> feeds) ----- - -This function receives a map of named Tensors as an input and in turn returns a map of named Tensors. Because the input and the output formats are equivalent, functions with this signature can be reused and composed into larger, complex functions. - -The names used in the inputs, and the outputs maps are strings of the form `operation_name: output_index` (output_index defaults to 0). The names must match the indexed operations inside the underlying TF graph. + - -The https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/Tensor[Tensor] class is a reference to the data used natively in the TF engine. The referenced data is not moved to JVM Heap unless explicitly materialized with `Tensor.copyTo()`. Exchanging Tensor references between functions prevents unnecessarily copying the data to the JVM heap. + - -Proposed data structure fits well with existing https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/Session.Runner[Session.Runner API], which accepts https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/Session.Runner.html#feed(java.lang.String,%20org.tensorflow.Tensor%3C?%3E)[indexed operations] as an input feed and returns list of tensors predefined by https://www.tensorflow.org/api_docs/java/reference/org/tensorflow/Session.Runner.html#fetch(java.lang.String)[fetch indexed operations]. - -The https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunner.java[GraphRunner] and the https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphDefinition.java[GraphDefinition] are the core abstractions used to define, load and inference TensorFlow models. https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunner.java[GraphRunner] implements the Function (e.g. Fn, Map>) definition and uses the TF Java API to run the underlying TF graph. The input Tensor map is fed to the Session Runner. After the graph is evaluated, a list of predefined fetch names is used to retrieve selected Tensors from the result as a named Tensor map. The https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunner.java#L70[withGraphDefinition(GraphDefinition)] method defines a new or loads a pre-trained TF graph, while the https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunner.java#L84[withSavedModel(path)] method helps to load a Tensorflow SavedModel. -The GraphDefinition argument is a functional interface and can therefore be used as the assignment target for a lambda expression or method reference. - -Following snippets illustrates how to use the `withGraphDefinition` to define a new TF Graph that computes the `y1 = x1 * 2` expression: - -[source,Java] ----- -myGraph = new GraphRunner("x1", "y1") - .withGraphDefinition( tf -> - tf.withName("y1").math.mul( - tf.withName("x1").placeholder(Integer.class), - tf.constant(2))); ----- - -The x1 and y1 constructor arguments define the input and output Tensor names (technically indexed operation names) used to feed in and fetch out data to and from the defined model. - -The GraphRunner https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AbstractGraphRunner.java#L65[apply] method helps evaluate/inference the so defined graph: - -[source,Java] ----- -Map> input = Collections.singletonMap("x1", Tensor.create(666)); -result = myGraph.apply(input); ----- - -Similarly, we can load a frozen/pre-trained model (https://github.com/tensorflow/models/tree/master/research/slim/nets/mobilenet#pretrained-models[MobileNetV2] model in this case) from an archive using the https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/ProtoBufGraphDefinition.java[ProtoBufGraphDefinitions] helper class. - -[source,Java] ----- -mibileNetV2 = new GraphRunner("input", "MobilenetV2/Predictions/Reshape_1") - .withGraphDefinition( - new ProtoBufGraphDefinition( -"https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.4_224.tgz#mobilenet_v2_1.4_224_frozen.pb", - cacheModel)); ----- - -You can load archives from `http://`, `file://` or `classpath://` locations. - -For loading a `SavedModel` use the `GraphRunner#withSavedModel method like this: - -[source,Java] ----- -ssdMibileNetV1Coco = new GraphRunner( Arrays.asList("image_tensor"), - Arrays.asList("detection_boxes", “detection_scores”, “detection_classes”)) - .withSavedModel( ”./ssd_mobilenet_v1_coco_2017_11_17/saved_model”, "serve"); ----- - -An example inference pipeline based on the proposed Functional Programming Model would look similar to this: - -image::{images-asciidoc}/tf_pipeline.png[TF pipelinne, scaledwidth="70%"] - -Every GraphRunner instance in the pipeline uses either an in-place defined, or a pre-trained TF graph. - -In practice, it would still be required to implement some input and output adapters for the logic that cannot be (or are not feasible to be) implemented with the native Java Ops API. But you have the freedom to choose what part of the processing logic to run as natively (e.g. Java Ops API) code and what is a plain Java. - -Furthermore, we are not limited to GraphRunner but any custom https://docs.oracle.com/javase/8/docs/api/java/util/function/Function.html[Function], Map> implementations can be used in the processing pipelines. In fact the https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/Functions.java[Functions] utilities use this approach. - -When appropriate any custom Function, https://docs.oracle.com/javase/8/docs/api/java/util/function/Supplier.html[Supplier]>, https://docs.oracle.com/javase/8/docs/api/java/util/function/Consumer.html[Consumer]> or the rest of the https://docs.oracle.com/javase/8/docs/api/java/util/function/package-frame.html[java.util.function] classes and interfaces can be used. - -For real time examples check the https://github.com/spring-cloud/stream-applications/tree/master/functions/function/image-recognition-function[image-recognition] and https://github.com/spring-cloud/stream-applications/tree/master/functions/function/semantic-segmentation-function[semantic-segmentation] implementations. - -== Features - -Following paragraphs discusses some features and techniques useful for composing graphs, memorizing and reusing intermediate Tensor values, managing the Tensor resources and so on. - -=== Input and Output Contracts - -The GraphRunner constructor expects two compulsory list fields: `feedNames` - list of names that the graph accepts as input Tensors and `fetchNames` - list of (Tensor) names that the graph would return. - -[source,Java] ----- -public GraphRunner(List feedNames, List fetchedNames) ----- - -Together those two lists define the input and output contract of the graph runner. + -The names used in the inputs, and the outputs maps are strings of the form `operation_name : output_index` (output_index defaults to 0). The names must match the indexed operations inside the underlying TF graph. - -=== Composition - -Because the https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunner.java[GraphRunner] function signature uses the same type for input and output parameters, the https://docs.oracle.com/javase/8/docs/api/java/util/function/Function.html[Functional] interface allows us to compose multiple graphs https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunner.java[GraphRunner] functions into a larger composite function: - -[source,Java] ----- -composed-graph = graph1.andThen(graph2)....andThen(graphN) ----- - -For example let's take two simple graphs: `G1 (y1 = x1 * 2)` and `G2 (y2 = x2 + 20)`. The composed graph `G = G1.andThen(G2)` is equivalent to `y = (x * 2 ) + 20`. - -The https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/FunctionComposition.java[FunctionComposition example] demonstrates how this works: - -[source,Java] ----- -try ( - - GraphRunner graph1 = new GraphRunner("x1", "y1") - .withGraphDefinition(tf -> tf.withName("y1").math.mul( - tf.withName("x1").placeholder(Integer.class), - tf.constant(2))); - - GraphRunner graph2 = new GraphRunner("x2", "y2") - .withGraphDefinition(tf -> tf.withName("y2").math.add( - tf.withName("x2").placeholder(Integer.class), - tf.constant(20))); - - Tensor x = Tensor.create(10); -) { - - Map> result = - graph1.andThen(graph2).apply(Collections.singletonMap("x", x)); - - System.out.println("Result is: " + result.get("y2").intValue()); // Result is: 40 -} ----- - -Note that the GraphRunner https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AbstractGraphRunner.java#L65[automatically binds] the singleton outputs (e.g fetch) with the singleton input (e.g. feeds). In the example above the GraphRunner automatically binds the `y1` tensor produced by `graph1` to the `x2` input placeholders expected by `graph2`. - -==== Multiple inputs/outputs - -When the composed graphs use multiple input and output parameters we need to explicitly bind the outputs from the upstream graph to the inputs of the downstream one. - -For example let’s Graph1 produces two outputs (e.g. fetchNames) y11 and y12 and Graph2 expects to inputs (e.g. feedNames) x21 and x22: - -|=== -|Graph1:|Graph2: -| y11 = x1 * 2 | y2 = x21 + x22 -| y12 = x1 * 3 | -|=== - -The composed graph would look like this: - -[source,Java] ----- -Composed = Graph1.andThen( map: y11 -> x21 and y12 -> x22).andThen(Graph2) ----- - -The https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/Functions.java#L72[Functions#rename] utility helps to define the input/output mappings as illustrated in the https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/FunctionCompositionMultipleInputsOutputs.java[FunctionCompositionMultipleInputsOutputs] example: - -[source,Java] ----- -try ( - - GraphRunner graph1 = new GraphRunner(Arrays.asList("x1"), Arrays.asList("y11", "y12")) - .withGraphDefinition(tf -> { - Placeholder x1 = tf.withName("x1").placeholder(Integer.class); - tf.withName("y11").math.mul(x1, tf.constant(2)); - tf.withName("y12").math.mul(x1, tf.constant(3)); - }); - - GraphRunner graph2 = new GraphRunner(Arrays.asList("x21", "x22"), Arrays.asList("y2")) - .withGraphDefinition(tf -> tf.withName("y2").math.add( - tf.withName("x21").placeholder(Integer.class), - tf.withName("x22").placeholder(Integer.class))); - - Tensor x = Tensor.create(10); -) { - - Map> result = - graph1 - .andThen( - Functions.rename( - "y11", "x21", - "y12", "x22" - )) - .andThen(graph2) - .apply(Collections.singletonMap("x", x)); - - System.out.println("Result is: " + result.get("y2").intValue()); // Result is: 50 -} ----- - -The Functions#rename(String...mappings) takes an even number of string pairs, where every even parameter represents the from and to name to map. Eg. The y11 above is mapped into x21 and y12 is mapped into x22. + -The https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AbstractGraphRunner.java#L129[GraphRunner#enableAutoBinding()] and https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AbstractGraphRunner.java#L124[GraphRunner#disableAutoBinding()] allow altering the autobinding behavior enforcing mapping even of singleton input/output graphs. - -=== Save and Close Obsolete Tensors - -The Tensors used as inputs (feeds) and outputs (fetches) by the GraphRunners have to be released (e.g. closed) when not used anymore. - -Because every sub-graph in a composite pipeline produces one or more pairs we need to track those references and close them. - -The https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunnerMemory.java[GraphRunnerMemory] is a handy utility Function implementation that keeps track of all input Tensor parameters passed through. It is https://docs.oracle.com/javase/8/docs/api/java/lang/AutoCloseable.html[AutoClosable] and will release all tracked Tensors when closed. - -The GraphRunnerMemory implements the same function signatures as the GraphRunner (e.g. Fun, Map>) and therefore can participate in composite graph definitions: - -[source,Java] ----- -try ( memory = new GraphRunnerMemory() ) { - composed-graph = - Graph1..andThen(memory) - .andThen(Graph2).andThen(memory) - … - .andThen(GraphN).andThen(memory) - …. - -} // releases all Tensors returned by the GraphRunners ----- - -The https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/ReleaseTensorParameters.java[ReleaseTensorParameters] example illustrates how to use the GraphRunnerMemory: - -[source,Java] ----- -try ( - Tensor x = Tensor.create(input); - GraphRunnerMemory memory = new GraphRunnerMemory(); -) { - - Map> result = - this.graph1.andThen(memory) - .andThen(this.graph2).andThen(memory) - .apply(Collections.singletonMap("x", x)); - - return result.get("y2").intValue(); -} - -// At that point all intermediate Tensors used by the GraphRunners are closed. ----- - -Note: the GraphRunnerMemory has some other very useful applications that we will highlight in the next paragraph. - -=== Enrich Graph Inputs - -For particular graphs in the composite pipeline, we can add an additional input parameters that were not produced by the upstream graph. - -With the help fo the https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/Functions.java#L42[Functions#enrichWith(name, Tensor)] utility function we can inject the additional parameters in the graph composition. - -In the following snippet we enrich the graph2’s input with an additional parameter (newParam): - -[source,Java] ----- -try ( - Tensor x = Tensor.create(input); - Tensor additionalTensor = Tensor.create(colorMap); -) { - - Map> result = - graph1 - .andThen(Functions.enrichWIth("newParam", additionalTensor) - .andThen(graph2) - .apply(Collections.singletonMap("x", x)); - - return result.get("y2").intValue(); -} ----- - -The https://github.com/spring-cloud/stream-applications/blob/master/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SemanticSegmentation.java#L150[SemanticSegmentation] implementation provides a real example how to enrich with parameters. - -=== Enrich Inputs from Saved Tensors - -We can combine the enricher approach with the https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunnerMemory.java[GraphRunnerMemory]. This allows us to enrich some downstream Graphs with tensor parameters computed in some of the upstream Graphs. The `Functions#enrichFromMemory(memory, tensorName)` utility function can enrich a graph input parameter by extracting one stored in the memory. - -For example let’s construct the following graph compositions: - ----- -graph1: y1 = x1 * 10 + -graph2: y2 = y1 * 200 + -graph3: y3 = y2 + y1 ----- - -[source,Java] ----- -try ( - Tensor x = Tensor.create(input); - GraphRunnerMemory memory = new GraphRunnerMemory(); -) { - - Map> result = - this.graph1.andThen(memory) // memorizes y1 - .andThen(graph2).andThen(memory) // memorizes y2 - .andThen(Functions.enrichFromMemory(memory, "y1")) // retrieve graph1’s output y1 and adds it as an input for the next function. - .andThen(Functions.rename( - "y1", "x31", // renames the input y1 into x31 - "y2", "x32" // renames the input y2 into x32 - )) - .andThen(graph3).andThen(memory) - .apply(Collections.singletonMap("x", x)); - - return result.get("y3").intValue(); -} ----- - -=== Load Frozen Models from Remote Archives - -The ProtoBufGraphDefinition extracts a pre-trained (frozen) Tensorflow model form a URI archive into byte array. It supports the `http(s)://`, `file://` and `classpath://` URI schemas. For this it uses the `ModelExtractor` and `CachedModelExtractor` utilities. - -Models can be extracted either from raw files or form compressed archives. When extracted from an archive the model file name can optionally be provided as a URI fragment. For example for resource: `http://myarchive.tar.gz#model.pb` -the `myarchive.tar.gz` is traversed to uncompress and extract the model.pb file as a byte array. If the file name is not provided as URI fragment then the first file in the archive with extension .pb is extracted. - -In addition, the CachedModelExtractor allows keeping a local copy (cache) of the model (protobuf) files extracted from the URI archive. - -|=== -|The https://github.com/spring-cloud/stream-applications/tree/master/functions/function/image-recognition-function[image-recognition] and https://github.com/spring-cloud/stream-applications/tree/master/functions/function/semantic-segmentation-function[semantic-segmentation] inference models implementations demonstrate the suggested programming model. - -|=== diff --git a/functions/common/tensorflow-common/pom.xml b/functions/common/tensorflow-common/pom.xml deleted file mode 100644 index a17cb06c5..000000000 --- a/functions/common/tensorflow-common/pom.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - tensorflow-common - tensorflow-common - tensorflow common - - - true - 1.15.0 - 1.26.0 - 2.7 - 3.10 - 3.0.3 - 1.7.26 - - - - - org.tensorflow - tensorflow - ${tensorflow.version} - - - org.tensorflow - proto - ${tensorflow.version} - - - - org.apache.commons - commons-compress - ${apache.commons.compress} - - - commons-io - commons-io - ${commons-io.version} - - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - - - org.slf4j - slf4j-api - ${slf4j-api.version} - - - org.pcollections - pcollections - ${pcollections.version} - - - - - - tensorflow-snapshots - https://oss.sonatype.org/content/repositories/snapshots/ - - true - - - - diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AbstractGraphRunner.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AbstractGraphRunner.java deleted file mode 100644 index 2958df5ee..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AbstractGraphRunner.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -import org.apache.commons.lang3.Validate; -import org.tensorflow.Session; -import org.tensorflow.Tensor; - -/** - * @author Christian Tzolov - */ -public abstract class AbstractGraphRunner implements Function>, Map>> { - - public abstract Session doGetSession(); - - /** - * Names expected in the named Tensor inside the input {@link AbstractGraphRunner#apply(Map)}. - * If the apply method will fail if the input map is missing some of the feedNames. - */ - private final List feedNames; - - /** - * Names expected {@link AbstractGraphRunner#apply(Map)} result map. - */ - private final List fetchNames; - - /** - * When set and the input takes a single feed, then the name of the input tensor is automatically mapped - * to the expected input name. E.g. no need to rename the input names explicitly. - */ - private boolean autoBinding; - - public AbstractGraphRunner(String feedName, String fetchedName) { - this(Arrays.asList(feedName), Arrays.asList(fetchedName)); - } - - public AbstractGraphRunner(List feedNames, List fetchedNames) { - this.feedNames = feedNames; - this.fetchNames = fetchedNames; - this.autoBinding = feedNames.size() == 1; - } - - @Override - public Map> apply(Map> feeds) { - - if (!this.isAutoBinding() && !feeds.keySet().containsAll(this.feedNames)) { - throw new IllegalArgumentException("Applied feeds:" + feeds.keySet() - + "\n, don't match the expected feeds contract:" + this.feedNames); - } - - if (this.isAutoBinding() && (feeds.size() != 1)) { - throw new IllegalArgumentException("Feed auto-binding expects a " + - "single feed tensors but found: " + feeds); - } - - Session.Runner runner = this.doGetSession().runner(); - - // Feed in the input named tensors - for (Map.Entry> feedEntry : feeds.entrySet()) { - String feedName = (this.isAutoBinding()) ? this.feedNames.get(0) : feedEntry.getKey(); - runner = runner.feed(feedName, feedEntry.getValue()); - } - - // Set the tensor name to be fetched after the evaluation - for (String fetchName : this.fetchNames) { - runner.fetch(fetchName); - } - - // Evaluate the input - List> outputTensors = runner.run(); - - // Extract the output tensors - Map> outTensorMap = new HashMap<>(); - for (int outputIndex = 0; outputIndex < this.fetchNames.size(); outputIndex++) { - outTensorMap.put(this.fetchNames.get(outputIndex), outputTensors.get(outputIndex)); - } - - return outTensorMap; - } - - public List getFeedNames() { - return this.feedNames; - } - - public String getSingleFeedName() { - Validate.isTrue(feedNames.size() == 1, "Assumes a single feed input"); - return this.feedNames.get(0); - } - - public List getFetchNames() { - return this.fetchNames; - } - - public String getSingleFetchName() { - Validate.isTrue(this.fetchNames.size() == 1, "Assumes a single fetch output"); - return this.fetchNames.get(0); - } - - public boolean isAutoBinding() { - return this.autoBinding; - } - - public AbstractGraphRunner disableAutoBinding() { - this.autoBinding = false; - return this; - } - - public AbstractGraphRunner enableAutoBinding() { - if (this.getFeedNames().size() != 1) { - throw new IllegalArgumentException("Auto-binding is permitted for Graphs with single input feed, but " + - " found: " + this.getFeedNames()); - } - this.autoBinding = true; - return this; - } - - @Override - public String toString() { - return String.format("(%s) -> (%s)", String.join(",", this.feedNames), - String.join(",", this.fetchNames)); - } -} diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AutoCloseableSession.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AutoCloseableSession.java deleted file mode 100644 index 29ae1ab50..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/AutoCloseableSession.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow; - -import org.tensorflow.Graph; -import org.tensorflow.Session; -import org.tensorflow.op.Ops; - -/** - * @author Christian Tzolov - */ -class AutoCloseableSession implements AutoCloseable { - - private Session session; - - /** - * Note: don't call this method inside the constructor. - */ - protected void init() { - Graph graph = this.doCreateGraph(); - this.doGraphDefinition(Ops.create(graph)); - this.session = new Session(graph); - } - - protected Graph doCreateGraph() { - return new Graph(); - } - - protected void doGraphDefinition(Ops tf) { - } - - protected Session getSession() { - if (this.session == null) { - init(); - } - return this.session; - } - - @Override - public void close() { - this.doClose(); - if (this.session != null) { - this.session.close(); - } - } - - protected void doClose() { - } -} diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/Functions.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/Functions.java deleted file mode 100644 index 0d2a22645..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/Functions.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow; - -import java.util.HashMap; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - -import org.tensorflow.Tensor; - -/** - * @author Christian Tzolov - */ -public final class Functions { - - private Functions() { - - } - - /** - * On every function call enrich the input tensorMap with an addition (tensorName, tensor) pair. - * - * @param tensorName tensor key to use in the map - * @param tensor new Tensor to add to the map - * @return Returns a copy of the input tensorMap enriched with the provided (tensorName, tensor). - */ - public static Function>, Map>> enrichWith( - String tensorName, Tensor tensor) { - return tensorMap -> enrich(tensorMap, tensorName, tensor); - } - - /** - * On function call retrieves a named tensor from the provided {@link GraphRunnerMemory} and uses it to enrich - * the input tensorMap. - * @param memory GraphRunnerMemory to retrieve the tensor from - * @param tensorName name of the tensor in GraphRunnerMemory to retrieve. - * @return Returns copy of the input tensorMap enriched with the tensor from the memory. - */ - public static Function>, Map>> enrichFromMemory( - GraphRunnerMemory memory, String tensorName) { - return tensorMap -> enrich(tensorMap, tensorName, memory.getTensorMap().get(tensorName)); - } - - private static Map> enrich(Map> inputTensorMap, String key, Tensor value) { - Map> newMap = new HashMap<>(inputTensorMap); - newMap.put(key, value); - return newMap; - } - - /** - * Renames the tensor names in the incoming tensorMap with the providing mappings. - * - * @param mapping Pairs of From and To names. E.g. fromName1, toName1, fromName2, toName2, ... fromNameN, toNameN - * Must be an even number. - * @return Map that renames the input tensorMap entries according to the mapping provided - */ - public static Function>, Map>> rename(String... mapping) { - - Map mappingMap = new HashMap<>(); - for (int i = 0; i < mapping.length; i = i + 2) { - mappingMap.put(mapping[i], mapping[i + 1]); - } - - return tensorMap -> tensorMap.entrySet().stream() - .filter(e -> mappingMap.containsKey(e.getKey())) - .collect(Collectors.toMap( - kv -> mappingMap.get(kv.getKey()), - kv -> kv.getValue() - )); - } -} diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphDefinition.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphDefinition.java deleted file mode 100644 index 1549df2f7..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphDefinition.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow; - -import org.tensorflow.op.Ops; - -/** - * @author Christian Tzolov - */ -@FunctionalInterface -public interface GraphDefinition { - void defineGraph(Ops tf); -} diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunner.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunner.java deleted file mode 100644 index e2df225d2..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunner.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow; - -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.lang3.Validate; -import org.tensorflow.SavedModelBundle; -import org.tensorflow.Session; -import org.tensorflow.op.Ops; - - -/** - * @author Christian Tzolov - */ -public class GraphRunner extends AbstractGraphRunner implements AutoCloseable { - - private SavedModelBundle savedModelBundle; - private AutoCloseableSession autoCloseableSession; - - public GraphRunner(List feedNames, String fetchedName) { - super(feedNames, Arrays.asList(fetchedName)); - } - - public GraphRunner(String feedName, List fetchedNames) { - super(Arrays.asList(feedName), fetchedNames); - } - - public GraphRunner(String feedName, String fetchedName) { - super(feedName, fetchedName); - } - - public GraphRunner(List feedNames, List fetchedNames) { - super(feedNames, fetchedNames); - } - - - @Override - public Session doGetSession() { - - if (this.autoCloseableSession != null && this.savedModelBundle != null) { - throw new IllegalStateException("Either SavedModel or GraphDefinition can be set! But both are set!"); - } - - if (this.autoCloseableSession != null) { - return this.autoCloseableSession.getSession(); - } - - if (this.savedModelBundle != null) { - return this.savedModelBundle.session(); - } - - throw new IllegalStateException("Either SavedModel or GraphDefinition can be set! None found"); - } - - public GraphRunner withGraphDefinition(GraphDefinition graphDefinition) { - Validate.isTrue(this.savedModelBundle == null, "Either SavedModel or GraphDefinition can be set! " + - "SavedModelBundle is found: " + this.savedModelBundle); - - this.autoCloseableSession = new AutoCloseableSession() { - @Override - protected void doGraphDefinition(Ops tf) { - graphDefinition.defineGraph(tf); - } - }; - - return this; - } - - public GraphRunner withSavedModel(String savedModelDir, String... tags) { - Validate.isTrue(this.autoCloseableSession == null, "Either SavedModel or GraphDefinition can be set! " + - "AutoCloseableSession is found: " + this.autoCloseableSession); - this.savedModelBundle = SavedModelBundle.load(savedModelDir, tags); - return this; - } - - @Override - public String toString() { - return String.format("(%s) -> (%s)", String.join(",", this.getFeedNames()), - String.join(",", this.getFetchNames())); - } - - @Override - public void close() { - if (this.savedModelBundle != null) { - this.savedModelBundle.close(); - } - if (this.autoCloseableSession != null) { - this.autoCloseableSession.close(); - } - } -} diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunnerMemory.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunnerMemory.java deleted file mode 100644 index 41a5acf8a..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/GraphRunnerMemory.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow; - -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; - -import org.pcollections.HashTreePMap; -import org.pcollections.PMap; -import org.tensorflow.Tensor; - -import org.springframework.cloud.fn.common.tensorflow.util.AutoCloseables; - - -/** - * Keeps all tensorMap input parameters. - */ -public class GraphRunnerMemory implements Function>, Map>>, AutoCloseable { - - private AtomicReference>> tensorMap = new AtomicReference<>(HashTreePMap.empty()); - - public Map> getTensorMap() { - return tensorMap.get(); - } - - @Override - public Map> apply(Map> tensorMap) { - this.tensorMap.getAndUpdate(pmap -> pmap.plusAll(tensorMap)); - return tensorMap; - } - - @Override - public void close() { - AutoCloseables.all(this.tensorMap.get()); - //this.tensorMap.get().clear(); - } -} - diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/ProtoBufGraphDefinition.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/ProtoBufGraphDefinition.java deleted file mode 100644 index e79be4b13..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/ProtoBufGraphDefinition.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow; - -import java.util.Iterator; - -import org.tensorflow.Graph; -import org.tensorflow.Operation; -import org.tensorflow.op.Ops; - -import org.springframework.cloud.fn.common.tensorflow.util.CachedModelExtractor; -import org.springframework.cloud.fn.common.tensorflow.util.ModelExtractor; -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.core.io.Resource; - -/** - * @author Christian Tzolov - */ -public class ProtoBufGraphDefinition implements GraphDefinition { - - /** - * Location of the pre-trained model archive. - */ - private final Resource modelLocation; - - /** - * If set true the pre-trained model is cached on the local file system. - */ - private final boolean cacheModel; - - public ProtoBufGraphDefinition(String modelUri, boolean cacheModel) { - this(new DefaultResourceLoader().getResource(modelUri), cacheModel); - } - - public ProtoBufGraphDefinition(Resource modelLocation, boolean cacheModel) { - this.modelLocation = modelLocation; - this.cacheModel = cacheModel; - } - - @Override - public void defineGraph(Ops tf) { - // Extract the pre-trained model as byte array. - byte[] model = this.cacheModel ? new CachedModelExtractor().getModel(this.modelLocation) - : new ModelExtractor().getModel(this.modelLocation); - // Import the pre-trained model - ((Graph) tf.scope().env()).importGraphDef(model); - //try { - // ((Graph) tf.scope().env()).importGraphDef(GraphDef.parseFrom(model)); - //} - //catch (InvalidProtocolBufferException e) { - // throw new RuntimeException(e); - //} - - Graph graph = ((Graph) tf.scope().env()); - Iterator ops = graph.operations(); - while (ops.hasNext()) { - System.out.println(ops.next().name()); - } - } -} diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/GraphicsUtils.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/GraphicsUtils.java deleted file mode 100644 index d1e7a64f6..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/GraphicsUtils.java +++ /dev/null @@ -1,639 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow.deprecated; - -import java.awt.BasicStroke; -import java.awt.Color; -import java.awt.Font; -import java.awt.FontMetrics; -import java.awt.Graphics2D; -import java.awt.Image; -import java.awt.RenderingHints; -import java.awt.Stroke; -import java.awt.geom.Rectangle2D; -import java.awt.image.BufferedImage; -import java.awt.image.DataBufferByte; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; - -import javax.imageio.ImageIO; - -import org.apache.commons.io.IOUtils; - -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.core.io.Resource; - - -/** - * Utility class used to provide some handy image manipulation functions. Among others it can provide contrast colors - * for image annotation labels and bounding boxes as well as functionality to draw later. - * - * @author Christian Tzolov - */ -public final class GraphicsUtils { - - private GraphicsUtils() { - } - - /** - * Default DEFAULT_FONT used in image label annotation. - */ - private static final Font DEFAULT_FONT = new Font("arial", Font.PLAIN, 12); - - /** - * Bounding box default line thickness. - */ - private static final float LINE_THICKNESS = 2; - - /** - * Color used when no multi-color is used. - */ - private static final Color AGNOSTIC_COLOR = new Color(167, 252, 0); - - /** - * in labels text offset. - */ - public static final int TITLE_OFFSET = 3; - - - /** - * Predefined contrasting colors used when drawing multiple objects in the same image. - */ - public static final Color aliceblue = new Color(240, 248, 255); /* color */ - /** Color. **/ - public static final Color antiquewhite = new Color(250, 235, 215); - /** Color. **/ - public static final Color aqua = new Color(0, 255, 255); // color - /** Color. **/ - public static final Color aquamarine = new Color(127, 255, 212); // color - /** Color. **/ - public static final Color azure = new Color(240, 255, 255); // color - /** Color. **/ - public static final Color beige = new Color(245, 245, 220); // color - /** Color. **/ - public static final Color bisque = new Color(255, 228, 196); - /** Color. **/ - public static final Color black = new Color(0, 0, 0); - /** Color. **/ - public static final Color blanchedalmond = new Color(255, 255, 205); - /** Color. **/ - public static final Color blue = new Color(0, 0, 255); - /** Color. **/ - public static final Color blueviolet = new Color(138, 43, 226); - /** Color. **/ - public static final Color brown = new Color(165, 42, 42); - /** Color. **/ - public static final Color burlywood = new Color(222, 184, 135); - /** Color. **/ - public static final Color cadetblue = new Color(95, 158, 160); - /** Color. **/ - public static final Color chartreuse = new Color(127, 255, 0); - /** Color. **/ - public static final Color chocolate = new Color(210, 105, 30); - /** Color. **/ - public static final Color coral = new Color(255, 127, 80); - /** Color. **/ - public static final Color cornflowerblue = new Color(100, 149, 237); - /** Color. **/ - public static final Color cornsilk = new Color(255, 248, 220); - /** Color. **/ - public static final Color crimson = new Color(220, 20, 60); - /** Color. **/ - public static final Color cyan = new Color(0, 255, 255); - /** Color. **/ - public static final Color darkblue = new Color(0, 0, 139); - /** Color. **/ - public static final Color darkcyan = new Color(0, 139, 139); - /** Color. **/ - public static final Color darkgoldenrod = new Color(184, 134, 11); - /** Color. **/ - public static final Color darkgray = new Color(169, 169, 169); - /** Color. **/ - public static final Color darkgreen = new Color(0, 100, 0); - /** Color. **/ - public static final Color darkkhaki = new Color(189, 183, 107); - /** Color. **/ - public static final Color darkmagenta = new Color(139, 0, 139); - /** Color. **/ - public static final Color darkolivegreen = new Color(85, 107, 47); - /** Color. **/ - public static final Color darkorange = new Color(255, 140, 0); - /** Color. **/ - public static final Color darkorchid = new Color(153, 50, 204); - /** Color. **/ - public static final Color darkred = new Color(139, 0, 0); - /** Color. **/ - public static final Color darksalmon = new Color(233, 150, 122); - /** Color. **/ - public static final Color darkseagreen = new Color(143, 188, 143); - /** Color. **/ - public static final Color darkslateblue = new Color(72, 61, 139); - /** Color. **/ - public static final Color darkslategray = new Color(47, 79, 79); - /** Color. **/ - public static final Color darkturquoise = new Color(0, 206, 209); - /** Color. **/ - public static final Color darkviolet = new Color(148, 0, 211); - /** Color. **/ - public static final Color deeppink = new Color(255, 20, 147); - /** Color. **/ - public static final Color deepskyblue = new Color(0, 191, 255); - /** Color. **/ - public static final Color dimgray = new Color(105, 105, 105); - /** Color. **/ - public static final Color dodgerblue = new Color(30, 144, 255); - /** Color. **/ - public static final Color firebrick = new Color(178, 34, 34); - /** Color. **/ - public static final Color floralwhite = new Color(255, 250, 240); - /** Color. **/ - public static final Color forestgreen = new Color(34, 139, 34); - /** Color. **/ - public static final Color fuchsia = new Color(255, 0, 255); - /** Color. **/ - public static final Color gainsboro = new Color(220, 220, 220); - /** Color. **/ - public static final Color ghostwhite = new Color(248, 248, 255); - /** Color. **/ - public static final Color gold = new Color(255, 215, 0); - /** Color. **/ - public static final Color goldenrod = new Color(218, 165, 32); - /** Color. **/ - public static final Color gray = new Color(128, 128, 128); - /** Color. **/ - public static final Color green = new Color(0, 128, 0); - /** Color. **/ - public static final Color greenyellow = new Color(173, 255, 47); - /** Color. **/ - public static final Color honeydew = new Color(240, 255, 240); - /** Color. **/ - public static final Color hotpink = new Color(255, 105, 180); - /** Color. **/ - public static final Color indianred = new Color(205, 92, 92); - /** Color. **/ - public static final Color indigo = new Color(75, 0, 130); - /** Color. **/ - public static final Color ivory = new Color(255, 240, 240); - /** Color. **/ - public static final Color khaki = new Color(240, 230, 140); - /** Color. **/ - public static final Color lavender = new Color(230, 230, 250); - /** Color. **/ - public static final Color lavenderblush = new Color(255, 240, 245); - /** Color. **/ - public static final Color lawngreen = new Color(124, 252, 0); - /** Color. **/ - public static final Color lemonchiffon = new Color(255, 250, 205); - /** Color. **/ - public static final Color lightblue = new Color(173, 216, 230); - /** Color. **/ - public static final Color lightcoral = new Color(240, 128, 128); - /** Color. **/ - public static final Color lightcyan = new Color(224, 255, 255); - /** Color. **/ - public static final Color lightgoldenrodyellow = new Color(250, 250, 210); - /** Color. **/ - public static final Color lightgreen = new Color(144, 238, 144); - /** Color. **/ - public static final Color lightgrey = new Color(211, 211, 211); - /** Color. **/ - public static final Color lightpink = new Color(255, 182, 193); - /** Color. **/ - public static final Color lightsalmon = new Color(255, 160, 122); - /** Color. **/ - public static final Color lightseagreen = new Color(32, 178, 170); - /** Color. **/ - public static final Color lightskyblue = new Color(135, 206, 250); - /** Color. **/ - public static final Color lightslategray = new Color(119, 136, 153); - /** Color. **/ - public static final Color lightsteelblue = new Color(176, 196, 222); - /** Color. **/ - public static final Color lightyellow = new Color(255, 255, 224); - /** Color. **/ - public static final Color lime = new Color(0, 255, 0); - /** Color. **/ - public static final Color limegreen = new Color(50, 205, 50); - /** Color. **/ - public static final Color linen = new Color(250, 240, 230); - /** Color. **/ - public static final Color magenta = new Color(255, 0, 255); - /** Color. **/ - public static final Color maroon = new Color(128, 0, 0); - /** Color. **/ - public static final Color mediumaquamarine = new Color(102, 205, 170); - /** Color. **/ - public static final Color mediumblue = new Color(0, 0, 205); - /** Color. **/ - public static final Color mediumorchid = new Color(186, 85, 211); - /** Color. **/ - public static final Color mediumpurple = new Color(147, 112, 219); - /** Color. **/ - public static final Color mediumseagreen = new Color(60, 179, 113); - /** Color. **/ - public static final Color mediumslateblue = new Color(123, 104, 238); - /** Color. **/ - public static final Color mediumspringgreen = new Color(0, 250, 154); - /** Color. **/ - public static final Color mediumturquoise = new Color(72, 209, 204); - /** Color. **/ - public static final Color mediumvioletred = new Color(199, 21, 133); - /** Color. **/ - public static final Color midnightblue = new Color(25, 25, 112); - /** Color. **/ - public static final Color mintcream = new Color(245, 255, 250); - /** Color. **/ - public static final Color mistyrose = new Color(255, 228, 225); - /** Color. **/ - public static final Color mocassin = new Color(255, 228, 181); - /** Color. **/ - public static final Color navajowhite = new Color(255, 222, 173); - /** Color. **/ - public static final Color navy = new Color(0, 0, 128); - /** Color. **/ - public static final Color oldlace = new Color(253, 245, 230); - /** Color. **/ - public static final Color olive = new Color(128, 128, 0); - /** Color. **/ - public static final Color olivedrab = new Color(107, 142, 35); - /** Color. **/ - public static final Color orange = new Color(255, 165, 0); - /** Color. **/ - public static final Color orangered = new Color(255, 69, 0); - /** Color. **/ - public static final Color orchid = new Color(218, 112, 214); - /** Color. **/ - public static final Color palegoldenrod = new Color(238, 232, 170); - /** Color. **/ - public static final Color palegreen = new Color(152, 251, 152); - /** Color. **/ - public static final Color paleturquoise = new Color(175, 238, 238); - /** Color. **/ - public static final Color palevioletred = new Color(219, 112, 147); - /** Color. **/ - public static final Color papayawhip = new Color(255, 239, 213); - /** Color. **/ - public static final Color peachpuff = new Color(255, 218, 185); - /** Color. **/ - public static final Color peru = new Color(205, 133, 63); - /** Color. **/ - public static final Color pink = new Color(255, 192, 203); - /** Color. **/ - public static final Color plum = new Color(221, 160, 221); - /** Color. **/ - public static final Color powderblue = new Color(176, 224, 230); - /** Color. **/ - public static final Color purple = new Color(128, 0, 128); - /** Color. **/ - public static final Color red = new Color(255, 0, 0); - /** Color. **/ - public static final Color rosybrown = new Color(188, 143, 143); - /** Color. **/ - public static final Color royalblue = new Color(65, 105, 225); - /** Color. **/ - public static final Color saddlebrown = new Color(139, 69, 19); - /** Color. **/ - public static final Color salmon = new Color(250, 128, 114); - /** Color. **/ - public static final Color sandybrown = new Color(244, 164, 96); - /** Color. **/ - public static final Color seagreen = new Color(46, 139, 87); - /** Color. **/ - public static final Color seashell = new Color(255, 245, 238); - /** Color. **/ - public static final Color sienna = new Color(160, 82, 45); - /** Color. **/ - public static final Color silver = new Color(192, 192, 192); - /** Color. **/ - public static final Color skyblue = new Color(135, 206, 235); - /** Color. **/ - public static final Color slateblue = new Color(106, 90, 205); - /** Color. **/ - public static final Color slategray = new Color(112, 128, 144); - /** Color. **/ - public static final Color snow = new Color(255, 250, 250); - /** Color. **/ - public static final Color springgreen = new Color(0, 255, 127); - /** Color. **/ - public static final Color steelblue = new Color(70, 138, 180); - /** Color. **/ - public static final Color tan = new Color(210, 180, 140); - /** Color. **/ - public static final Color teal = new Color(0, 128, 128); - /** Color. **/ - public static final Color thistle = new Color(216, 191, 216); - /** Color. **/ - public static final Color tomato = new Color(253, 99, 71); - /** Color. **/ - public static final Color turquoise = new Color(64, 224, 208); - /** Color. **/ - public static final Color violet = new Color(238, 130, 238); - /** Color. **/ - public static final Color wheat = new Color(245, 222, 179); - /** Color. **/ - public static final Color white = new Color(255, 255, 255); - /** Color. **/ - public static final Color whitesmoke = new Color(245, 245, 245); - /** Color. **/ - public static final Color yellow = new Color(255, 255, 0); - /** Color. **/ - public static final Color yellowgreen = new Color(154, 205, 50); - - /** - * Limbs color list. - */ - public static final Color[] LIMBS_COLORS = new Color[] { - new Color(153, 0, 0), // 0 (1 -> 2) - new Color(153, 51, 0), // 1 (1 -> 5) - new Color(153, 102, 0), // 2 (2 -> 3) - new Color(153, 153, 0), // 3 (3 -> 4) - new Color(102, 153, 0), // 4 (5 -> 6) - new Color(51, 153, 0), // 5 (6 -> 7) - new Color(0, 153, 0), // 6 (1 -> 8) - new Color(0, 153, 51), // 7 (8 -> 9) - new Color(0, 153, 102), // 8 (9 -> 10) - new Color(0, 153, 153), // 9 (1 -> 11) - new Color(0, 102, 153), // 10 (11 -> 12) - new Color(0, 51, 153), // 11 (12 -> 13) - new Color(0, 0, 153), // 12 (1 -> 0) - new Color(51, 0, 153), // 13 (0 -> 14) - new Color(102, 0, 153), // 14 (14 -> 16) - new Color(153, 0, 153), // 15 (0 -> 15) - new Color(153, 0, 102), // 16 (15 -> 17) - - new Color(153, 0, 51), // 17 (2 -> 16) - new Color(153, 153, 153), // 18 (5 -> 17) - }; - - /** - * Constants lists. - */ - private static final Color[] CLASS_COLOR = new Color[] { - aliceblue, chartreuse, aqua, aquamarine, azure, beige, bisque, - blanchedalmond, blueviolet, burlywood, cadetblue, antiquewhite, - chocolate, coral, cornflowerblue, cornsilk, crimson, cyan, - darkcyan, darkgoldenrod, darkgray, darkkhaki, darkorange, - darkorchid, darksalmon, darkseagreen, darkturquoise, darkviolet, - deeppink, deepskyblue, dodgerblue, firebrick, floralwhite, - forestgreen, fuchsia, gainsboro, ghostwhite, gold, goldenrod, - salmon, tan, honeydew, hotpink, indianred, ivory, khaki, - lavender, lavenderblush, lawngreen, lemonchiffon, lightblue, - lightcoral, lightcyan, lightgoldenrodyellow, lightgreen, lightgrey, - lightgreen, lightpink, lightsalmon, lightseagreen, lightskyblue, - lightslategray, lightslategray, lightsteelblue, lightyellow, lime, - limegreen, linen, magenta, mediumaquamarine, mediumorchid, - mediumpurple, mediumseagreen, mediumslateblue, mediumspringgreen, - mediumturquoise, mediumvioletred, mintcream, mistyrose, mocassin, - navajowhite, oldlace, olive, olivedrab, orange, orangered, - orchid, palegoldenrod, palegreen, paleturquoise, palevioletred, - papayawhip, peachpuff, peru, pink, plum, powderblue, purple, - red, rosybrown, royalblue, saddlebrown, green, sandybrown, - seagreen, seashell, sienna, silver, skyblue, slateblue, - slategray, slategray, snow, springgreen, steelblue, greenyellow, - teal, thistle, tomato, turquoise, violet, wheat, white, - whitesmoke, yellow, yellowgreen - }; - - /** - * List of constants. - */ - public static final Color[] CLASS_COLOR2 = new Color[] { - yellow, yellowgreen, turquoise, springgreen, skyblue, slateblue, red, violet, olivedrab, royalblue, - darkorange, mediumblue, deeppink, chartreuse, orchid, palegreen, aqua, orange, navy - }; - - /** - * Return different color for each Id. It rotates when the ID exceeds the number of predefined colors. - * @param id the unique id to pick color for. - * @return a distinct color computed from the input #id - */ - public static Color getClassColor(int id) { - return CLASS_COLOR[id % CLASS_COLOR.length]; - } - - /** - * Augments the input image fromMemory a labeled rectangle (e.g. bounding box) fromMemory coordinates: (x1, y1, x2, y2). - * - * @param image Input image to be augmented fromMemory labeled rectangle. - * @param cid Unique id used to select the color of the rectangle. Used only if the colorAgnostic is set to false. - * @param title rectangle title - * @param x1 top left corner for the bounding box - * @param y1 top left corner for the bounding box - * @param x2 bottom right corner for the bounding box - * @param y2 bottom right corner for the bounding box - * @param colorAgnostic If set to false the cid is used to select the bounding box color. Uses the - * AGNOSTIC_COLOR otherwise. - */ - public static void drawBoundingBox(BufferedImage image, int cid, String title, int x1, int y1, int x2, int y2, - boolean colorAgnostic) { - - Graphics2D g = image.createGraphics(); - g.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING, RenderingHints.VALUE_TEXT_ANTIALIAS_ON); - - Color labelColor = colorAgnostic ? AGNOSTIC_COLOR : GraphicsUtils.getClassColor(cid); - g.setColor(labelColor); - - g.setFont(DEFAULT_FONT); - FontMetrics fontMetrics = g.getFontMetrics(); - - Stroke oldStroke = g.getStroke(); - g.setStroke(new BasicStroke(LINE_THICKNESS)); - g.drawRect(x1, y1, (x2 - x1), (y2 - y1)); - g.setStroke(oldStroke); - - Rectangle2D rect = fontMetrics.getStringBounds(title, g); - - g.setColor(labelColor); - g.fillRect(x1, y1 - fontMetrics.getAscent(), - (int) rect.getWidth() + 2 * TITLE_OFFSET, (int) rect.getHeight()); - - g.setColor(getTextColor(labelColor)); - g.drawString(title, x1 + TITLE_OFFSET, y1); - } - - /** - * Depends on the darkness of the background, pick a dark or light DEFAULT_FONT color. - * @param backGroundColor background color within which the text is drawn - * @return a text color, that contrast to the given background color. - */ - private static Color getTextColor(Color backGroundColor) { - double y = (299 * backGroundColor.getRed() + 587 * backGroundColor.getGreen() + - 114 * backGroundColor.getBlue()) / 1000; - return y >= 128 ? Color.black : Color.white; - } - - public static BufferedImage createMaskImage(float[][] maskPixels, - int scaledWidth, int scaledHeight, Color maskColor) { - - int maskWidth = maskPixels.length; - int maskHeight = maskPixels[0].length; - int[] maskArray = new int[maskWidth * maskHeight]; - int k = 0; - for (int i = 0; i < maskHeight; i++) { - for (int j = 0; j < maskWidth; j++) { - maskArray[k++] = grayScaleToARGB(maskPixels[i][j], maskColor); - } - } - - // Turn the pixel array into image; - BufferedImage maskImage = new BufferedImage(maskWidth, maskHeight, BufferedImage.TYPE_INT_ARGB); - maskImage.setRGB(0, 0, maskWidth, maskHeight, maskArray, 0, maskWidth); - - // Stretch the image to fit the target box width and height! - return toBufferedImage(maskImage.getScaledInstance(scaledWidth, scaledHeight, Image.SCALE_DEFAULT)); - } - - /** - * Converts an gray scale (e.g. value between 0 to 1) into ARGB. - * - * @param grayScale - value between 0 and 1 - * @param maskColor - desired mask color - * @return Returns a ARGB color based on the grayscale and the mask colors - */ - private static int grayScaleToARGB(float grayScale, Color maskColor) { - if (maskColor != null) { - float r = col(maskColor.getRed(), grayScale); - float g = col(maskColor.getGreen(), grayScale); - float b = col(maskColor.getBlue(), grayScale); - float t = grayScale * 0.7f; - return new Color(r, g, b, t).getRGB(); - } - - return new Color(grayScale, grayScale, grayScale, grayScale).getRGB(); - } - - private static float col(int channelColor, float grayScale) { - //return ((float) channelColor / 255) * grayScale; - return ((float) channelColor / 255); - } - - public static BufferedImage toBufferedImage(Image img) { - //if (img instanceof BufferedImage) { - // return (BufferedImage) img; - //} - - // Create a buffered image fromMemory transparency - BufferedImage bimage = new BufferedImage(img.getWidth(null), img.getHeight(null), BufferedImage.TYPE_INT_ARGB); - - // Draw the image on to the buffered image - Graphics2D bGr = bimage.createGraphics(); - bGr.drawImage(img, 0, 0, null); - bGr.dispose(); - - // Return the buffered image - return bimage; - } - - public static BufferedImage overlayImages(BufferedImage bgImage, BufferedImage fgImage, int fgX, int fgY) { - // Foreground image width and height cannot be greater than background image width and height. - if (fgImage.getHeight() > bgImage.getHeight() - || fgImage.getWidth() > fgImage.getWidth()) { - throw new IllegalArgumentException( - "Foreground Image Is Bigger In One or Both Dimensions" - + "nCannot proceed fromMemory overlay." - + "nn Please use smaller Image for foreground"); - } - - // Create a Graphics from the background image - Graphics2D g = bgImage.createGraphics(); - - //Set Antialias Rendering - g.setRenderingHint(RenderingHints.KEY_ANTIALIASING, RenderingHints.VALUE_ANTIALIAS_ON); - - //Draw background image at location (0,0) - g.drawImage(bgImage, 0, 0, null); - - // Draw foreground image at location (fgX,fgy) - g.drawImage(fgImage, fgX, fgY, null); - - g.dispose(); - return bgImage; - } - - /** - * Convert {@link BufferedImage} to byte array. - * - * @param image the image to be converted - * @param format the output image format - * @return New array of bytes - */ - public static byte[] toImageByteArray(BufferedImage image, String format) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - - try { - ImageIO.write(image, format, baos); - byte[] bytes = baos.toByteArray(); - return bytes; - } - catch (IOException e) { - throw new IllegalStateException(e); - } - finally { - try { - baos.close(); - } - catch (IOException e) { - throw new IllegalStateException(e); - } - } - } - - /** - * - * @param bufferedImage buffer to be converted in to raw array - * @return flat byte array representing the buffered image - */ - public static byte[] toRawByteArray(BufferedImage bufferedImage) { - return ((DataBufferByte) bufferedImage.getRaster().getDataBuffer()).getData(); - } - - /** - * BufferedImage.TYPE_3BYTE_BGR, BufferedImage.TYPE_3BYTE_BGR. - * @param image image to be converted into buffer - * @param imageType desired type - * @return BufferedImage - */ - public static BufferedImage toBufferedImageType(BufferedImage image, int imageType) { - if (image.getType() == imageType) { - return image; - } - BufferedImage outputImage = new BufferedImage(image.getWidth(), image.getHeight(), imageType); - outputImage.getGraphics().drawImage(image, 0, 0, null); - return outputImage; - } - - public static byte[] toImageToBytes(String imageUri) throws IOException { - try (InputStream is = new DefaultResourceLoader().getResource(imageUri).getInputStream()) { - return IOUtils.toByteArray(is); - } - } - - /** - * Loads a resource as byte array. Supports http:, file: and classpath: URI schemas - * @param resourceUri resource URI - * @return Returns resources referred by the resourceUri as a byte array - * @throws IOException failure due to missing resource or invalid URI - */ - public static byte[] loadAsByteArray(String resourceUri) throws IOException { - Resource expectedPoseResponse = new DefaultResourceLoader().getResource(resourceUri); - try (InputStream is = expectedPoseResponse.getInputStream()) { - return IOUtils.toByteArray(is); - } - } - -} diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/JsonMapperFunction.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/JsonMapperFunction.java deleted file mode 100644 index 62463bf19..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/JsonMapperFunction.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow.deprecated; - -import java.util.function.Function; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * Maps domain objects into JSON strings. - * - * @author Christian Tzolov - */ -public class JsonMapperFunction implements Function { - - private static final Log logger = LogFactory.getLog(JsonMapperFunction.class); - - @Override - public String apply(Object o) { - try { - return new ObjectMapper().writeValueAsString(o); - } - catch (JsonProcessingException e) { - logger.error("Failed to encode the object detections into JSON message", e); - } - return "ERROR"; - - } -} diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/TensorFlowService.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/TensorFlowService.java deleted file mode 100644 index fc750bfd0..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/TensorFlowService.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.cloud.fn.common.tensorflow.deprecated; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.function.Function; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.tensorflow.Graph; -import org.tensorflow.Session; -import org.tensorflow.Session.Runner; -import org.tensorflow.Tensor; - -import org.springframework.cloud.fn.common.tensorflow.util.CachedModelExtractor; -import org.springframework.cloud.fn.common.tensorflow.util.ModelExtractor; -import org.springframework.core.io.Resource; - -/** - * @author Christian Tzolov - */ -public class TensorFlowService implements Function>, Map>>, AutoCloseable { - - private static final Log logger = LogFactory.getLog(TensorFlowService.class); - - private final Session session; - private final List fetchedNames; - private final boolean autoCloseFeedTensors; - - public TensorFlowService(Resource modelLocation, List fetchedNames) { - this(modelLocation, fetchedNames, false); - } - - public TensorFlowService(Resource modelLocation, List fetchedNames, boolean cacheModel) { - this(modelLocation, fetchedNames, cacheModel, false); - } - - public TensorFlowService(Resource modelLocation, List fetchedNames, boolean cacheModel, - boolean autoCloseFeedTensors) { - - if (logger.isInfoEnabled()) { - logger.info("Loading TensorFlow graph model: " + modelLocation); - } - - this.autoCloseFeedTensors = autoCloseFeedTensors; - this.fetchedNames = fetchedNames; - Graph graph = new Graph(); - byte[] model = cacheModel ? new CachedModelExtractor().getModel(modelLocation) : new ModelExtractor().getModel(modelLocation); - graph.importGraphDef(model); - this.session = new Session(graph); - } - - /** - * Evaluates a pre-trained tensorflow model (encoded as {@link Graph}). Use the feeds parameter to feed in the - * model input data and fetch-names to specify the output tensors. - * - * @param feeds Named map of input tensors. - * @return Returns the computed output tensors. The names of the output tensors is defined by the fetchedNames - * argument - */ - @Override - public Map> apply(Map> feeds) { - - Runner runner = this.session.runner(); - - // Keep tensor references to release them in the finally block - Tensor[] feedTensors = new Tensor[feeds.size()]; - try { - // Feed in the input named tensors - int inputIndex = 0; - for (Entry> e : feeds.entrySet()) { - String feedName = e.getKey(); - feedTensors[inputIndex] = e.getValue(); - runner = runner.feed(feedName, feedTensors[inputIndex]); - inputIndex++; - } - - // Set the tensor name to be fetched after the evaluation - for (String fetchName : this.fetchedNames) { - runner.fetch(fetchName); - } - - // Evaluate the input - List> outputTensors = runner.run(); - - // Extract the output tensors - Map> outTensorMap = new HashMap<>(); - for (int outputIndex = 0; outputIndex < this.fetchedNames.size(); outputIndex++) { - outTensorMap.put(this.fetchedNames.get(outputIndex), outputTensors.get(outputIndex)); - } - return outTensorMap; - } - finally { - if (this.autoCloseFeedTensors) { - // Release all feed tensors - for (Tensor tensor : feedTensors) { - if (tensor != null) { - tensor.close(); - } - } - } - } - - } - - @Override - public void close() { - logger.info("Close TensorFlow Session!"); - if (this.session != null) { - this.session.close(); - } - } -} diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/AutoCloseables.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/AutoCloseables.java deleted file mode 100644 index 68b4ce849..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/AutoCloseables.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow.util; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Map; -import java.util.Objects; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Utilities for AutoCloseable classes. - * Based on the Apache Drill AutoCloseables implementation. - */ -public final class AutoCloseables { - - private AutoCloseables() { - - } - - private static final Logger LOGGER = LoggerFactory.getLogger(AutoCloseables.class); - - public static AutoCloseable all(final Collection autoCloseables) { - return () -> close(autoCloseables); - } - - public static AutoCloseable all(final Map... autoCloseables) { - return () -> close(autoCloseables); - } - - /** - * Closes all autoCloseables if not null and suppresses exceptions by adding them to t. - * @param t the throwable to add suppressed exception to - * @param autoCloseables the closeables to close - */ - public static void close(Throwable t, AutoCloseable... autoCloseables) { - close(t, Arrays.asList(autoCloseables)); - } - - /** - * Closes all autoCloseables if not null and suppresses exceptions by adding them to t. - * @param t the throwable to add suppressed exception to - * @param autoCloseables the closeables to close - */ - public static void close(Throwable t, Collection autoCloseables) { - try { - close(autoCloseables); - } - catch (Exception e) { - t.addSuppressed(e); - } - } - - /** - * Closes all autoCloseables if not null and suppresses subsequent exceptions if more than one. - * @param autoCloseables the closeables to close - */ - public static void close(AutoCloseable... autoCloseables) throws Exception { - close(Arrays.asList(autoCloseables)); - } - - /** - * Closes all autoCloseables if not null and suppresses subsequent exceptions if more than one. - * @param autoCloseables the closeables to close - */ - public static void close(Iterable autoCloseables) throws Exception { - Exception topLevelException = null; - for (AutoCloseable closeable : autoCloseables) { - try { - if (closeable != null) { - closeable.close(); - } - } - catch (Exception e) { - if (topLevelException == null) { - topLevelException = e; - } - else { - topLevelException.addSuppressed(e); - } - } - } - if (topLevelException != null) { - throw topLevelException; - } - } - - /** - * Closes all autoCloseables entry values if not null and suppresses subsequent exceptions if more than one. - * @param closableMaps the closeables to close - */ - public static void close(Map... closableMaps) throws Exception { - Exception topLevelException = null; - for (Map closableMap : closableMaps) { - - for (Object key : closableMap.keySet()) { - AutoCloseable closeable = closableMap.get(key); - try { - if (closeable != null) { - closeable.close(); - } - } - catch (Exception e) { - if (topLevelException == null) { - topLevelException = e; - } - else { - topLevelException.addSuppressed(e); - } - } - } - - closableMap.keySet(); - } - if (topLevelException != null) { - throw topLevelException; - } - - } - - /** - * Close all without caring about thrown exceptions. - * @param closeables - array containing auto closeables - */ - public static void closeSilently(AutoCloseable... closeables) { - Arrays.stream(closeables).filter(Objects::nonNull) - .forEach(target -> { - try { - target.close(); - } - catch (Exception e) { - LOGGER.warn(String.format("Exception was thrown while closing auto closeable: %s", target), e); - } - }); - } - -} diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/CachedModelExtractor.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/CachedModelExtractor.java deleted file mode 100644 index f3414984f..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/CachedModelExtractor.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow.util; - -import java.io.File; -import java.io.FileInputStream; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Validate; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.core.io.Resource; - -/** - * Extends the {@link ModelExtractor} to allow keeping a local copy (cache) of the loaded model (protobuf) files. - * @author Christian Tzolov - */ -public class CachedModelExtractor extends ModelExtractor { - - private static final Log logger = LogFactory.getLog(CachedModelExtractor.class); - - /** - * Parent folder under which the model files are cached. - */ - public String cacheRootDirectory = new File(System.getProperty("java.io.tmpdir"), "mind-model").getAbsolutePath(); - - public String getCacheRootDirectory() { - return cacheRootDirectory; - } - - public void setCacheRootDirectory(String cacheRootDirectory) { - this.cacheRootDirectory = cacheRootDirectory; - } - - public CachedModelExtractor() { - super(); - } - - public CachedModelExtractor(String frozenGraphFileExtension) { - super(frozenGraphFileExtension); - } - - @Override - public byte[] getModel(String modelUri) { - return this.getModel(new DefaultResourceLoader().getResource(modelUri)); - } - - @Override - public byte[] getModel(Resource modelResource) { - try { - File rootFolder = new File(this.cacheRootDirectory); - if (!rootFolder.exists()) { - logger.info("Create Model Cache root folder: " + rootFolder.getAbsolutePath()); - rootFolder.mkdirs(); - } - - Validate.isTrue(rootFolder.isDirectory(), "The cache root folder must be a Directory"); - - String fileName = modelResource.getFilename(); - String fragment = modelResource.getURI().getFragment(); - File cachedFile = StringUtils.isEmpty(fragment) ? new File(rootFolder, fileName) : - new File(rootFolder, fileName + "_" + fragment); - if (cachedFile.exists()) { - logger.info("Load model " + modelResource.toString() + " from cache: " + cacheRootDirectory); - return IOUtils.toByteArray(new FileInputStream(cachedFile)); - } - - byte[] model = super.getModel(modelResource); - - // cache the file - FileUtils.writeByteArrayToFile(cachedFile, model); - logger.info("Caching the " + modelResource.toString() + " model at: " + cachedFile); - - return model; - } - catch (Exception e) { - throw new IllegalStateException("Failed to extract a model from: " + modelResource.getDescription(), e); - } - } - - public void emptyModelCache() { - File rootFolder = new File(this.cacheRootDirectory); - if (rootFolder.exists()) { - logger.info("Empty Model Cache at:" + rootFolder.getAbsolutePath()); - rootFolder.delete(); - rootFolder.mkdirs(); - } - } -} diff --git a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/ModelExtractor.java b/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/ModelExtractor.java deleted file mode 100644 index 165024109..000000000 --- a/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/util/ModelExtractor.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow.util; - -import java.io.BufferedInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.X509Certificate; -import java.util.Optional; - -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLSession; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; - -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.ArchiveInputStream; -import org.apache.commons.compress.archivers.ArchiveStreamFactory; -import org.apache.commons.compress.compressors.CompressorInputStream; -import org.apache.commons.compress.compressors.CompressorStreamFactory; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.Validate; - -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.core.io.Resource; - - -/** - * Extracts a pre-trained (frozen) Tensorflow model URI into byte array. The 'http://', 'file://' and 'classpath://' - * URI schemas are supported. - * - * Models can be extract either from raw files or form compressed archives. When extracted from an archive the model - * file name can optionally be provided as an URI fragment. For example for resource: http://myarchive.tar.gz#model.pb - * the myarchive.tar.gz is traversed to uncompress and extract the model.pb file as byte array. - * If the file name is not provided as URI fragment then the first file in the archive with extension .pb is extracted. - * - * @author Christian Tzolov - */ -public class ModelExtractor { - - private static final String DEFAULT_FROZEN_GRAPH_FILE_EXTENSION = ".pb"; - - /** - * When an archive resource if referred, but no fragment URI is provided (to specify the target file name in - * the archive) then the extractor selects the first file in the archive with the extension that match - * the frozenGraphFileExtension (defaults to .pb). - */ - public final String frozenGraphFileExtension; - - public ModelExtractor() { - this(DEFAULT_FROZEN_GRAPH_FILE_EXTENSION); - } - - public ModelExtractor(String frozenGraphFileExtension) { - this.frozenGraphFileExtension = frozenGraphFileExtension; - } - - public byte[] getModel(String modelUri) { - return getModel(new DefaultResourceLoader().getResource(modelUri)); - } - - public byte[] getModel(Resource modelResource) { - - Validate.notNull(modelResource, "Not null model resource is required!"); - - try (InputStream is = modelResource.getInputStream(); InputStream bi = new BufferedInputStream(is)) { - - String[] archiveCompressor = detectArchiveAndCompressor(modelResource.getFilename()); - String archive = archiveCompressor[0]; - String compressor = archiveCompressor[1]; - String fragment = modelResource.getURI().getFragment(); - - - if (StringUtils.isNotBlank(compressor)) { - try (CompressorInputStream cis = new CompressorStreamFactory().createCompressorInputStream(compressor, bi)) { - if (StringUtils.isNotBlank(archive)) { - try (ArchiveInputStream ais = new ArchiveStreamFactory().createArchiveInputStream(archive, cis)) { - // Compressor fromMemory Archive - return findInArchiveStream(fragment, ais); - } - } - else { // Compressor only - return IOUtils.toByteArray(cis); - } - } - } - else if (StringUtils.isNotBlank(archive)) { // Archive only - try (ArchiveInputStream ais = new ArchiveStreamFactory().createArchiveInputStream(archive, bi)) { - return findInArchiveStream(fragment, ais); - } - } - else { - // No compressor nor Archive - return IOUtils.toByteArray(bi); - } - } - catch (Exception e) { - throw new IllegalStateException("Failed to extract a model from: " + modelResource.getDescription(), e); - } - } - - /** - * Traverses the Archive to find either an entry that matches the modelFileNameInArchive name (if not empty) or - * and entry that ends in .pb if the modelFileNameInArchive is empty. - * - * @param modelFileNameInArchive Optional name of the archive entry that represents the frozen model file. If empty - * the archive will be searched for the first entry that ends in .pb - * @param archive Archive stream to be traversed - * - */ - private byte[] findInArchiveStream(String modelFileNameInArchive, ArchiveInputStream archive) throws IOException { - ArchiveEntry entry; - while ((entry = archive.getNextEntry()) != null) { - //System.out.println(entry.getName() + " : " + entry.isDirectory()); - - if (archive.canReadEntryData(entry) && !entry.isDirectory()) { - if ((StringUtils.isNotBlank(modelFileNameInArchive) && entry.getName().endsWith(modelFileNameInArchive)) || - (!StringUtils.isNotBlank(modelFileNameInArchive) && entry.getName().endsWith(this.frozenGraphFileExtension))) { - return IOUtils.toByteArray(archive); - } - } - } - throw new IllegalArgumentException("No model is found in the archive"); - } - - /** - * Detect the Archive and the Compressor from the file extension. - * - * @param fileName File name with extension. - * @return Returns a tuple of the detected (Archive, Compressor). Null stands for not available - * archive or detector. The (null, null) response stands for no Archive or Compressor discovered. - */ - private String[] detectArchiveAndCompressor(String fileName) { - - String normalizedFileName = fileName.trim().toLowerCase(); - - if (normalizedFileName.endsWith(".tar.gz") - || normalizedFileName.endsWith(".tgz") - || normalizedFileName.endsWith(".taz")) { - return new String[] { ArchiveStreamFactory.TAR, CompressorStreamFactory.GZIP }; - } - else if (normalizedFileName.endsWith(".tar.bz2") - || normalizedFileName.endsWith(".tbz2") - || normalizedFileName.endsWith(".tbz")) { - return new String[] { ArchiveStreamFactory.TAR, CompressorStreamFactory.BZIP2 }; - } - else if (normalizedFileName.endsWith(".cpgz")) { - return new String[] { ArchiveStreamFactory.CPIO, CompressorStreamFactory.GZIP }; - } - else if (hasArchive(normalizedFileName)) { - return new String[] { findArchive(normalizedFileName).get(), null }; - } - else if (hasCompressor(normalizedFileName)) { - return new String[] { null, findCompressor(normalizedFileName).get() }; - } - else if (normalizedFileName.endsWith(".gzip")) { - return new String[] { null, CompressorStreamFactory.GZIP }; - } - else if (normalizedFileName.endsWith(".bz2") - || normalizedFileName.endsWith(".bz")) { - return new String[] { null, CompressorStreamFactory.BZIP2 }; - } - - // No archived/compressed - return new String[] { null, null }; - } - - private boolean hasArchive(String normalizedFileName) { - return findArchive(normalizedFileName).isPresent(); - } - - private Optional findArchive(String normalizedFileName) { - return new ArchiveStreamFactory().getInputStreamArchiveNames() - .stream().filter(arch -> normalizedFileName.endsWith("." + arch)).findFirst(); - } - - private boolean hasCompressor(String normalizedFileName) { - return findCompressor(normalizedFileName).isPresent(); - } - - private Optional findCompressor(String normalizedFileName) { - return new CompressorStreamFactory().getInputStreamCompressorNames() - .stream().filter(compressor -> normalizedFileName.endsWith("." + compressor)).findFirst(); - } - - static { - disableSslVerification(); - } - - private static void disableSslVerification() { - try { - // Create a trust manager that does not validate certificate chains - TrustManager[] trustAllCerts = new TrustManager[] { new X509TrustManager() { - public X509Certificate[] getAcceptedIssuers() { - return null; - } - - public void checkClientTrusted(X509Certificate[] certs, String authType) { - } - - public void checkServerTrusted(X509Certificate[] certs, String authType) { - } - } - }; - - // Install the all-trusting trust manager - SSLContext sc = SSLContext.getInstance("SSL"); - sc.init(null, trustAllCerts, new java.security.SecureRandom()); - HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); - - // Create all-trusting host name verifier - HostnameVerifier allHostsValid = new HostnameVerifier() { - public boolean verify(String hostname, SSLSession session) { - return true; - } - }; - - // Install the all-trusting host verifier - HttpsURLConnection.setDefaultHostnameVerifier(allHostsValid); - } - catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - } - catch (KeyManagementException e) { - e.printStackTrace(); - } - } - -} diff --git a/functions/common/tensorflow-common/src/main/resources/images/programming_model.png b/functions/common/tensorflow-common/src/main/resources/images/programming_model.png deleted file mode 100644 index 5bbb15518..000000000 Binary files a/functions/common/tensorflow-common/src/main/resources/images/programming_model.png and /dev/null differ diff --git a/functions/common/tensorflow-common/src/main/resources/images/tf_pipeline.png b/functions/common/tensorflow-common/src/main/resources/images/tf_pipeline.png deleted file mode 100644 index d4d06f90d..000000000 Binary files a/functions/common/tensorflow-common/src/main/resources/images/tf_pipeline.png and /dev/null differ diff --git a/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/EnrichFromMemory.java b/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/EnrichFromMemory.java deleted file mode 100644 index c6faf07d9..000000000 --- a/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/EnrichFromMemory.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow; - -import java.util.Arrays; -import java.util.Collections; -import java.util.Map; - -import org.tensorflow.Tensor; - -/** - * @author Christian Tzolov - */ -public class EnrichFromMemory implements AutoCloseable { - - private final GraphRunner graph1; - private final GraphRunner graph2; - private final GraphRunner graph3; - - public EnrichFromMemory() { - this.graph1 = new GraphRunner("x1", "y1") - .withGraphDefinition(tf -> tf.withName("y1").math.mul( - tf.withName("x1").placeholder(Integer.class), - tf.constant(10))); - - this.graph2 = new GraphRunner("x2", "y2") - .withGraphDefinition(tf -> tf.withName("y2").math.mul( - tf.withName("x2").placeholder(Integer.class), - tf.constant(20))); - - this.graph3 = new GraphRunner(Arrays.asList("x31", "x32"), Arrays.asList("y3")) - .withGraphDefinition(tf -> tf.withName("y3").math.add( - tf.withName("x31").placeholder(Integer.class), - tf.withName("x32").placeholder(Integer.class))); - } - - public int compute(Integer input) { - try ( - Tensor x = Tensor.create(input); - GraphRunnerMemory memory = new GraphRunnerMemory(); - ) { - - Map> result = - this.graph1.andThen(memory) - .andThen(graph2).andThen(memory) - .andThen(Functions.enrichFromMemory(memory, "y1")) // retrieves the graph1's y1 output and adds it as a parameter with the same name - .andThen(Functions.rename( - "y1", "x31", // renames the input y1 into x31 - "y2", "x32" // renames the input y2 into x32 - )) - .andThen(graph3).andThen(memory) - .apply(Collections.singletonMap("x", x)); - - memory.getTensorMap().entrySet().forEach(e -> System.out.println(" " + e)); - - return result.get("y3").intValue(); - } - } - - @Override - public void close() { - this.graph1.close(); - this.graph2.close(); - } - - public static void main(String[] args) { - try (EnrichFromMemory example = new EnrichFromMemory()) { - - for (int x = 0; x < 5; x++) { - System.out.println("For x = " + x + ", y = " + example.compute(x)); - } - } - } -} - diff --git a/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/FunctionComposition.java b/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/FunctionComposition.java deleted file mode 100644 index 192751c82..000000000 --- a/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/FunctionComposition.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow; - -import java.util.Collections; -import java.util.Map; - -import org.tensorflow.Tensor; - -/** - * @author Christian Tzolov - */ -public final class FunctionComposition { - - private FunctionComposition() { - - } - - // y = (x * 2) + 20 - // - // y1 = x1 * 2 , where x1 == x - // y2 = x2 + 20 , where x2 == y1 and y = y2 - public static void main(String[] args) { - try ( - GraphRunner graph1 = new GraphRunner("x1", "y1") - .withGraphDefinition(tf -> tf.withName("y1").math.mul( - tf.withName("x1").placeholder(Integer.class), - tf.constant(2))); - GraphRunner graph2 = new GraphRunner("x2", "y2") - .withGraphDefinition(tf -> tf.withName("y2").math.add( - tf.withName("x2").placeholder(Integer.class), - tf.constant(20))); - Tensor x = Tensor.create(10); - ) { - - Map> result = graph1.andThen(graph2).apply(Collections.singletonMap("x", x)); - - System.out.println("Result is: " + result.get("y2").intValue()); - // Result is: 40 - } - - } -} - diff --git a/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/FunctionCompositionMultipleInputsOutputs.java b/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/FunctionCompositionMultipleInputsOutputs.java deleted file mode 100644 index cb233580a..000000000 --- a/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/FunctionCompositionMultipleInputsOutputs.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow; - -import java.util.Arrays; -import java.util.Collections; -import java.util.Map; - -import org.tensorflow.Tensor; -import org.tensorflow.op.core.Placeholder; - -/** - * @author Christian Tzolov - */ -public final class FunctionCompositionMultipleInputsOutputs { - - private FunctionCompositionMultipleInputsOutputs() { - - } - - // y = (x * 2) + (x * 3) - // - // y11 = x1 * 2 , where x1 == x - // y12 = x1 * 3 , where x1 == x - // y2 = x21 + x22 , where x21 == y11, x22 == y12 and y == y2 - public static void main(String[] args) { - try ( - GraphRunner graph1 = new GraphRunner(Arrays.asList("x1"), Arrays.asList("y11", "y12")) - .withGraphDefinition(tf -> { - Placeholder x1 = tf.withName("x1").placeholder(Integer.class); - tf.withName("y11").math.mul(x1, tf.constant(2)); - tf.withName("y12").math.mul(x1, tf.constant(3)); - }); - GraphRunner graph2 = new GraphRunner(Arrays.asList("x21", "x22"), Arrays.asList("y2")) - .withGraphDefinition(tf -> tf.withName("y2").math.add( - tf.withName("x21").placeholder(Integer.class), - tf.withName("x22").placeholder(Integer.class))); - Tensor x = Tensor.create(10); - ) { - - Map> result = graph1 - .andThen(Functions.rename( - "y11", "x21", - "y12", "x22" - )) - .andThen(graph2) - .apply(Collections.singletonMap("x", x)); - - System.out.println("Result is: " + result.get("y2").intValue()); // Result is: 50 - } - } -} - diff --git a/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/ReleaseTensorParameters.java b/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/ReleaseTensorParameters.java deleted file mode 100644 index 77fe49cb0..000000000 --- a/functions/common/tensorflow-common/src/test/java/org/springframework/cloud/fn/common/tensorflow/ReleaseTensorParameters.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.tensorflow; - -import java.util.Collections; -import java.util.Map; - -import org.tensorflow.Tensor; - -/** - * @author Christian Tzolov - */ -public class ReleaseTensorParameters implements AutoCloseable { - - private final GraphRunner graph1; - private final GraphRunner graph2; - - public ReleaseTensorParameters() { - this.graph1 = new GraphRunner("x1", "y1") - .withGraphDefinition(tf -> tf.withName("y1").math.mul( - tf.withName("x1").placeholder(Integer.class), - tf.constant(2))); - - this.graph2 = new GraphRunner("x2", "y2") - .withGraphDefinition(tf -> tf.withName("y2").math.add( - tf.withName("x2").placeholder(Integer.class), - tf.constant(20))); - } - - // y = (x * 2) + 20 - public int compute(Integer input) { - try ( - Tensor x = Tensor.create(input); - GraphRunnerMemory memory = new GraphRunnerMemory(); - ) { - - Map> result = - this.graph1.andThen(memory) - .andThen(graph2).andThen(memory) - .apply(Collections.singletonMap("x", x)); - - memory.getTensorMap().entrySet().forEach(e -> System.out.println(" " + e)); - - return result.get("y2").intValue(); - } - } - - @Override - public void close() { - this.graph1.close(); - this.graph2.close(); - } - - public static void main(String[] args) { - try (ReleaseTensorParameters example = new ReleaseTensorParameters()) { - - for (int x = 0; x < 5; x++) { - System.out.println("For x = " + x + ", y = " + example.compute(x)); - } - } - } -} - diff --git a/functions/common/twitter-common/pom.xml b/functions/common/twitter-common/pom.xml deleted file mode 100644 index e31925b08..000000000 --- a/functions/common/twitter-common/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - twitter-common - twitter-common - Twitter common - - - 4.0.7 - - - - - org.twitter4j - twitter4j-stream - ${twitter4j.version} - - - - org.springframework.integration - spring-integration-ip - - - - diff --git a/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/Cursor.java b/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/Cursor.java deleted file mode 100644 index ba6de3575..000000000 --- a/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/Cursor.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.twitter; - -/** - * @author Christian Tzolov - */ -public class Cursor { - private long cursor = -1; - - public long getCursor() { - return cursor; - } - - public void updateCursor(long newCursor) { - this.cursor = (newCursor > 0) ? newCursor : -1; - } - - @Override - public String toString() { - return "Cursor{cursor=" + cursor + '}'; - } -} diff --git a/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionConfiguration.java b/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionConfiguration.java deleted file mode 100644 index b5b65c784..000000000 --- a/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionConfiguration.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.twitter; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import twitter4j.Twitter; -import twitter4j.TwitterFactory; -import twitter4j.TwitterObjectFactory; -import twitter4j.TwitterStream; -import twitter4j.TwitterStreamFactory; -import twitter4j.conf.ConfigurationBuilder; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.util.MimeTypeUtils; - -/** - * - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties({ TwitterConnectionProperties.class }) -public class TwitterConnectionConfiguration { - - private static final Log logger = LogFactory.getLog(TwitterConnectionConfiguration.class); - - @Bean - public twitter4j.conf.Configuration twitterConfiguration(TwitterConnectionProperties properties, - Function toConfigurationBuilder) { - return toConfigurationBuilder.apply(properties).build(); - } - - @Bean - public Twitter twitter(twitter4j.conf.Configuration configuration) { - return new TwitterFactory(configuration).getInstance(); - } - - @Bean - public TwitterStream twitterStream(twitter4j.conf.Configuration configuration) { - return new TwitterStreamFactory(configuration).getInstance(); - } - - @Bean - public Function toConfigurationBuilder() { - return properties -> new ConfigurationBuilder() - .setJSONStoreEnabled(properties.isRawJson()) - .setDebugEnabled(properties.isDebugEnabled()) - .setOAuthConsumerKey(properties.getConsumerKey()) - .setOAuthConsumerSecret(properties.getConsumerSecret()) - .setOAuthAccessToken(properties.getAccessToken()) - .setOAuthAccessTokenSecret(properties.getAccessTokenSecret()); - } - - @Bean - public Function> json(ObjectMapper mapper) { - return objects -> { - try { - String json = mapper.writeValueAsString(objects); - - return MessageBuilder - .withPayload(json.getBytes()) - .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON_VALUE) - .build(); - } - catch (JsonProcessingException e) { - logger.error("Status to JSON conversion error!", e); - } - return null; - }; - } - - /** - * Retrieves the raw JSON form of the provided object. - * - * Note that raw JSON forms can be retrieved only from the same thread invoked the last method - * call and will become inaccessible once another method call. - * - * @return Function that can retrieve the raw JSON object from the objects returned by the Twitter4J's APIs. - */ - @Bean - public Function rawJsonExtractor() { - return response -> { - if (response instanceof List) { - List responses = (List) response; - List rawJsonList = new ArrayList<>(); - for (Object object : responses) { - rawJsonList.add(TwitterObjectFactory.getRawJSON(object)); - } - return rawJsonList; - } - else { - return TwitterObjectFactory.getRawJSON(response); - } - }; - } - - @Bean - public Function> managedJson(TwitterConnectionProperties properties, - Function rawJsonExtractor, Function> json) { - return list -> (properties.isRawJson()) ? rawJsonExtractor.andThen(json).apply(list) : json.apply(list); - } -} diff --git a/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionProperties.java b/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionProperties.java deleted file mode 100644 index dd8e5df0d..000000000 --- a/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionProperties.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.twitter; - -import jakarta.validation.constraints.NotEmpty; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("twitter.connection") -@Validated -public class TwitterConnectionProperties { - - /** - * Your Twitter key. - */ - @NotEmpty - private String consumerKey; - - /** - * Your Twitter secret. - */ - @NotEmpty - private String consumerSecret; - - /** - * Your Twitter token. - */ - @NotEmpty - private String accessToken; - - /** - * Your Twitter token secret. - */ - @NotEmpty - private String accessTokenSecret; - - /** - * Enables Twitter4J debug mode. - */ - private boolean debugEnabled = false; - - /** - * Enable caching the original (raw) JSON objects as returned by the Twitter APIs. - * When set to False the result will use the Twitter4J's json representations. - * When set to True the result will use the original Twitter APISs json representations. - */ - private boolean rawJson = true; - - public String getConsumerKey() { - return consumerKey; - } - - public void setConsumerKey(String consumerKey) { - this.consumerKey = consumerKey; - } - - public String getConsumerSecret() { - return consumerSecret; - } - - public void setConsumerSecret(String consumerSecret) { - this.consumerSecret = consumerSecret; - } - - public String getAccessToken() { - return accessToken; - } - - public void setAccessToken(String accessToken) { - this.accessToken = accessToken; - } - - public String getAccessTokenSecret() { - return accessTokenSecret; - } - - public void setAccessTokenSecret(String accessTokenSecret) { - this.accessTokenSecret = accessTokenSecret; - } - - public boolean isDebugEnabled() { - return debugEnabled; - } - - public void setDebugEnabled(boolean debugEnabled) { - this.debugEnabled = debugEnabled; - } - - public boolean isRawJson() { - return this.rawJson; - } - - public void setRawJson(boolean rawJson) { - this.rawJson = rawJson; - } -} diff --git a/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/util/TwitterTestUtils.java b/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/util/TwitterTestUtils.java deleted file mode 100644 index 25b276791..000000000 --- a/functions/common/twitter-common/src/main/java/org/springframework/cloud/fn/common/twitter/util/TwitterTestUtils.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.twitter.util; - - -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.function.Function; - -import twitter4j.conf.ConfigurationBuilder; - -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.util.StreamUtils; - -/** - * @author Christian Tzolov - */ -public class TwitterTestUtils { - - public Function mockTwitterUrls(String baseUrl) { - return configBuilder -> { - configBuilder.setRestBaseURL(baseUrl + "/"); - configBuilder.setStreamBaseURL(baseUrl + "/stream/"); - configBuilder.setUserStreamBaseURL(baseUrl + "/user/"); - configBuilder.setSiteStreamBaseURL(baseUrl + "/site/"); - configBuilder.setUploadBaseURL(baseUrl + "/upload/"); - - configBuilder.setOAuthAccessTokenURL(baseUrl + "/oauth/access_token"); - configBuilder.setOAuthAuthenticationURL(baseUrl + "/oauth/authenticate"); - configBuilder.setOAuthAuthorizationURL(baseUrl + "/oauth/authorize"); - configBuilder.setOAuthRequestTokenURL(baseUrl + "/oauth/request_token"); - configBuilder.setOAuth2TokenURL(baseUrl + "/oauth2/token"); - configBuilder.setOAuth2InvalidateTokenURL(baseUrl + "/oauth2/invalidate_token"); - - return configBuilder; - }; - } - - /** - * Load Spring Resource as String. - * @param resourcePath Resource path (accepts file:// , classpath:// and http:// uri schemas) - * @return Returns text (UTF8) representation of the resource pointed by the resourcePath - */ - public static String asString(String resourcePath) { - try { - return StreamUtils.copyToString(new DefaultResourceLoader().getResource(resourcePath).getInputStream(), - Charset.forName("UTF-8")); - } - catch (IOException e) { - throw new RuntimeException("Can not load resource:" + resourcePath, e); - } - } - -} diff --git a/functions/common/xmpp-common/pom.xml b/functions/common/xmpp-common/pom.xml deleted file mode 100644 index aba1c006f..000000000 --- a/functions/common/xmpp-common/pom.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - xmpp-common - xmpp-common - XMPP common - - - - org.springframework.integration - spring-integration-xmpp - - - - diff --git a/functions/common/xmpp-common/src/main/java/org/springframework/cloud/fn/common/xmpp/XmppConnectionFactoryConfiguration.java b/functions/common/xmpp-common/src/main/java/org/springframework/cloud/fn/common/xmpp/XmppConnectionFactoryConfiguration.java deleted file mode 100644 index 1918f59c9..000000000 --- a/functions/common/xmpp-common/src/main/java/org/springframework/cloud/fn/common/xmpp/XmppConnectionFactoryConfiguration.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2014-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.xmpp; - -import org.jivesoftware.smack.tcp.XMPPTCPConnectionConfiguration; -import org.jxmpp.stringprep.XmppStringprepException; -import org.jxmpp.util.XmppStringUtils; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.xmpp.config.XmppConnectionFactoryBean; -import org.springframework.util.StringUtils; - -@Configuration -@EnableConfigurationProperties(XmppConnectionFactoryProperties.class) -public class XmppConnectionFactoryConfiguration { - - @Bean - @ConditionalOnMissingBean - public XmppConnectionFactoryBean xmppConnectionFactoryBean(XmppConnectionFactoryProperties properties) throws XmppStringprepException { - - XmppConnectionFactoryBean xmppConnectionFactoryBean = new XmppConnectionFactoryBean(); - xmppConnectionFactoryBean.setSubscriptionMode(properties.getSubscriptionMode()); - - XMPPTCPConnectionConfiguration.Builder builder = XMPPTCPConnectionConfiguration.builder(); - builder.setSecurityMode(properties.getSecurityMode()); - builder.setHost(properties.getHost()); - builder.setPort(properties.getPort()); - if (StringUtils.hasText(properties.getResource())) { - builder.setResource(properties.getResource()); - } - - if (StringUtils.hasText(properties.getServiceName())) { - builder.setUsernameAndPassword(properties.getUser(), properties.getPassword()) - .setXmppDomain(properties.getServiceName()); - } - else { - builder.setUsernameAndPassword(XmppStringUtils.parseLocalpart(properties.getUser()), properties.getPassword()) - .setXmppDomain(properties.getUser()); - } - - xmppConnectionFactoryBean.setConnectionConfiguration(builder.build()); - - return xmppConnectionFactoryBean; - } - -} diff --git a/functions/common/xmpp-common/src/main/java/org/springframework/cloud/fn/common/xmpp/XmppConnectionFactoryProperties.java b/functions/common/xmpp-common/src/main/java/org/springframework/cloud/fn/common/xmpp/XmppConnectionFactoryProperties.java deleted file mode 100644 index 99cd071e3..000000000 --- a/functions/common/xmpp-common/src/main/java/org/springframework/cloud/fn/common/xmpp/XmppConnectionFactoryProperties.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2014-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.common.xmpp; - -import jakarta.validation.constraints.NotEmpty; -import org.jivesoftware.smack.ConnectionConfiguration; -import org.jivesoftware.smack.roster.Roster; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * - * @author Daniel Frey - * @since 4.0.0 - */ -@ConfigurationProperties("xmpp.factory") -@Validated -public class XmppConnectionFactoryProperties { - - /** - * The Resource to bind to on the XMPP Host. - * - Can be empty, server will generate one if not set - */ - private String resource; - - /** - * The User the connection should connect as. - */ - private String user; - - /** - * The Password for the connected user. - */ - private String password; - - /** - * The Service Name to set for the XMPP Domain. - */ - private String serviceName; - - /** - * XMPP Host server to connect to. - */ - private String host; - - /** - * Port for connecting to the host. - * - Default Client Port: 5222 - */ - private int port = 5222; - - private Roster.SubscriptionMode subscriptionMode = Roster.getDefaultSubscriptionMode(); - - private ConnectionConfiguration.SecurityMode securityMode = ConnectionConfiguration.SecurityMode.required; - - public void setResource(String resource) { - this.resource = resource; - } - - public String getResource() { - return resource; - } - - public void setUser(String user) { - this.user = user; - } - - @NotEmpty(message = "user is required") - public String getUser() { - return user; - } - - public void setPassword(String password) { - this.password = password; - } - - @NotEmpty(message = "password is required") - public String getPassword() { - return password; - } - - public void setServiceName(String serviceName) { - this.serviceName = serviceName; - } - - public String getServiceName() { - return serviceName; - } - - public void setHost(String host) { - this.host = host; - } - - @NotEmpty(message = "host is required") - public String getHost() { - return host; - } - - public void setPort(int port) { - this.port = port; - } - - public int getPort() { - return port; - } - - public void setSubscriptionMode(Roster.SubscriptionMode subscriptionMode) { - this.subscriptionMode = subscriptionMode; - } - - public Roster.SubscriptionMode getSubscriptionMode() { - return subscriptionMode; - } - - public void setSecurityMode(ConnectionConfiguration.SecurityMode securityMode) { - this.securityMode = securityMode; - } - - public ConnectionConfiguration.SecurityMode getSecurityMode() { - return securityMode; - } - -} diff --git a/functions/consumer/analytics-consumer/README.adoc b/functions/consumer/analytics-consumer/README.adoc deleted file mode 100644 index c845085dc..000000000 --- a/functions/consumer/analytics-consumer/README.adoc +++ /dev/null @@ -1,177 +0,0 @@ -# Analytics Consumer - -The `analytics-consumer` is a Java https://docs.oracle.com/javase/8/docs/api/java/util/function/Consumer.html[Consumer>] that computes analytics from the input data messages and publishes them as metrics to various monitoring systems. -It leverages the https://micrometer.io[micrometer library] for providing a uniform programming experience across the most popular https://micrometer.io/docs[monitoring systems] and uses https://docs.spring.io/spring-integration/reference/html/spel.html#spel[Spring Expression Language (SpEL)] for defining how the metric names, values and tags are computed from the input data. - -The analytics-consumer can produce two metrics types: - -- https://micrometer.io/docs/concepts#_counters[Counter] - reports a single metric, a count, that increments by a fixed, positive amount. Counters can be used for computing the rates of how the data changes in time. -- https://micrometer.io/docs/concepts#_gauges[Gauge] - reports the current value. Typical examples for gauges would be the size of a collection or map or number of threads in a running state. - -A https://micrometer.io/docs/concepts#_meters[Meter] (e.g Counter or Gauge) is uniquely identified by its `name` and `dimensions` (the term dimensions and tags is used interchangeably). Dimensions allow a particular named metric to be sliced to drill down and reason about the data. - -NOTE: As a metrics is uniquely identified by its `name` and `dimensions`, you can assign multiple tags (e.g. key/value pairs) to every metric, but you cannot randomly change those tags afterwards! Monitoring systems such as Prometheus will complain if a metric with the same name has different sets of tags. - -## Beans for injection - -Add the analytics-consumer dependency to your POM: - -[source,xml] ----- - - org.springframework.cloud.fn - analytics-consumer - 1.0.0-SNAPSHOT - ----- - -Import the https://github.com/spring-cloud/stream-applications/blob/master/functions/consumer/analytics-consumer/src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerConfiguration.java[AnalyticsConsumerConfiguration] in the application and inject the following consumer bean: - -[source,java] ----- - Consumer> analyticsConsumer ----- - -For every input https://docs.spring.io/spring-integration/reference/html/message.html[Message] the `analyticsConsumer` computes the defined metrics and eventually, with the help of the micrometer library, publishes them to the backend monitoring systems. The https://docs.spring.io/spring-integration/reference/html/message.html[Message] is a generic container for data. Each Message instance includes a payload and headers containing user-extensible properties as key-value pairs. Any object can be provided as the payload. -The https://docs.spring.io/spring-integration/reference/html/message.html#message-builder[MessageBuilder] helps to create a Message instance from any payload content and assign any key/value as a header: - -[source,java] ----- - Message myMessage = MessageBuilder - .withPayload("My message text") - .setHeader("kind", "CUSTOM") - .setHeader("foo", "bar") - .build(); ----- - -The `SpEL` expressions use the `headers` and `payload` keywords to access message’s headers and payload values. For example a counter metrics can have a value amount computed from the size of the input message payload add a `my_tag` tag, extracted from the `kind` header value: - -[source,properties] ----- -analytics.amount-expression=payload.lenght() -analytics.tag.expression.my_tag=headers['kind'] ----- - -Review the https://github.com/spring-cloud/stream-applications/blob/master/functions/consumer/analytics-consumer/src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerProperties.java[AnalyticsConsumerProperties]'s javadocs for further details how to use the SpEL properties. - -By default, Micrometer is packed with a SimpleMeterRegistry that holds the latest value of each meter in memory and doesn’t export the data anywhere. -To enable support for another monitoring system you have to add the spring-boot-starter-actuator dependency and the micrometer dependency of the monitoring system of choice: - -[source,xml] ----- - - org.springframework.boot - spring-boot-starter-actuator - CHANGE TO LATEST VERSION - - - - io.micrometer - micrometer-registry-[MONITORING SYSTEM NAME] - ${micrometer.version} - ----- - -Follow the https://docs.spring.io/spring-boot/docs/2.3.1.RELEASE/reference/html/production-ready-features.html#production-ready-metrics-export[configuration instructions] for the selected monitoring system. All monitoring configuration properties start with a prefix: `management.metrics.export`. - -== Configuration Options - -All `analytics-consumer` configuration properties use the `analytics` prefix. For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerProperties.java[AnalyticsConsumerProperties]. - -All monitoring configuration properties start with a prefix `management.metrics.export`. For configuring a particular monitoring system follow the provided https://docs.spring.io/spring-boot/docs/2.3.1.RELEASE/reference/html/production-ready-features.html#production-ready-metrics-export[configuration instructions]. - -==== Sample Configuration - -Following examples show how to configure `counter` and `gauge` metrics over a series of stock-exchange messages like this: - -[source,json] ----- -{ - "data": { - "symbol": "AAPL", - "exchange": "XNAS", - "open": 318.66, - "close": 316.85, - "volume": 25672211.0 - } -} ----- - -The following configuration will create a `counter` metrics called `stockrates` with two tags: `symbol` and `exchange` computed from the json fields: - -.Counter Metrcis Configuration - count stock transactions -|=== -|Property |Description - -|analytics.meter-type=counter -|Counter meter type (default) - -|analytics.name=stockrates -|Metrics name - -|analytics.tag.expression.symbol=#jsonPath(payload,'$.data.symbol') -|Add tag `symbol` equal to the `date.symbol` field in the json messages. - -|analytics.tag.expression.exchange=#jsonPath(payload,'$.data.exchange') -|Add tag `exchange` equal to the `date.exchange` field in the json messages. - -|=== - -Now you can use the `stockrates` metrics to measure the rates at which the stock transactions occur over a given time interval. Furthermore, you can aggregate those rates by the `symbol` and `exchange` tags. - -To measure the transaction volumes contained in the data.volume JSON fields, you can build a GAUGE metrics like this: - -.Gauge Metrcis Configuration - compute stock volumes -|=== -|Property |Description - -|analytics.meter-type=gauge -|Gauge meter type - -|analytics.name=stockvolumes -|Metrics name - -|analytics.tag.expression.symbol=#jsonPath(payload,'$.data.symbol') -|Add tag `symbol` equal to the `date.symbol` field in the json messages. - -|analytics.tag.expression.exchange=#jsonPath(payload,'$.data.exchange') -|Add tag `exchange` equal to the `date.exchange` field in the json messages. - -|analytics.tag.amount-expression=#jsonPath(payload,'$.data.volume') -|Set the Gauge to the `data/volume` field values. -|=== - -Then use the `stockvolumes` metrics to graph, in real-time, the transaction volumes changes over time. You can aggregate those volumes by the `symbol` and `exchange` tags. - -WARNING: Micrometer implements the Gauges for the purpose of data sampling! There is no information about what might have occurred between two consecutive samples. Any intermediate values set on a gauge are lost by the time the gauge value is reported to a metrics backend. - -To enable one or more https://micrometer.io/docs[supported monitoring systems] you need to add a configuration like this: - -.Wavefront Configuration. -|=== -|Property |Description - -|management.metrics.export.wavefront.enabled=true -|Enable or disable the monitoring system. (enabled by default). - -|management.metrics.export.wavefront.uri=YOUR_WAVEFRONT_SERVER_URI -|UIR of your Wavefront server or Wavefront Proxy. - -|management.metrics.export.wavefront.api-token=YOUR_API_TOKEN -|Wavefront access token. - -|management.metrics.export.wavefront.source=stock-exchange-demo -|The `source` is used to distinct your metrics on the Wavefront server. - -|=== - - -== Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/analytics[test suite] for the various ways, this consumer is used. - -== Other usage - -* See the https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/analytics-sink/README.adoc[Analytics Sink README] where this consumer is used to create a Spring Cloud Stream application where it makes a Counter sink. - -* https://docs.google.com/document/d/1BHBjgMmg4a1ue2wr-dmPTfgaN0so4ufw2XkG541Ac9Q/edit?usp=sharing[Stock Exchange Sample]. diff --git a/functions/consumer/analytics-consumer/pom.xml b/functions/consumer/analytics-consumer/pom.xml deleted file mode 100644 index 5bb00bf33..000000000 --- a/functions/consumer/analytics-consumer/pom.xml +++ /dev/null @@ -1,46 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - analytics-consumer - analytics-consumer - Spring Native Consumer for computing meters - - - - org.springframework.cloud.fn - payload-converter-function - ${project.version} - - - org.springframework.boot - spring-boot-starter-actuator - test - - - com.fasterxml.jackson.core - jackson-databind - compile - - - io.micrometer - micrometer-core - - - io.micrometer - micrometer-registry-wavefront - 1.9.4 - test - - - - diff --git a/functions/consumer/analytics-consumer/src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerConfiguration.java b/functions/consumer/analytics-consumer/src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerConfiguration.java deleted file mode 100644 index 45e7aeaa8..000000000 --- a/functions/consumer/analytics-consumer/src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerConfiguration.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.analytics; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import io.micrometer.core.instrument.Meter; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; -import io.micrometer.core.instrument.Tags; -import io.micrometer.core.instrument.composite.CompositeMeterRegistry; -import io.micrometer.core.instrument.simple.SimpleMeterRegistry; - -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.SpelExpressionConverterConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Lazy; -import org.springframework.expression.EvaluationContext; -import org.springframework.messaging.Message; -import org.springframework.util.CollectionUtils; -import org.springframework.util.ObjectUtils; -import org.springframework.util.StringUtils; - -/** - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties(AnalyticsConsumerProperties.class) -public class AnalyticsConsumerConfiguration { - - /** Default tag value. Used to fill the tag when the actual value is missing. */ - public static final String UNAVAILABLE_TAG = "NA"; - - private final Map gaugeValues = new ConcurrentHashMap<>(); - - @Bean(name = "analyticsConsumer") - public Consumer> analyticsConsumer(AnalyticsConsumerProperties properties, MeterRegistry[] meterRegistries, - @Lazy @Qualifier(SpelExpressionConverterConfiguration.INTEGRATION_EVALUATION_CONTEXT) EvaluationContext context) { - - // If the CompositeMeterRegistry is present the it already contains all non-composite registries. - // In this case we override the input meterRegistries to use the CompositeMeterRegistry only. - final MeterRegistry[] finalMeterRegistries = Stream.of(meterRegistries) - .filter(CompositeMeterRegistry.class::isInstance) - .findFirst() - .map(meterRegistry -> new MeterRegistry[] { meterRegistry }) - .orElse(meterRegistries); - - return message -> { - - CharSequence meterNameRaw = properties.getComputedNameExpression().getValue(context, message, CharSequence.class); - String meterName = StringUtils.isEmpty(meterNameRaw) ? "empty" : meterNameRaw.toString(); - - // All fixed tags together are passed with every meter update. - Tags fixedTags = this.toTags(properties.getTag().getFixed()); - - double amount = properties.getComputedAmountExpression().getValue(context, message, double.class); - - Map> allGroupedTags = new HashMap<>(); - // Tag Expressions - if (properties.getTag().getExpression() != null) { - - Map> groupedTags = properties.getTag().getExpression().entrySet().stream() - // maps a pair into [, ... ] Tag array. - .map(namedExpression -> - toList(namedExpression.getValue().getValue(context, message)).stream() - .map(tagValue -> Tag.of(namedExpression.getKey(), tagValue)) - .collect(Collectors.toList())).flatMap(List::stream) - .collect(Collectors.groupingBy(Tag::getKey, Collectors.toList())); - allGroupedTags.putAll(groupedTags); - } - - this.recordMetrics(finalMeterRegistries, meterName, fixedTags, allGroupedTags, amount, properties.getMeterType()); - }; - } - - /** - * Converts a key/value Map into Tag(key,value) list. Filters out the empty key/value pairs. - * - * @param keyValueMap key/value map to convert into tags. - * @return Returns Tags list representing every non-empty key/value pair. - */ - protected Tags toTags(Map keyValueMap) { - return CollectionUtils.isEmpty(keyValueMap) ? Tags.empty() : - Tags.of(keyValueMap.entrySet().stream() - .filter(e -> StringUtils.hasText(e.getKey()) && StringUtils.hasText(e.getValue())) - .map(e -> Tag.of(e.getKey(), e.getValue())) - .collect(Collectors.toList())); - } - - /** - * Converts the input value into an list of values. If the value is not a collection/array type the result - * is a single element list. For collection/array input value the result is the list of stringified content of - * this collection. - * - * @param value input value can be array, collection or single value. - * @return Returns value list. - */ - protected List toList(Object value) { - if (value == null) { - // Ensure that the tag is present in the meter metrics, even if empty. - // TSDB as Prometheus do not tolerate same meters to have different tags signatures. - return Collections.singletonList(UNAVAILABLE_TAG); - } - - if ((value instanceof Collection) || ObjectUtils.isArray(value)) { - - Collection valueCollection = (value instanceof Collection) ? (Collection) value - : Arrays.asList(ObjectUtils.toObjectArray(value)); - List list = valueCollection.stream() - .filter(Objects::nonNull) - .map(Object::toString) - .filter(StringUtils::hasText) - .collect(Collectors.toList()); - return CollectionUtils.isEmpty(list) ? Collections.singletonList(UNAVAILABLE_TAG) : list; - } - else { - return Collections.singletonList(value.toString()); - } - } - - private void recordMetrics(MeterRegistry[] meterRegistries, String meterName, Tags fixedTags, - Map> groupedTags, double amount, AnalyticsConsumerProperties.MeterType meterType) { - if (!CollectionUtils.isEmpty(groupedTags)) { - groupedTags.values().stream().map(List::size).max(Integer::compareTo).ifPresent( - max -> { - for (int i = 0; i < max; i++) { - Tags currentTags = Tags.of(fixedTags); - for (Map.Entry> e : groupedTags.entrySet()) { - currentTags = (e.getValue().size() > i) ? - currentTags.and(e.getValue().get(i)) : - currentTags.and(Tags.of(e.getKey(), "")); - } - - // Update the meterName for every configured MaterRegistry. - record(meterRegistries, meterName, currentTags, amount, meterType); - } - } - ); - } - else { - // Update the meterName for every configured MaterRegistry. - record(meterRegistries, meterName, fixedTags, amount, meterType); - } - } - - private void record(MeterRegistry[] meterRegistries, String meterName, - Iterable tags, double meterAmount, AnalyticsConsumerProperties.MeterType meterType) { - - for (MeterRegistry meterRegistry : meterRegistries) { - if (meterType == AnalyticsConsumerProperties.MeterType.gauge) { - Meter.Id gaugeId = new Meter.Id(meterName, Tags.of(tags), null, null, Meter.Type.GAUGE); - if (!this.gaugeValues.containsKey(gaugeId)) { - this.gaugeValues.put(gaugeId, new AtomicLong((long) meterAmount)); - } - else { - this.gaugeValues.get(gaugeId).set((long) meterAmount); - } - if (!isMeterRegistryContainsGauge(meterRegistry, gaugeId)) { - meterRegistry.gauge(meterName, tags, this.gaugeValues.get(gaugeId), AtomicLong::doubleValue); - } - } - else if (meterType == AnalyticsConsumerProperties.MeterType.counter) { - meterRegistry.counter(meterName, tags).increment(meterAmount); - } - else { - throw new RuntimeException("Unknown meter type:" + meterType); - } - } - } - - private boolean isMeterRegistryContainsGauge(MeterRegistry meterRegistry, Meter.Id gaugeId) { - return meterRegistry.find(gaugeId.getName()).gauges().stream() - .anyMatch(gauge -> gauge.getId().equals(gaugeId)); - } - - @Bean - @ConditionalOnMissingBean - public SimpleMeterRegistry simpleMeterRegistry() { - return new SimpleMeterRegistry(); - } -} diff --git a/functions/consumer/analytics-consumer/src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerProperties.java b/functions/consumer/analytics-consumer/src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerProperties.java deleted file mode 100644 index 347bcf4e0..000000000 --- a/functions/consumer/analytics-consumer/src/main/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerProperties.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.analytics; - -import java.util.Map; - -import jakarta.validation.constraints.AssertTrue; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.expression.common.LiteralExpression; -import org.springframework.validation.annotation.Validated; - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("analytics") -@Validated -public class AnalyticsConsumerProperties { - - enum MeterType { - /** Uses the Micrometer Counter meter type. It accumulates intermediate counts toward the point where - * the data is sent to the metrics backend.*/ - counter, - /** Uses the Micrometer Gauge meter type. Gauges sample the input time series. Any intermediate values set on - * a gauge are lost by the time the gauge value is reported to a metrics backend. - * TIP: Never gauge something you can count with a Counter!*/ - gauge - } - - /** - * Micrometer meter type used to report the metrics to the backend. - */ - private MeterType meterType = MeterType.counter; - - /** - * The name of the output metrics. - * The 'name' and 'nameExpression' are mutually exclusive. Only one of them can be set. - */ - private String name; - - /** - * A SpEL expression to compute the output metrics name from the input message. - * The 'name' and 'nameExpression' are mutually exclusive. Only one of them can be set. - */ - private Expression nameExpression; - - /** - * If neither `name` nor `nameExpression` is set the name of the output metrics defaults to the - * Spring application name. - */ - @Value("${spring.application.name:analytics}") - private String defaultName; - - /** - * A SpEL expression to compute the output metrics value (e.g. amount). - * It defaults to 1.0 - */ - private Expression amountExpression; - - /** - * Fixed and computed tags to be assignee with the output metric. - */ - private final MetricsTag tag = new MetricsTag(); - - public MetricsTag getTag() { - return tag; - } - - public MeterType getMeterType() { - return meterType; - } - - public void setMeterType(MeterType meterType) { - this.meterType = meterType; - } - - public String getName() { - if (name == null && nameExpression == null) { - return defaultName; - } - return name; - } - - public void setName(String name) { - this.name = name; - } - - public Expression getNameExpression() { - return nameExpression; - } - - public void setNameExpression(Expression nameExpression) { - this.nameExpression = nameExpression; - } - - public Expression getAmountExpression() { - return amountExpression; - } - - public void setAmountExpression(Expression amountExpression) { - this.amountExpression = amountExpression; - } - - public Expression getComputedAmountExpression() { - return (amountExpression != null ? amountExpression : new LiteralExpression("1.0")); - } - - public Expression getComputedNameExpression() { - return (nameExpression != null ? nameExpression : new LiteralExpression(getName())); - } - - @AssertTrue(message = "exactly one of 'name' and 'nameExpression' must be set") - public boolean isExclusiveOptions() { - return getName() != null ^ getNameExpression() != null; - } - - @Override - public String toString() { - return "AnalyticsFunctionProperties{" + - "defaultName='" + defaultName + '\'' + - ", name=" + name + - ", tag=" + tag + - '}'; - } - - public static class MetricsTag { - - /** - * DEPRECATED: Please use the analytics.tag.expression with literal SpEL expression. - * - * Custom, fixed Tags. Those tags have constant values, created once and then sent along with every - * published metrics. The convention to define a fixed Tags is: - * - * analytics.tag.fixed.[tag-name]=[tag-value] - * - */ - @Deprecated - private Map fixed; - - /** - * Computes tags from SpEL expression. - * Single SpEL expression can produce an array of values, which in turn means distinct name/value tags. - * Every name/value tag will produce a separate meter increment. - * Tag expression format is: analytics.tag.expression.[tag-name]=[SpEL expression] - */ - private Map expression; - - public Map getFixed() { - return fixed; - } - - public void setFixed(Map fixed) { - this.fixed = fixed; - } - - public Map getExpression() { - return expression; - } - - public void setExpression(Map expression) { - this.expression = expression; - } - - @Override - public String toString() { - return "MetricsTag{" + - "fixed=" + fixed + - ", expression=" + expression + - '}'; - } - } -} diff --git a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerParentTest.java b/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerParentTest.java deleted file mode 100644 index 9473a8215..000000000 --- a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/AnalyticsConsumerParentTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.analytics; - -import java.util.function.Consumer; - -import io.micrometer.core.instrument.simple.SimpleMeterRegistry; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.annotation.DirtiesContext; - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { "management.metrics.export.wavefront.enabled=false" }) -@DirtiesContext -public class AnalyticsConsumerParentTest { - - @Autowired - protected SimpleMeterRegistry meterRegistry; - - @Autowired - protected Consumer> analyticsConsumer; - - protected Message message(String payload) { - return MessageBuilder.withPayload(payload.getBytes()).build(); - } - - @SpringBootApplication - static class AnalyticsConsumerTestApplication { - } -} diff --git a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/CountWithAmountTest.java b/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/CountWithAmountTest.java deleted file mode 100644 index 48fd47983..000000000 --- a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/CountWithAmountTest.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.analytics; - -import org.junit.jupiter.api.Test; - -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Christian Tzolov - */ -@TestPropertySource(properties = { - "analytics.name=counter666", - "analytics.tag.expression.foo='bar'", - "analytics.amount-expression=payload.length()" -}) -class CountWithAmountTest extends AnalyticsConsumerParentTest { - - @Test - void testCounterSink() { - String message = "hello world message"; - double messageSize = Long.valueOf(message.length()).doubleValue(); - analyticsConsumer.accept(new GenericMessage(message)); - assertThat(meterRegistry.find("counter666").counter().count()).isEqualTo(messageSize); - } -} diff --git a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/EmptyTagsTests.java b/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/EmptyTagsTests.java deleted file mode 100644 index 23344129a..000000000 --- a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/EmptyTagsTests.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.analytics; - -import java.util.Collection; - -import io.micrometer.core.instrument.Counter; -import org.junit.jupiter.api.Test; - -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Christian Tzolov - */ -@TestPropertySource(properties = { - "analytics.name=counter666", - "analytics.tag.fixed.foo=", - "analytics.tag.expression.tag666=#jsonPath(payload,'$..noField')", - "analytics.tag.expression.test=#jsonPath(payload,'$..test')" -}) -class EmptyTagsTests extends AnalyticsConsumerParentTest { - - @Test - void testCounterSink() { - - analyticsConsumer.accept(message("{\"test\": \"Bar\"}")); - - Collection fixedTagsCounters = meterRegistry.find("counter666").tagKeys("foo").counters(); - assertThat(fixedTagsCounters.size()).isEqualTo(0); - - Collection expressionTagsCounters = meterRegistry.find("counter666").tagKeys("tag666").counters(); - assertThat(expressionTagsCounters.size()).isEqualTo(1); - assertThat(meterRegistry.find("counter666").meter().getId().getTag("tag666")).isEqualTo(AnalyticsConsumerConfiguration.UNAVAILABLE_TAG); - - Collection testExpTagsCounters = meterRegistry.find("counter666").tagKeys("test").counters(); - assertThat(testExpTagsCounters.size()).isEqualTo(1); - assertThat(meterRegistry.find("counter666").meter().getId().getTag("test")).isEqualTo("Bar"); - } -} diff --git a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/ExpressionCounterNameTests.java b/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/ExpressionCounterNameTests.java deleted file mode 100644 index 2cec7676b..000000000 --- a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/ExpressionCounterNameTests.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.analytics; - -import java.util.stream.IntStream; - -import org.junit.jupiter.api.Test; - -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Christian Tzolov - */ -@TestPropertySource(properties = { - "analytics.name-expression=payload" -}) -public class ExpressionCounterNameTests extends AnalyticsConsumerParentTest { - - @Test - void testCounterSink() { - IntStream.range(0, 13).forEach(i -> analyticsConsumer.accept(new GenericMessage("hello"))); - assertThat(meterRegistry.find("hello").counter().count()).isEqualTo(13.0); - } -} diff --git a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/FixedTagsTests.java b/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/FixedTagsTests.java deleted file mode 100644 index a10e8071d..000000000 --- a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/FixedTagsTests.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.analytics; - -import java.util.stream.IntStream; -import java.util.stream.StreamSupport; - -import io.micrometer.core.instrument.Meter; -import org.junit.jupiter.api.Test; - -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Christian Tzolov - */ -@TestPropertySource(properties = { - "analytics.name=counter666", - "analytics.tag.fixed.foo=bar", - "analytics.tag.fixed.gork=bork" -}) -public class FixedTagsTests extends AnalyticsConsumerParentTest { - - @Test - void testАnalyticsSink() { - IntStream.range(0, 13).forEach(i -> analyticsConsumer.accept(new GenericMessage("hello"))); - Meter counterMeter = meterRegistry.find("counter666").meter(); - assertThat(StreamSupport.stream(counterMeter.measure().spliterator(), false) - .mapToDouble(m -> m.getValue()).sum()).isEqualTo(13.0); - - assertThat(counterMeter.getId().getTags().size()).isEqualTo(2); - assertThat(counterMeter.getId().getTag("foo")).isEqualTo("bar"); - assertThat(counterMeter.getId().getTag("gork")).isEqualTo("bork"); - } -} diff --git a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/GaugeWithAmountTest.java b/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/GaugeWithAmountTest.java deleted file mode 100644 index 3ad284455..000000000 --- a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/GaugeWithAmountTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.analytics; - -import org.junit.jupiter.api.Test; - -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Christian Tzolov - */ -@TestPropertySource(properties = { - "analytics.meter-type=gauge", - "analytics.name=myGauge", - "analytics.tag.expression.foo='bar'", - "analytics.amount-expression=payload.length()" -}) -class GaugeWithAmountTest extends AnalyticsConsumerParentTest { - - @Test - void testАnalyticsSink() { - String messageSmall = "hello"; - analyticsConsumer.accept(new GenericMessage(messageSmall)); - assertThat(meterRegistry.find("myGauge").gauge().value()).isEqualTo(size(messageSmall)); - - assertThat(meterRegistry.find("myGauge").gauge().getId().getTags()).hasSize(1); - assertThat(meterRegistry.find("myGauge").gauge().getId().getTag("foo")).isEqualTo("bar"); - - String messageMiddle = "hello world"; - analyticsConsumer.accept(new GenericMessage(messageMiddle)); - assertThat(meterRegistry.find("myGauge").gauge().value()).isEqualTo(size(messageMiddle)); - - String messageLarge = "hello world, hello people!"; - analyticsConsumer.accept(new GenericMessage(messageLarge)); - assertThat(meterRegistry.find("myGauge").gauge().value()).isEqualTo(size(messageLarge)); - - } - - private double size(String msg) { - return Long.valueOf(msg.length()).doubleValue(); - } -} diff --git a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/LiteralTagExpressionsTests.java b/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/LiteralTagExpressionsTests.java deleted file mode 100644 index 2b8410d94..000000000 --- a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/LiteralTagExpressionsTests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.analytics; - -import java.util.stream.IntStream; - -import io.micrometer.core.instrument.Counter; -import org.junit.jupiter.api.Test; - -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Christian Tzolov - */ -@TestPropertySource(properties = { - "analytics.name=counter666", - "analytics.tag.expression.foo='bar'", - "analytics.tag.expression.gork='bork'" -}) -public class LiteralTagExpressionsTests extends AnalyticsConsumerParentTest { - - @Test - void testCounterSink() { - - IntStream.range(0, 13).forEach(i -> analyticsConsumer.accept(new GenericMessage("hello"))); - - Counter fooCounter = meterRegistry.find("counter666").tag("foo", "bar").counter(); - assertThat(fooCounter.count()).isEqualTo(13.0); - - Counter gorkCounter = meterRegistry.find("counter666").tag("gork", "bork").counter(); - assertThat(gorkCounter.count()).isEqualTo(13.0); - - assertThat(fooCounter.getId()).isEqualTo(gorkCounter.getId()); - } -} diff --git a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/NullTagsTests.java b/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/NullTagsTests.java deleted file mode 100644 index 40f40a0a1..000000000 --- a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/NullTagsTests.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.analytics; - -import java.util.Collection; - -import io.micrometer.core.instrument.Counter; -import org.junit.jupiter.api.Test; - -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Christian Tzolov - */ -@TestPropertySource(properties = { - "analytics.name=counter666", - "analytics.tag.fixed.foo=", - "analytics.tag.expression.tag666=#jsonPath(payload,'$..noField')", - "analytics.tag.expression.test=#jsonPath(payload,'$..test')" -}) -public class NullTagsTests extends AnalyticsConsumerParentTest { - - @Test - void testАnalyticsSink() { - - analyticsConsumer.accept(message("{\"test\": null}")); - - Collection fixedTagsCounters = meterRegistry.find("counter666").tagKeys("foo").counters(); - assertThat(fixedTagsCounters.size()).isEqualTo(0); - - Collection expressionTagsCounters = meterRegistry.find("counter666").tagKeys("tag666").counters(); - assertThat(expressionTagsCounters.size()).isEqualTo(1); - assertThat(meterRegistry.find("counter666").meter().getId().getTag("tag666")).isEqualTo(AnalyticsConsumerConfiguration.UNAVAILABLE_TAG); - - Collection testExpTagsCounters = meterRegistry.find("counter666").tagKeys("test").counters(); - assertThat(testExpTagsCounters.size()).isEqualTo(1); - assertThat(meterRegistry.find("counter666").meter().getId().getTag("test")).isEqualTo(AnalyticsConsumerConfiguration.UNAVAILABLE_TAG); - } -} diff --git a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/StockExchangeAnalyticsTests.java b/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/StockExchangeAnalyticsTests.java deleted file mode 100644 index c8a463c01..000000000 --- a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/analytics/StockExchangeAnalyticsTests.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.analytics; - -import java.io.IOException; -import java.util.Collection; - -import io.micrometer.core.instrument.Counter; -import org.junit.jupiter.api.Test; - -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.context.TestPropertySource; -import org.springframework.util.StreamUtils; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Christian Tzolov - */ - -@TestPropertySource(properties = { - "analytics.meter-type=counter", - "analytics.name=stocks", - "analytics.tag.expression.symbol=#jsonPath(payload,'$.data.symbol')", - "analytics.tag.expression.exchange=#jsonPath(payload,'$.data.exchange')" -}) -public class StockExchangeAnalyticsTests extends AnalyticsConsumerParentTest { - - @Test - public void testCounter() throws IOException { - byte[] messageAppl = StreamUtils.copyToByteArray( - new DefaultResourceLoader().getResource("classpath:/data/stock_appl.json").getInputStream()); - - analyticsConsumer.accept(MessageBuilder.withPayload(messageAppl).build()); - analyticsConsumer.accept(MessageBuilder.withPayload(messageAppl).build()); - analyticsConsumer.accept(MessageBuilder.withPayload(messageAppl).build()); - - byte[] messageVmw = StreamUtils.copyToByteArray( - new DefaultResourceLoader().getResource("classpath:/data/stock_vmw.json").getInputStream()); - - analyticsConsumer.accept(MessageBuilder.withPayload(messageVmw).build()); - analyticsConsumer.accept(MessageBuilder.withPayload(messageVmw).build()); - - Collection counters = meterRegistry.find("stocks").counters(); - - assertThat(counters).hasSize(2); - - //Iterator itr = counters.iterator(); - // - //Counter applCounter = itr.next(); - //assertThat(applCounter.count()).isEqualTo(3); - //assertThat(applCounter.getId().getTags()).contains(Tag.of("symbol", "AAPL"), Tag.of("exchange", "XNAS")); - // - //Counter vmwCounter = itr.next(); - //assertThat(vmwCounter.count()).isEqualTo(2); - //assertThat(vmwCounter.getId().getTags()).contains(Tag.of("symbol", "VMW"), Tag.of("exchange", "NYSE")); - - } -} diff --git a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/demo/StockExchangeAnalyticsExample.java b/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/demo/StockExchangeAnalyticsExample.java deleted file mode 100644 index 2f74f52dc..000000000 --- a/functions/consumer/analytics-consumer/src/test/java/org/springframework/cloud/fn/consumer/demo/StockExchangeAnalyticsExample.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.demo; - -import java.util.Random; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import io.micrometer.core.instrument.MeterRegistry; - -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.cloud.fn.consumer.analytics.AnalyticsConsumerConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; - -/** - * Sample Spring Boot Application that uses the analyticsConsumer to compute running stats from - * stock exchange messages. - * - * Counter configuration: - * - * --analytics.meter-type=counter - * --analytics.name=stocks - * --analytics.tag.expression.symbol=#jsonPath(payload,'$.data.symbol') - * --analytics.tag.expression.exchange=#jsonPath(payload,'$.data.exchange') - * - * - * Gauge configuration: - * - * --analytics.meter-type=gauge - * --analytics.name=stocks - * --analytics.tag.expression.symbol=#jsonPath(payload,'$.data.symbol') - * --analytics.tag.expression.exchange=#jsonPath(payload,'$.data.exchange') - * --analytics.amount-expression=#jsonPath(payload,'$.data.volume') - * - * - * Sample Wavefront configuration: - * - * --management.metrics.export.wavefront.enabled=true - * --management.metrics.export.wavefront.uri=YOUR_WAVEFRONT_SERVER_URI - * --management.metrics.export.wavefront.api-token=YOUR_API_TOKEN - * --management.metrics.export.wavefront.source=stock-exchange-demo - * - * - * @author Christian Tzolov - */ -@Import(AnalyticsConsumerConfiguration.class) -@SpringBootApplication -public class StockExchangeAnalyticsExample { - - public static void main(String[] args) { - SpringApplication.run(StockExchangeAnalyticsExample.class, args); - } - - @Bean - public CommandLineRunner commandLineRunner(Consumer> analyticsConsumer, - MeterRegistry meterRegistry, Supplier stockMessageGenerator) { - - // Run every second. - return args -> Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { - - String message = stockMessageGenerator.get(); - - // Submit new message using the stockMessageGenerator to generate random stock messages. - analyticsConsumer.accept(MessageBuilder.withPayload(message).build()); - - // Print current stock meters - System.out.println(meterRegistry.getMeters().stream() - .filter(meter -> meter.getId().getName().contains("stocks")) - .map(meter -> meter.getId().getType() + " | " + meter.getId() + " | " + meter.measure()) - .collect(Collectors.joining("\n")) + - "\n========================================================================="); - - }, 0, 1000, TimeUnit.MILLISECONDS); - } - - @Bean - public Supplier stockMessageGenerator() { - final Random random = new Random(); - final String[][] STOCKS = new String[][] { { "NASDAQ", "GOOGL" }, { "AMS", "TOM2" }, { "NYSE", "CLDR" }, - { "NYSE", "VMW" }, { "NYSE", "IBM" }, { "NASDAQ", "MSFT" }, { "NASDAQ", "AAPL" } }; - - return () -> { - int stockIndex = random.nextInt(STOCKS.length); - return "{\n" + - " \"data\": {\n" + - " \"symbol\": \"" + STOCKS[stockIndex][1] + "\",\n" + - " \"exchange\": \"" + STOCKS[stockIndex][0] + "\",\n" + - " \"open\": " + (1 + 10 * random.nextDouble()) + ",\n" + - " \"close\": " + (1 + 10 * random.nextDouble()) + ",\n" + - " \"volume\": " + (1000 + 100000 * random.nextDouble()) + "\n" + - " }\n" + - "}"; - }; - } -} - diff --git a/functions/consumer/analytics-consumer/src/test/resources/data/stock_appl.json b/functions/consumer/analytics-consumer/src/test/resources/data/stock_appl.json deleted file mode 100644 index 887a0adf3..000000000 --- a/functions/consumer/analytics-consumer/src/test/resources/data/stock_appl.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "data": { - "date": "2020-05-21T00:00:00+0000", - "symbol": "AAPL", - "exchange": "XNAS", - "open": 318.66, - "high": 320.89, - "low": 315.87, - "close": 316.85, - "volume": 25672211.0 - } -} diff --git a/functions/consumer/analytics-consumer/src/test/resources/data/stock_vmw.json b/functions/consumer/analytics-consumer/src/test/resources/data/stock_vmw.json deleted file mode 100644 index 752d43e6e..000000000 --- a/functions/consumer/analytics-consumer/src/test/resources/data/stock_vmw.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "data": { - "date": "2020-05-21T00:00:00+0000", - "symbol": "VMW", - "exchange": "NYSE", - "open": 160.16, - "high": 180.91, - "low": 165.87, - "close": 170.85, - "volume": 5672211.0 - } -} diff --git a/functions/consumer/cassandra-consumer/README.adoc b/functions/consumer/cassandra-consumer/README.adoc deleted file mode 100644 index fdf9886f3..000000000 --- a/functions/consumer/cassandra-consumer/README.adoc +++ /dev/null @@ -1,32 +0,0 @@ -# Cassandra Consumer - -This module provides a Cassandra Consumer that can be reused and composed in other applications. -Internally it uses the `CassandraMessageHandler` from Spring Integration. -`CassandraConsumerFunction` is implemented as a `java.util.function.Function`. - -## Beans for injection - -You can import the `CassnadraConsumerConfiguration` in the application and then inject the following bean. - -`cassandraConsumerFunction` - -You can use `cassandraConsumerFunction` as a qualifier when injecting. - -Type for injection: `Function>` - -You can ignore the return value from the function as this is a consumer and simply will send the data to Cassandra. - -## Configuration Options - -All configuration properties are prefixed with either `cassandra` or `cassandra.cluster`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerProperties.java[CassandraConsumerProperties]. -See link:src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraClusterProperties.java[this] also. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/cassandra[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/main/applications/sink/cassandra-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a Cassandra sink. diff --git a/functions/consumer/cassandra-consumer/pom.xml b/functions/consumer/cassandra-consumer/pom.xml deleted file mode 100644 index c164751d0..000000000 --- a/functions/consumer/cassandra-consumer/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - cassandra-consumer - cassandra-consumer - Cassandra Consumer - - - - org.springframework.boot - spring-boot-starter-data-cassandra-reactive - - - org.springframework.integration - spring-integration-cassandra - - - org.testcontainers - cassandra - ${testcontainers.version} - test - - - - diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerConfiguration.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerConfiguration.java deleted file mode 100644 index 2750c32ae..000000000 --- a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerConfiguration.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra; - -import java.sql.Date; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; -import java.util.function.Function; - -import com.datastax.oss.driver.api.core.ConsistencyLevel; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.util.StdDateFormat; -import reactor.core.publisher.Mono; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.consumer.cassandra.cluster.CassandraAppClusterConfiguration; -import org.springframework.cloud.fn.consumer.cassandra.query.ColumnNameExtractor; -import org.springframework.cloud.fn.consumer.cassandra.query.InsertQueryColumnNameExtractor; -import org.springframework.cloud.fn.consumer.cassandra.query.UpdateQueryColumnNameExtractor; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.data.cassandra.core.InsertOptions; -import org.springframework.data.cassandra.core.ReactiveCassandraOperations; -import org.springframework.data.cassandra.core.UpdateOptions; -import org.springframework.data.cassandra.core.WriteResult; -import org.springframework.data.cassandra.core.cql.WriteOptions; -import org.springframework.integration.JavaUtils; -import org.springframework.integration.cassandra.outbound.CassandraMessageHandler; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.dsl.IntegrationFlowBuilder; -import org.springframework.integration.support.json.Jackson2JsonObjectMapper; -import org.springframework.integration.transformer.AbstractPayloadTransformer; -import org.springframework.messaging.MessageHandler; -import org.springframework.util.StringUtils; - -/** - * @author Artem Bilan - * @author Thomas Risberg - * @author Ashu Gairola - * @author Akos Ratku - */ -@AutoConfiguration -@EnableConfigurationProperties(CassandraConsumerProperties.class) -@Import(CassandraAppClusterConfiguration.class) -public class CassandraConsumerConfiguration { - - @Autowired - private CassandraConsumerProperties cassandraSinkProperties; - - @Bean - public IntegrationFlow cassandraConsumerFlow(MessageHandler cassandraSinkMessageHandler, - ObjectMapper objectMapper) { - - IntegrationFlowBuilder integrationFlowBuilder = IntegrationFlow.from(CassandraConsumerFunction.class); - String ingestQuery = this.cassandraSinkProperties.getIngestQuery(); - if (StringUtils.hasText(ingestQuery)) { - integrationFlowBuilder.transform( - new PayloadToMatrixTransformer(objectMapper, ingestQuery, - CassandraMessageHandler.Type.UPDATE == this.cassandraSinkProperties.getQueryType() - ? new UpdateQueryColumnNameExtractor() - : new InsertQueryColumnNameExtractor())); - } - return integrationFlowBuilder - .handle(cassandraSinkMessageHandler) - .get(); - } - - @Bean - public MessageHandler cassandraSinkMessageHandler(ReactiveCassandraOperations cassandraOperations) { - CassandraMessageHandler.Type queryType = - Optional.ofNullable(this.cassandraSinkProperties.getQueryType()) - .orElse(CassandraMessageHandler.Type.INSERT); - - CassandraMessageHandler cassandraMessageHandler = new CassandraMessageHandler(cassandraOperations, queryType); - cassandraMessageHandler.setProducesReply(true); - int ttl = this.cassandraSinkProperties.getTtl(); - ConsistencyLevel consistencyLevel = this.cassandraSinkProperties.getConsistencyLevel(); - if (consistencyLevel != null || ttl > 0) { - - WriteOptions.WriteOptionsBuilder writeOptionsBuilder = - switch (queryType) { - case INSERT -> InsertOptions.builder(); - case UPDATE -> UpdateOptions.builder(); - default -> WriteOptions.builder(); - }; - - JavaUtils.INSTANCE - .acceptIfNotNull(consistencyLevel, writeOptionsBuilder::consistencyLevel) - .acceptIfCondition(ttl > 0, ttl, writeOptionsBuilder::ttl); - - cassandraMessageHandler.setWriteOptions(writeOptionsBuilder.build()); - } - - JavaUtils.INSTANCE - .acceptIfHasText(this.cassandraSinkProperties.getIngestQuery(), cassandraMessageHandler::setIngestQuery) - .acceptIfNotNull(this.cassandraSinkProperties.getStatementExpression(), - cassandraMessageHandler::setStatementExpression); - - return cassandraMessageHandler; - } - - private static boolean isUuid(String uuid) { - if (uuid.length() == 36) { - String[] parts = uuid.split("-"); - if (parts.length == 5) { - return (parts[0].length() == 8) && (parts[1].length() == 4) && - (parts[2].length() == 4) && (parts[3].length() == 4) && - (parts[4].length() == 12); - } - } - return false; - } - - - private static class PayloadToMatrixTransformer extends AbstractPayloadTransformer>> { - - private final Jackson2JsonObjectMapper jsonObjectMapper; - - private final List columns = new LinkedList<>(); - - private final ISO8601StdDateFormat dateFormat = new ISO8601StdDateFormat(); - - PayloadToMatrixTransformer(ObjectMapper objectMapper, String query, ColumnNameExtractor columnNameExtractor) { - this.jsonObjectMapper = new Jackson2JsonObjectMapper(objectMapper); - this.columns.addAll(columnNameExtractor.extract(query)); - this.jsonObjectMapper.getObjectMapper() - .configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); - } - - @Override - @SuppressWarnings("unchecked") - protected List> transformPayload(Object payload) { - if (payload instanceof List) { - return (List>) payload; - } - else { - try { - List> model = this.jsonObjectMapper.fromJson(payload, List.class); - List> data = new ArrayList<>(model.size()); - for (Map entity : model) { - List row = new ArrayList<>(this.columns.size()); - for (String column : this.columns) { - Object value = entity.get(column); - if (value instanceof String string) { - if (this.dateFormat.looksLikeISO8601(string)) { - synchronized (this.dateFormat) { - value = new Date(this.dateFormat.parse(string).getTime()).toLocalDate(); - } - } - if (isUuid(string)) { - value = UUID.fromString(string); - } - } - row.add(value); - } - data.add(row); - } - return data; - } - catch (Exception ex) { - throw new IllegalArgumentException("Cannot parse json into matrix", ex); - } - } - } - - } - - /* - * We need this to provide visibility to the protected method. - */ - @SuppressWarnings("serial") - private static class ISO8601StdDateFormat extends StdDateFormat { - - @Override - protected boolean looksLikeISO8601(String dateStr) { - return super.looksLikeISO8601(dateStr); - } - - } - - interface CassandraConsumerFunction extends Function> { - - } - -} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerProperties.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerProperties.java deleted file mode 100644 index 13869726e..000000000 --- a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerProperties.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2019-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra; - -import com.datastax.oss.driver.api.core.ConsistencyLevel; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.integration.cassandra.outbound.CassandraMessageHandler; - -/** - * @author Artem Bilan - * @author Thomas Risberg - */ -@ConfigurationProperties("cassandra") -public class CassandraConsumerProperties { - - /** - * Time-to-live option of WriteOptions. - */ - private int ttl; - - /** - * QueryType for Cassandra Sink. - */ - private CassandraMessageHandler.Type queryType; - - /** - * Ingest Cassandra query. - */ - private String ingestQuery; - - /** - * Expression in Cassandra query DSL style. - */ - private Expression statementExpression; - - /** - * The consistency level for write operation. - */ - private ConsistencyLevel consistencyLevel; - - public int getTtl() { - return this.ttl; - } - - public void setTtl(int ttl) { - this.ttl = ttl; - } - - public CassandraMessageHandler.Type getQueryType() { - return this.queryType; - } - - public void setQueryType(CassandraMessageHandler.Type queryType) { - this.queryType = queryType; - } - - public String getIngestQuery() { - return this.ingestQuery; - } - - public void setIngestQuery(String ingestQuery) { - this.ingestQuery = ingestQuery; - } - - public Expression getStatementExpression() { - return this.statementExpression; - } - - public void setStatementExpression(Expression statementExpression) { - this.statementExpression = statementExpression; - } - - public ConsistencyLevel getConsistencyLevel() { - return this.consistencyLevel; - } - - public void setConsistencyLevel(ConsistencyLevel consistencyLevel) { - this.consistencyLevel = consistencyLevel; - } - -} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraAppClusterConfiguration.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraAppClusterConfiguration.java deleted file mode 100644 index 89b5501af..000000000 --- a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraAppClusterConfiguration.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2019-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra.cluster; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.util.Arrays; -import java.util.Scanner; - -import com.datastax.oss.driver.api.core.CqlSession; -import com.datastax.oss.driver.api.core.CqlSessionBuilder; -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.BeanInitializationException; -import org.springframework.beans.factory.support.BeanDefinitionRegistry; -import org.springframework.boot.autoconfigure.cassandra.CassandraProperties; -import org.springframework.boot.autoconfigure.cassandra.CqlSessionBuilderCustomizer; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.autoconfigure.domain.EntityScanPackages; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.boot.context.properties.PropertyMapper; -import org.springframework.boot.context.properties.bind.Binder; -import org.springframework.context.EnvironmentAware; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.DependsOn; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.ImportBeanDefinitionRegistrar; -import org.springframework.context.annotation.Lazy; -import org.springframework.core.env.Environment; -import org.springframework.core.type.AnnotationMetadata; -import org.springframework.data.cassandra.config.CqlSessionFactoryBean; -import org.springframework.data.cassandra.core.ReactiveCassandraTemplate; -import org.springframework.data.cassandra.core.cql.ReactiveCqlOperations; -import org.springframework.data.cassandra.core.cql.generator.CreateKeyspaceCqlGenerator; -import org.springframework.data.cassandra.core.cql.keyspace.CreateKeyspaceSpecification; -import org.springframework.util.StringUtils; - -/** - * @author Artem Bilan - * @author Thomas Risberg - * @author Rob Hardt - */ -@Configuration -@EnableConfigurationProperties(CassandraClusterProperties.class) -@Import(CassandraAppClusterConfiguration.CassandraPackageRegistrar.class) -public class CassandraAppClusterConfiguration { - - @Bean - public CqlSessionBuilderCustomizer clusterBuilderCustomizer( - CassandraClusterProperties cassandraClusterProperties) { - - PropertyMapper map = PropertyMapper.get(); - return builder -> - map.from(cassandraClusterProperties::isSkipSslValidation) - .whenTrue() - .toCall(() -> { - try { - builder.withSslContext(TrustAllSSLContextFactory.getSslContext()); - } - catch (NoSuchAlgorithmException | KeyManagementException ex) { - throw new BeanInitializationException( - "Unable to configure a Cassandra cluster using SSL.", ex); - } - - }); - } - - @Bean - @ConditionalOnProperty("cassandra.cluster.create-keyspace") - public Object keyspaceCreator(CassandraProperties cassandraProperties, CqlSessionBuilder cqlSessionBuilder) { - CreateKeyspaceSpecification createKeyspaceSpecification = - CreateKeyspaceSpecification - .createKeyspace(cassandraProperties.getKeyspaceName()) - .withSimpleReplication() - .ifNotExists(); - - String createKeySpaceQuery = new CreateKeyspaceCqlGenerator(createKeyspaceSpecification).toCql(); - try (var systemSession = cqlSessionBuilder.withKeyspace(CqlSessionFactoryBean.CASSANDRA_SYSTEM_SESSION).build()) { - systemSession.execute(createKeySpaceQuery); - } - - return null; - } - - @Bean - @Lazy - @DependsOn("keyspaceCreator") - public CqlSession cassandraSession(CqlSessionBuilder cqlSessionBuilder) { - return cqlSessionBuilder.build(); - } - - - @Bean - @ConditionalOnProperty("cassandra.cluster.init-script") - public Object keyspaceInitializer(CassandraClusterProperties cassandraClusterProperties, - ReactiveCassandraTemplate reactiveCassandraTemplate) throws IOException { - - String scripts = - new Scanner(cassandraClusterProperties.getInitScript().getInputStream(), - StandardCharsets.UTF_8) - .useDelimiter("\\A") - .next(); - - ReactiveCqlOperations reactiveCqlOperations = - reactiveCassandraTemplate.getReactiveCqlOperations(); - - Flux.fromArray(StringUtils.delimitedListToStringArray(scripts, ";", "\r\n\f")) - .filter(StringUtils::hasText) // an empty String after the last ';' - .concatMap(script -> reactiveCqlOperations.execute(script + ";")) - .blockLast(); - - return null; - - } - - static class CassandraPackageRegistrar implements ImportBeanDefinitionRegistrar, EnvironmentAware { - - private Environment environment; - - @Override - public void setEnvironment(Environment environment) { - this.environment = environment; - } - - @Override - public void registerBeanDefinitions(AnnotationMetadata importingClassMetadata, - BeanDefinitionRegistry registry) { - - Binder.get(this.environment) - .bind("cassandra.cluster.entity-base-packages", String[].class) - .map(Arrays::asList) - .ifBound(packagesToScan -> EntityScanPackages.register(registry, packagesToScan)); - } - - } - -} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraClusterProperties.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraClusterProperties.java deleted file mode 100644 index 0d2f46aa7..000000000 --- a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/CassandraClusterProperties.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra.cluster; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.core.io.Resource; - -/** - * Common properties for the cassandra modules. - * - * @author Artem Bilan - * @author Thomas Risberg - * @author Rob Hardt - */ -@ConfigurationProperties("cassandra.cluster") -public class CassandraClusterProperties { - - /** - * Flag to create (or not) keyspace on application startup. - */ - private boolean createKeyspace; - - /** - * Resource with CQL scripts (delimited by ';') to initialize keyspace schema. - */ - private Resource initScript; - - /** - * Flag to validate the Servers' SSL certs. - */ - private boolean skipSslValidation; - - /** - * Base packages to scan for entities annotated with Table annotations. - */ - private String[] entityBasePackages = { }; - - - public void setCreateKeyspace(boolean createKeyspace) { - this.createKeyspace = createKeyspace; - } - - public void setInitScript(Resource initScript) { - this.initScript = initScript; - } - - public void setSkipSslValidation(boolean skipSslValidation) { - this.skipSslValidation = skipSslValidation; - } - - public boolean isCreateKeyspace() { - return this.createKeyspace; - } - - public Resource getInitScript() { - return this.initScript; - } - - public boolean isSkipSslValidation() { - return this.skipSslValidation; - } - - public String[] getEntityBasePackages() { - return this.entityBasePackages; - } - - public void setEntityBasePackages(String[] entityBasePackages) { - this.entityBasePackages = entityBasePackages; - } - -} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/TrustAllSSLContextFactory.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/TrustAllSSLContextFactory.java deleted file mode 100644 index 5987b88ba..000000000 --- a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/cluster/TrustAllSSLContextFactory.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2019-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra.cluster; - -import java.security.KeyManagementException; -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; -import java.security.cert.X509Certificate; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.X509TrustManager; - - -/** - * Helper to provide an SSL Context that does not validate - * certificates presented in the SSL handshake. - * - * The usual caveats apply. - * - * @author Rob Hardt - * @author Artem Bilan - */ -final class TrustAllSSLContextFactory { - - private TrustAllSSLContextFactory() { - - } - - static SSLContext getSslContext() throws NoSuchAlgorithmException, KeyManagementException { - - TrustManager[] trustAllCerts = new TrustManager[] { - new X509TrustManager() { - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - - @Override - public void checkClientTrusted(X509Certificate[] certs, String authType) { - } - - @Override - public void checkServerTrusted(X509Certificate[] certs, String authType) { - } - - } - }; - - SSLContext sc = SSLContext.getInstance("SSL"); - sc.init(null, trustAllCerts, new SecureRandom()); - return sc; - } - -} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/ColumnNameExtractor.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/ColumnNameExtractor.java deleted file mode 100644 index 92bb70b94..000000000 --- a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/ColumnNameExtractor.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2017-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra.query; - -import java.util.List; - -/** - * @author Akos Ratku - * @author Artem Bilan - */ -@FunctionalInterface -public interface ColumnNameExtractor { - - List extract(String query); - -} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/InsertQueryColumnNameExtractor.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/InsertQueryColumnNameExtractor.java deleted file mode 100644 index befdc6ca9..000000000 --- a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/InsertQueryColumnNameExtractor.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2017-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra.query; - -import java.util.LinkedList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.springframework.util.StringUtils; - -/** - * @author Akos Ratku - * @author Artem Bilan - */ -public class InsertQueryColumnNameExtractor implements ColumnNameExtractor { - - private static final Pattern PATTERN = Pattern.compile(".+\\((.+)\\).+(?:values\\s*\\((.+)\\))"); - - @Override - public List extract(String query) { - List extractedColumns = new LinkedList<>(); - Matcher matcher = PATTERN.matcher(query); - if (matcher.matches()) { - String[] columns = StringUtils.delimitedListToStringArray(matcher.group(1), ",", " "); - String[] params = StringUtils.delimitedListToStringArray(matcher.group(2), ",", " "); - for (int i = 0; i < columns.length; i++) { - String param = params[i]; - if (param.equals("?")) { - extractedColumns.add(columns[i]); - } - else if (param.startsWith(":")) { - extractedColumns.add(param.substring(1)); - } - } - } - else { - throw new IllegalArgumentException("Invalid CQL insert query syntax: " + query); - } - return extractedColumns; - } - -} diff --git a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/UpdateQueryColumnNameExtractor.java b/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/UpdateQueryColumnNameExtractor.java deleted file mode 100644 index 7352f961d..000000000 --- a/functions/consumer/cassandra-consumer/src/main/java/org/springframework/cloud/fn/consumer/cassandra/query/UpdateQueryColumnNameExtractor.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2017-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra.query; - -import java.util.LinkedList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.springframework.util.StringUtils; - -/** - * @author Akos Ratku - * @author Artem Bilan - */ -public class UpdateQueryColumnNameExtractor implements ColumnNameExtractor { - - private static final Pattern PATTERN = Pattern.compile("(?i)(?<=set)(.*)(?=where)where(.*)"); - - @Override - public List extract(String query) { - List extractedColumns = new LinkedList<>(); - Matcher matcher = PATTERN.matcher(query); - if (matcher.find()) { - String[] settings = StringUtils.delimitedListToStringArray(matcher.group(1), ",", " "); - String[] where = StringUtils.delimitedListToStringArray(matcher.group(2), ",", " "); - readPairs(extractedColumns, settings); - readPairs(extractedColumns, where); - } - else { - throw new IllegalArgumentException("Invalid CQL update query syntax: " + query); - } - return extractedColumns; - } - - protected void readPairs(List extractedColumns, String[] settings) { - for (String setting : settings) { - String[] columnValuePair = StringUtils.delimitedListToStringArray(setting, "=", " "); - if (columnValuePair[1].startsWith(":") || columnValuePair[1].equals("?")) { - extractedColumns.add(columnValuePair[0]); - } - } - } - -} diff --git a/functions/consumer/cassandra-consumer/src/main/resources/application.properties b/functions/consumer/cassandra-consumer/src/main/resources/application.properties deleted file mode 100644 index 8b1378917..000000000 --- a/functions/consumer/cassandra-consumer/src/main/resources/application.properties +++ /dev/null @@ -1 +0,0 @@ - diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerApplicationTests.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerApplicationTests.java deleted file mode 100644 index 9ef3b8107..000000000 --- a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraConsumerApplicationTests.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2019-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra; - -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; -import java.util.function.Function; - -import org.junit.jupiter.api.AfterEach; -import reactor.core.publisher.Mono; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.consumer.cassandra.domain.Book; -import org.springframework.context.annotation.Import; -import org.springframework.data.cassandra.core.CassandraOperations; -import org.springframework.data.cassandra.core.WriteResult; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; - -/** - * @author Artem Bilan - */ -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { - "spring.cassandra.keyspace-name=" + CassandraConsumerApplicationTests.CASSANDRA_KEYSPACE, - "cassandra.cluster.createKeyspace=true" }) -@DirtiesContext -abstract class CassandraConsumerApplicationTests implements CassandraContainerTest { - - static final String CASSANDRA_KEYSPACE = "test"; - - @Autowired - protected CassandraOperations cassandraTemplate; - - @Autowired - protected Function> cassandraConsumer; - - - @DynamicPropertySource - static void registerConfigurationProperties(DynamicPropertyRegistry registry) { - registry.add("spring.cassandra.localDatacenter", () -> CASSANDRA_CONTAINER.getLocalDatacenter()); - registry.add("spring.cassandra.contactPoints", () -> - Optional.of(CASSANDRA_CONTAINER.getContactPoint()) - .map(contactPoint -> contactPoint.getAddress().getHostAddress() + ':' + contactPoint.getPort()) - .get()); - } - - @AfterEach - void tearDown() { - this.cassandraTemplate.truncate(Book.class); - } - - protected static List getBookList(int numBooks) { - - List books = new ArrayList<>(); - for (int i = 0; i < numBooks; i++) { - books.add( - new Book( - UUID.randomUUID(), - "Spring Cloud Data Flow Guide", - "SCDF Guru", - i * 10 + 5, - LocalDate.now(), - true)); - } - - return books; - } - - @SpringBootConfiguration - @EnableAutoConfiguration - @Import(CassandraConsumerConfiguration.class) - static class CassandraConsumerTestApplication { - - } - -} diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraContainerTest.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraContainerTest.java deleted file mode 100644 index 2a99ea7e8..000000000 --- a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraContainerTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2022-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra; - -import org.junit.jupiter.api.BeforeAll; -import org.testcontainers.containers.CassandraContainer; -import org.testcontainers.junit.jupiter.Testcontainers; - -/** - * The base contract for JUnit tests based on the container for Apache Cassandra. - * The Testcontainers 'reuse' option must be disabled,so, Ryuk container is started - * and will clean all the containers up from this test suite after JVM exit. - * Since the MqSQL container instance is shared via static property, it is going to be - * started only once per JVM, therefore the target Docker container is reused automatically. - * - * @author Artem Bilan - * - * @since 6.0 - */ -@Testcontainers(disabledWithoutDocker = true) -public interface CassandraContainerTest { - - CassandraContainer CASSANDRA_CONTAINER = new CassandraContainer<>("cassandra:4.1"); - - @BeforeAll - static void startContainer() { - System.setProperty("datastax-java-driver.basic.request.timeout", "10 seconds"); - CASSANDRA_CONTAINER.start(); - } - -} diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraEntityInsertTests.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraEntityInsertTests.java deleted file mode 100644 index f2b329c77..000000000 --- a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraEntityInsertTests.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2019-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra; - -import java.time.LocalDate; -import java.util.UUID; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Mono; -import reactor.test.StepVerifier; - -import org.springframework.cloud.fn.consumer.cassandra.domain.Book; -import org.springframework.data.cassandra.core.WriteResult; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Artem Bilan - */ -@TestPropertySource(properties = { - "spring.cassandra.schema-action=RECREATE", - "cassandra.cluster.entity-base-packages=org.springframework.cloud.fn.consumer.cassandra.domain" }) -class CassandraEntityInsertTests extends CassandraConsumerApplicationTests { - - @Test - void testInsert() { - Book book = - new Book( - UUID.randomUUID(), - "Spring Integration Cassandra", - "Cassandra Guru", - 521, - LocalDate.now(), - true); - - Mono result = this.cassandraConsumer.apply(book); - - StepVerifier.create(result) - .expectNextCount(1) - .then(() -> - assertThat(this.cassandraTemplate.query(Book.class) - .count()) - .isEqualTo(1)) - .verifyComplete(); - } - -} diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestInsertTests.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestInsertTests.java deleted file mode 100644 index 57643b9e8..000000000 --- a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestInsertTests.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2019-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra; - -import java.util.List; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Mono; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cloud.fn.consumer.cassandra.domain.Book; -import org.springframework.data.cassandra.core.WriteResult; -import org.springframework.integration.support.json.Jackson2JsonObjectMapper; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Artem Bilan - */ -@TestPropertySource(properties = { - "cassandra.cluster.init-script=init-db.cql", - "cassandra.ingest-query=" + - "insert into book (isbn, title, author, pages, saleDate, inStock) values (?, ?, ?, ?, ?, ?)" }) -class CassandraIngestInsertTests extends CassandraConsumerApplicationTests { - - @Test - void testIngestQuery(@Autowired ObjectMapper objectMapper) throws Exception { - List books = getBookList(5); - - Jackson2JsonObjectMapper mapper = new Jackson2JsonObjectMapper(objectMapper); - - Mono result = - this.cassandraConsumer.apply(mapper.toJson(books)); - - StepVerifier.create(result) - .expectNextCount(1) - .then(() -> - assertThat(this.cassandraTemplate.query(Book.class) - .count()) - .isEqualTo(5)) - .verifyComplete(); - } - -} diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestNamedParamsTests.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestNamedParamsTests.java deleted file mode 100644 index 63b9bd574..000000000 --- a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestNamedParamsTests.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2019-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra; - -import java.util.List; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Mono; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cloud.fn.consumer.cassandra.domain.Book; -import org.springframework.data.cassandra.core.WriteResult; -import org.springframework.integration.support.json.Jackson2JsonObjectMapper; -import org.springframework.test.context.TestPropertySource; -import org.springframework.util.StringUtils; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Artem Bilan - */ -@TestPropertySource(properties = { - "cassandra.cluster.init-script=init-db.cql", - "cassandra.ingest-query=" + - "insert into book (isbn, title, author, pages, saleDate, inStock) " + - "values (:myIsbn, :myTitle, :myAuthor, ?, ?, ?)" }) -class CassandraIngestNamedParamsTests extends CassandraConsumerApplicationTests { - - @Test - void testIngestQuery(@Autowired ObjectMapper objectMapper) throws Exception { - List books = getBookList(5); - - Jackson2JsonObjectMapper mapper = new Jackson2JsonObjectMapper(objectMapper); - - String booksJsonWithNamedParams = mapper.toJson(books); - booksJsonWithNamedParams = StringUtils.replace(booksJsonWithNamedParams, "isbn", "myIsbn"); - booksJsonWithNamedParams = StringUtils.replace(booksJsonWithNamedParams, "title", "myTitle"); - booksJsonWithNamedParams = StringUtils.replace(booksJsonWithNamedParams, "author", "myAuthor"); - - Mono result = - this.cassandraConsumer.apply(booksJsonWithNamedParams); - - StepVerifier.create(result) - .expectNextCount(1) - .then(() -> - assertThat(this.cassandraTemplate.query(Book.class) - .count()) - .isEqualTo(5)) - .verifyComplete(); - } - -} diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestUpdateTests.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestUpdateTests.java deleted file mode 100644 index 8d5156ffb..000000000 --- a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/CassandraIngestUpdateTests.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2019-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra; - -import java.util.List; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Mono; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cloud.fn.consumer.cassandra.domain.Book; -import org.springframework.data.cassandra.core.WriteResult; -import org.springframework.integration.support.json.Jackson2JsonObjectMapper; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Artem Bilan - */ -@TestPropertySource(properties = { - "cassandra.cluster.init-script=init-db.cql", - "cassandra.ingest-query=" + - "update book set inStock = :inStock, author = :author, pages = :pages, " + - "saleDate = :saleDate, title = :title where isbn = :isbn", - "cassandra.queryType=UPDATE" }) -class CassandraIngestUpdateTests extends CassandraConsumerApplicationTests { - - @Test - void testIngestQuery(@Autowired ObjectMapper objectMapper) throws Exception { - List books = getBookList(5); - - Jackson2JsonObjectMapper mapper = new Jackson2JsonObjectMapper(objectMapper); - - Mono result = - this.cassandraConsumer.apply(mapper.toJson(books)); - - StepVerifier.create(result) - .expectNextCount(1) - .then(() -> - assertThat(this.cassandraTemplate.query(Book.class) - .count()) - .isEqualTo(5)) - .verifyComplete(); - } - -} diff --git a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/domain/Book.java b/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/domain/Book.java deleted file mode 100644 index 7ac253fed..000000000 --- a/functions/consumer/cassandra-consumer/src/test/java/org/springframework/cloud/fn/consumer/cassandra/domain/Book.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2019-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.cassandra.domain; - -import java.time.LocalDate; -import java.util.UUID; - -import org.springframework.data.cassandra.core.mapping.Indexed; -import org.springframework.data.cassandra.core.mapping.PrimaryKey; -import org.springframework.data.cassandra.core.mapping.Table; - -/** - * Test POJO. - * - * @author David Webb - * @author Artem Bilan - */ -@Table("book") -public record Book( - @PrimaryKey UUID isbn, - String title, - @Indexed String author, - Integer pages, - LocalDate saleDate, - Boolean isInStock) { - -} diff --git a/functions/consumer/cassandra-consumer/src/test/resources/init-db.cql b/functions/consumer/cassandra-consumer/src/test/resources/init-db.cql deleted file mode 100644 index 6c1397dc8..000000000 --- a/functions/consumer/cassandra-consumer/src/test/resources/init-db.cql +++ /dev/null @@ -1,10 +0,0 @@ -DROP TABLE IF EXISTS book; - -CREATE TABLE book ( - isbn uuid PRIMARY KEY, - author text, - instock boolean, - pages int, - saledate date, - title text -); diff --git a/functions/consumer/elasticsearch-consumer/README.adoc b/functions/consumer/elasticsearch-consumer/README.adoc deleted file mode 100644 index 8c6a8fa50..000000000 --- a/functions/consumer/elasticsearch-consumer/README.adoc +++ /dev/null @@ -1,35 +0,0 @@ -# Elasticsearch Consumer - -A consumer that allows you to index document records into Elasticsearch. - -## Beans for injection - -You can import `ElasticsearchConsumerConfiguration` in the application and then inject the following bean. - -`Consumer> elasticsearchConsumer` - -After injecting this `Consumer` bean in the application, it's `accept` method can be called programmatically to index Elasticsearch documents. -This consumer only supports indexing documents with JSON formats. -The JSON document can be provided using one of the following methods. - -* JSON string -* `java.util.Map` -* `XContentBuilder` provided by Elasticsearch - -## Configuration Options - -All configuration properties are prefixed with `elasticsearch.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerConfiguration.java[ElasticsearchConsumerProperties]. - -In addition to these options, the consumer makes use of Spring Boot autoconfiguration for Elasticsearch. -Therefore you need to use https://github.com/spring-projects/spring-boot/blob/master/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/elasticsearch/ElasticsearchRestClientProperties.java[these properties] for configuration Elasticsearch. -See this https://docs.spring.io/spring-boot/docs/current/reference/htmlsingle/#boot-features-elasticsearch[section] from Spring Boot docs. - -## Examples - -See this link:src/test/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerApplicationTests.java[test suite] for seeing the Elasticsearch consumer in action. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/elasticsearch-sink/README.adoc[README] where this consumer is used for creating an Elasticsearch sink. \ No newline at end of file diff --git a/functions/consumer/elasticsearch-consumer/pom.xml b/functions/consumer/elasticsearch-consumer/pom.xml deleted file mode 100644 index 829b8791e..000000000 --- a/functions/consumer/elasticsearch-consumer/pom.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - - 8.5.0 - - - elasticsearch-consumer - elasticsearch-consumer - elasticsearch consumer - - - - co.elastic.clients - elasticsearch-java - ${elasticsearch.version} - - - org.elasticsearch - elasticsearch-x-content - ${elasticsearch.version} - - - - org.springframework.boot - spring-boot-starter-data-elasticsearch - - - org.springframework.cloud.fn - payload-converter-function - ${project.version} - - - org.testcontainers - elasticsearch - ${testcontainers.version} - test - - - diff --git a/functions/consumer/elasticsearch-consumer/src/main/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerConfiguration.java b/functions/consumer/elasticsearch-consumer/src/main/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerConfiguration.java deleted file mode 100644 index 636a22a6f..000000000 --- a/functions/consumer/elasticsearch-consumer/src/main/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerConfiguration.java +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright 2020-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.elasticsearch; - -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.function.Consumer; -import java.util.stream.StreamSupport; - -import co.elastic.clients.elasticsearch.ElasticsearchAsyncClient; -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.elasticsearch._types.Time; -import co.elastic.clients.elasticsearch.core.BulkRequest; -import co.elastic.clients.elasticsearch.core.BulkResponse; -import co.elastic.clients.elasticsearch.core.IndexRequest; -import co.elastic.clients.elasticsearch.core.IndexResponse; -import co.elastic.clients.elasticsearch.core.bulk.BulkResponseItem; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.beans.factory.FactoryBean; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.aggregator.AbstractAggregatingMessageGroupProcessor; -import org.springframework.integration.aggregator.MessageCountReleaseStrategy; -import org.springframework.integration.config.AggregatorFactoryBean; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.dsl.IntegrationFlowBuilder; -import org.springframework.integration.expression.ValueExpression; -import org.springframework.integration.store.MessageGroup; -import org.springframework.integration.store.MessageGroupStore; -import org.springframework.integration.store.SimpleMessageStore; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHandler; -import org.springframework.util.Assert; -import org.springframework.util.StringUtils; - -/** - * @author Soby Chacko - * @author Andrea Montemaggio - * @author Corneil du Plessis - */ -@Configuration -@EnableConfigurationProperties(ElasticsearchConsumerProperties.class) -public class ElasticsearchConsumerConfiguration { - - /** - * Message header for the Index id. - */ - public static final String INDEX_ID_HEADER = "INDEX_ID"; - - /** - * Message header for the Index name. - */ - public static final String INDEX_NAME_HEADER = "INDEX_NAME"; - - private static final Log logger = LogFactory.getLog(ElasticsearchConsumerConfiguration.class); - - @Bean - FactoryBean aggregator(MessageGroupStore messageGroupStore, ElasticsearchConsumerProperties consumerProperties) { - AggregatorFactoryBean aggregatorFactoryBean = new AggregatorFactoryBean(); - aggregatorFactoryBean.setCorrelationStrategy(message -> ""); - aggregatorFactoryBean.setReleaseStrategy(new MessageCountReleaseStrategy(consumerProperties.getBatchSize())); - if (consumerProperties.getGroupTimeout() >= 0) { - aggregatorFactoryBean.setGroupTimeoutExpression(new ValueExpression<>(consumerProperties.getGroupTimeout())); - } - aggregatorFactoryBean.setMessageStore(messageGroupStore); - - // Currently, there is no way to customize the splitting behavior of an aggregator receiving - // a Collection> from the configured MessageGroupProcessor. - // Thus, fooling the aggregator with a wrapper of Message is just a straightforward way to preserve the - // individual message headers and release an entire batch to downstream indexing handler. - aggregatorFactoryBean.setProcessorBean(new AbstractAggregatingMessageGroupProcessor() { - @Override - protected Object aggregatePayloads(MessageGroup group, Map defaultHeaders) { - Collection> messages = group.getMessages(); - Assert.notEmpty(messages, this.getClass().getSimpleName() + " cannot process empty message groups"); - List payloads = new ArrayList(messages.size()); - for (Message message : messages) { - payloads.add(new MessageWrapper(message)); - } - return payloads; - } - }); - aggregatorFactoryBean.setExpireGroupsUponCompletion(true); - aggregatorFactoryBean.setSendPartialResultOnExpiry(true); - - return aggregatorFactoryBean; - } - - @Bean - MessageGroupStore messageGroupStore() { - SimpleMessageStore messageGroupStore = new SimpleMessageStore(); - messageGroupStore.setTimeoutOnIdle(true); - messageGroupStore.setCopyOnGet(false); - return messageGroupStore; - } - - @Bean - IntegrationFlow elasticsearchConsumerFlow( - @Qualifier("aggregator") MessageHandler aggregator, - ElasticsearchConsumerProperties properties, - @Qualifier("indexingHandler") MessageHandler indexingHandler - ) { - - final IntegrationFlowBuilder builder = - IntegrationFlow.from(MessageConsumer.class, gateway -> gateway.beanName("elasticsearchConsumer")); - if (properties.getBatchSize() > 1) { - builder.handle(aggregator); - } - return builder.handle(indexingHandler).get(); - } - - @Bean - public MessageHandler indexingHandler( - ElasticsearchClient elasticsearchClient, - ElasticsearchConsumerProperties consumerProperties - ) { - return message -> { - if (message.getPayload() instanceof Iterable) { - BulkRequest.Builder builder = new BulkRequest.Builder(); - StreamSupport.stream(((Iterable) message.getPayload()).spliterator(), false) - .filter(MessageWrapper.class::isInstance) - .map(itemPayload -> ((MessageWrapper) itemPayload).getMessage()) - .map(m -> buildIndexRequest(m, consumerProperties)) - .forEach(indexRequest -> - builder.operations(builder1 -> builder1.index(idx -> idx.index(indexRequest.index()).id(indexRequest.id()).document(indexRequest.document()))) - ); - - index(elasticsearchClient, builder.build(), consumerProperties.isAsync()); - } - else { - IndexRequest request = buildIndexRequest(message, consumerProperties); - index(elasticsearchClient, request, consumerProperties.isAsync()); - } - }; - } - - private IndexRequest buildIndexRequest(Message message, ElasticsearchConsumerProperties consumerProperties) { - IndexRequest.Builder requestBuilder = new IndexRequest.Builder(); - - String index = consumerProperties.getIndex(); - if (message.getHeaders().containsKey(INDEX_NAME_HEADER)) { - index = (String) message.getHeaders().get(INDEX_NAME_HEADER); - } - requestBuilder.index(index); - - String id = ""; - if (message.getHeaders().containsKey(INDEX_ID_HEADER)) { - id = (String) message.getHeaders().get(INDEX_ID_HEADER); - } - else if (consumerProperties.getId() != null) { - id = consumerProperties.getId().getValue(message, String.class); - } - requestBuilder.id(id); - - if (message.getPayload() instanceof String) { - requestBuilder.withJson(new StringReader((String) message.getPayload())); - } - else if (message.getPayload() instanceof Map) { - requestBuilder.document(message.getPayload()); - } - - if (StringUtils.hasText(consumerProperties.getRouting())) { - requestBuilder.routing(consumerProperties.getRouting()); - } - if (consumerProperties.getTimeoutSeconds() > 0) { - requestBuilder.timeout(new Time.Builder().time(consumerProperties.getTimeoutSeconds() + "s").build()); - } - - return requestBuilder.build(); - } - - private void index(ElasticsearchClient elasticsearchClient, BulkRequest request, boolean isAsync) { - if (isAsync) { - ElasticsearchAsyncClient elasticsearchAsyncClient = new ElasticsearchAsyncClient(elasticsearchClient._transport()); - CompletableFuture responseCompletableFuture = elasticsearchAsyncClient.bulk(request); - responseCompletableFuture.whenComplete((bulkResponse, x) -> { - if (x != null) { - throw new IllegalStateException("Error occurred while performing bulk index operation: " + x.getMessage(), x); - } - else { - handleBulkResponse(bulkResponse); - } - }); - - } - else { - try { - BulkResponse bulkResponse = elasticsearchClient.bulk(request); - handleBulkResponse(bulkResponse); - } - catch (IOException e) { - throw new IllegalStateException("Error occurred while performing bulk index operation: " + e.getMessage(), e); - } - } - } - - private void index(ElasticsearchClient elasticsearchClient, IndexRequest request, boolean isAsync) { - if (isAsync) { - ElasticsearchAsyncClient elasticsearchAsyncClient = new ElasticsearchAsyncClient(elasticsearchClient._transport()); - CompletableFuture responseCompletableFuture = elasticsearchAsyncClient.index(request); - responseCompletableFuture.whenComplete((indexResponse, x) -> { - if (x != null) { - throw new IllegalStateException("Error occurred while indexing document: " + x.getMessage(), x); - } - else { - handleResponse(indexResponse); - } - }); - } - else { - try { - IndexResponse response = elasticsearchClient.index(request); - handleResponse(response); - } - catch (IOException e) { - throw new IllegalStateException("Error occurred while indexing document: " + e.getMessage(), e); - } - } - } - - private void handleBulkResponse(BulkResponse response) { - if (logger.isDebugEnabled() || response.errors()) { - for (BulkResponseItem itemResponse : response.items()) { - if (itemResponse.error() != null) { - if (logger.isDebugEnabled()) { - logger.debug("itemResponse.error=" + itemResponse.error()); - } - logger.error(String.format("Index operation [id=%s, index=%s] failed: %s", - itemResponse.id(), itemResponse.index(), itemResponse.error().toString()) - ); - } - else { - var r = itemResponse.get(); - if (r != null) { - if (logger.isDebugEnabled()) { - logger.debug("itemResponse:" + r); - } - logger.debug(String.format("Index operation [id=%s, index=%s] succeeded: document [id=%s, version=%s] was written on shard %s.", - itemResponse.id(), itemResponse.index(), r.source().get("id"), r.source().get("version"), r.source().get("shardId")) - ); - } - else { - logger.debug(String.format("Index operation [id=%s, index=%s] succeeded", itemResponse.id(), itemResponse.index())); - } - } - } - } - - if (response.errors()) { - String error = response.items() - .stream() - .map(bulkResponseItem -> bulkResponseItem.error() != null ? bulkResponseItem.error().toString() : "") - .reduce((errorCause, errorCause2) -> errorCause != null ? errorCause + " : " + errorCause2 : errorCause2) - .orElseGet(response::toString); - throw new IllegalStateException("Bulk indexing operation completed with failures: " + error); - } - } - - private void handleResponse(IndexResponse response) { - logger.debug(String.format("Index operation [index=%s] succeeded: document [id=%s, version=%d] was written on shard %s.", - response.index(), response.id(), response.version(), response.shards().toString()) - ); - } - - static class MessageWrapper { - private final Message message; - - MessageWrapper(Message message) { - this.message = message; - } - - public Message getMessage() { - return message; - } - } - - private interface MessageConsumer extends Consumer> { - - } - -} diff --git a/functions/consumer/elasticsearch-consumer/src/main/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerProperties.java b/functions/consumer/elasticsearch-consumer/src/main/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerProperties.java deleted file mode 100644 index a6d7badcf..000000000 --- a/functions/consumer/elasticsearch-consumer/src/main/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerProperties.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2020-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.elasticsearch; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; - -/** - * @author Soby Chacko - * @author Andrea Montemaggio - */ -@ConfigurationProperties("elasticsearch.consumer") -public class ElasticsearchConsumerProperties { - - /** - * The id of the document to index. - * If set, the INDEX_ID header value overrides this property on a per message basis. - */ - Expression id; - - /** - * Name of the index. - * If set, the INDEX_NAME header value overrides this property on a per message basis. - */ - String index; - - /** - * Indicates the shard to route to. - * If not provided, Elasticsearch will default to a hash of the document id. - */ - String routing; - - /** - * Timeout for the shard to be available. - * If not set, it defaults to 1 minute set by the Elasticsearch client. - */ - long timeoutSeconds; - - /** - * Indicates whether the indexing operation is async or not. - * By default indexing is done synchronously. - */ - boolean async; - - /** - * Number of items to index for each request. It defaults to 1. - * For values greater than 1 bulk indexing API will be used. - */ - int batchSize = 1; - - /** - * Timeout in milliseconds after which message group is flushed when bulk indexing is active. - * It defaults to -1, meaning no automatic flush of idle message groups occurs. - */ - long groupTimeout = -1L; - - public Expression getId() { - return id; - } - - public void setId(Expression id) { - this.id = id; - } - - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - - public String getRouting() { - return routing; - } - - public void setRouting(String routing) { - this.routing = routing; - } - - public long getTimeoutSeconds() { - return timeoutSeconds; - } - - public void setTimeoutSeconds(long timeoutSeconds) { - this.timeoutSeconds = timeoutSeconds; - } - - public boolean isAsync() { - return async; - } - - public void setAsync(boolean async) { - this.async = async; - } - - public int getBatchSize() { - return batchSize; - } - - public void setBatchSize(int batchSize) { - this.batchSize = batchSize; - } - - public long getGroupTimeout() { - return groupTimeout; - } - - public void setGroupTimeout(long groupTimeout) { - this.groupTimeout = groupTimeout; - } -} diff --git a/functions/consumer/elasticsearch-consumer/src/test/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerApplicationTests.java b/functions/consumer/elasticsearch-consumer/src/test/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerApplicationTests.java deleted file mode 100644 index 1ca21ad7c..000000000 --- a/functions/consumer/elasticsearch-consumer/src/test/java/org/springframework/cloud/fn/consumer/elasticsearch/ElasticsearchConsumerApplicationTests.java +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Copyright 2020-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.elasticsearch; - -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import java.util.UUID; -import java.util.function.Consumer; - -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.elasticsearch._types.ElasticsearchException; -import co.elastic.clients.elasticsearch.core.GetRequest; -import co.elastic.clients.elasticsearch.core.GetResponse; -import co.elastic.clients.json.JsonData; -import org.awaitility.Awaitility; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.utility.DockerImageName; - -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.cloud.fn.common.config.SpelExpressionConverterConfiguration; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.elasticsearch.client.ClientConfiguration; -import org.springframework.data.elasticsearch.client.elc.ElasticsearchConfiguration; -import org.springframework.lang.NonNull; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; -import static org.assertj.core.api.Assertions.assertThatIllegalStateException; - -// import static org.elasticsearch.core.Strings.*; - -/** - * @author Soby Chacko - * @author Andrea Montemaggio - */ -@Tag("integration") -@Testcontainers(disabledWithoutDocker = true) -public class ElasticsearchConsumerApplicationTests { - private static final Logger log = LoggerFactory.getLogger(ElasticsearchConsumerApplicationTests.class); - - @Container - static final ElasticsearchContainer elasticsearch = new ElasticsearchContainer( - DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch") - .withTag("7.17.7") - ).withStartupTimeout(Duration.ofSeconds(120)) - .withStartupAttempts(3); - - private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() - .withUserConfiguration(ElasticsearchConsumerTestApplication.class, SpelExpressionConverterConfiguration.class); - - @Test - public void testBasicJsonString() { - this.contextRunner - .withPropertyValues("elasticsearch.consumer.index=foo", "elasticsearch.consumer.id=1", - "spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress()) - .run(context -> { - final Consumer> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class); - - final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564," - + "\"fullName\":\"John Doe\"}"; - final Message message = MessageBuilder.withPayload(jsonObject).build(); - - elasticsearchConsumer.accept(message); - final ElasticsearchClient elasticsearchClient = context.getBean(ElasticsearchClient.class); - final GetRequest getRequest = new GetRequest.Builder().index("foo").id("1").build(); - final GetResponse response = elasticsearchClient.get(getRequest, JsonData.class); - - assertThat(response.found()).isTrue(); - assertThat(response.source()).isNotNull(); - assertThat(response.source().toJson()).isEqualTo(JsonData.fromJson(jsonObject).toJson()); - }); - } - - @Test - public void testIdPassedAsMessageHeader() { - this.contextRunner - .withPropertyValues("elasticsearch.consumer.index=foo", - "spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress()) - .run(context -> { - final Consumer> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class); - - final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564," - + "\"fullName\":\"John Doe\"}"; - final Message message = MessageBuilder.withPayload(jsonObject) - .setHeader(ElasticsearchConsumerConfiguration.INDEX_ID_HEADER, "2").build(); - log.info("elasticsearchConsumer.accept:{}", message); - elasticsearchConsumer.accept(message); - - final ElasticsearchClient elasticsearchClient = context.getBean(ElasticsearchClient.class); - final GetRequest getRequest = new GetRequest.Builder().index("foo").id("2").build(); - final GetResponse response = elasticsearchClient.get(getRequest, JsonData.class); - assertThat(response.found()).isTrue(); - assertThat(response.source()).isNotNull(); - assertThat(response.source().toJson()).isEqualTo(JsonData.fromJson(jsonObject).toJson()); - assertThat(response.id()).isEqualTo("2"); - }); - } - - @Test - public void testJsonAsMap() { - this.contextRunner - .withPropertyValues("elasticsearch.consumer.index=foo", "elasticsearch.consumer.id=3", - "spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress()) - .run(context -> { - final Consumer> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class); - - final Map jsonMap = new HashMap<>(); - jsonMap.put("age", 10); - jsonMap.put("dateOfBirth", 1471466076564L); - jsonMap.put("fullName", "John Doe"); - final Message> message = MessageBuilder.withPayload(jsonMap).build(); - log.info("elasticsearchConsumer.accept:{}", message); - elasticsearchConsumer.accept(message); - final ElasticsearchClient elasticsearchClient = context.getBean(ElasticsearchClient.class); - final GetRequest getRequest = new GetRequest.Builder().index("foo").id("3").build(); - final GetResponse response = elasticsearchClient.get(getRequest, HashMap.class); - - assertThat(response.found()).isTrue(); - HashMap map = response.source(); - - jsonMap.entrySet().forEach(entry -> { - Object value = map.get(entry.getKey()); - assertThat(value).isNotNull(); - assertThat(value).isEqualTo(entry.getValue()); - }); - assertThat(response.id()).isEqualTo("3"); - }); - } - - @Test - public void testAsyncIndexing() { - this.contextRunner - .withPropertyValues("elasticsearch.consumer.index=foo", "elasticsearch.consumer.async=true", - "elasticsearch.consumer.id=5", - "spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress()) - .run(context -> { - final Consumer> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class); - - final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564," - + "\"fullName\":\"John Doe\"}"; - final Message message = MessageBuilder.withPayload(jsonObject).build(); - log.info("elasticsearchConsumer.accept:{}", message); - elasticsearchConsumer.accept(message); - - final ElasticsearchClient elasticsearchClient = context.getBean(ElasticsearchClient.class); - final GetRequest getRequest = new GetRequest.Builder().index("foo").id("5").build(); - - Awaitility.given() - .ignoreException(ElasticsearchException.class) - .await() - .until(() -> elasticsearchClient.get(getRequest, JsonData.class).found()); - }); - } - - @Test - public void testBulkIndexingWithIdFromHeader() { - this.contextRunner - .withPropertyValues("elasticsearch.consumer.index=foo_" + UUID.randomUUID(), "elasticsearch.consumer.batch-size=10", - "spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress()) - .run(context -> { - final Consumer> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class); - final ElasticsearchConsumerProperties properties = context.getBean(ElasticsearchConsumerProperties.class); - final ElasticsearchClient elasticsearchClient = context.getBean(ElasticsearchClient.class); - - - for (int i = 0; i < properties.getBatchSize(); i++) { - final GetRequest getRequest = new GetRequest.Builder().index(properties.getIndex()).id(Integer.toString(i)).build(); - assertThatExceptionOfType(ElasticsearchException.class) - .isThrownBy(() -> elasticsearchClient.get(getRequest, JsonData.class)) - .withFailMessage("Expected index not found exception for message %d") - .withMessageContaining("index_not_found_exception"); - - final Message message = MessageBuilder - .withPayload("{\"seq\":" + i + ",\"age\":10,\"dateOfBirth\":1471466076564," - + "\"fullName\":\"John Doe\"}") - .setHeader(ElasticsearchConsumerConfiguration.INDEX_ID_HEADER, Integer.toString(i)) - .build(); - log.info("elasticsearchConsumer.accept:{}", message); - elasticsearchConsumer.accept(message); - } - - for (int i = 0; i < properties.getBatchSize(); i++) { - final GetRequest getRequest = new GetRequest.Builder().index(properties.getIndex()).id(Integer.toString(i)).build(); - final GetResponse response = elasticsearchClient.get(getRequest, JsonData.class); - - assertThat(response.found()) - .withFailMessage("Document with id=%d cannot be found.", i) - .isTrue(); - assertThat(response.source().toJson().asJsonObject().get("seq").toString()).isEqualTo(Integer.toString(i)); - } - }); - } - - @Test - public void testBulkIndexingItemFailure() { - this.contextRunner - .withPropertyValues("elasticsearch.consumer.index=foo_" + UUID.randomUUID(), "elasticsearch.consumer.batch-size=10", - "spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress()) - .run(context -> { - final Consumer> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class); - final ElasticsearchConsumerProperties properties = context.getBean(ElasticsearchConsumerProperties.class); - final ElasticsearchClient elasticsearchClient = context.getBean(ElasticsearchClient.class); - - - for (int i = 0; i < properties.getBatchSize(); i++) { - final GetRequest getRequest = new GetRequest.Builder().index(properties.getIndex()).id(Integer.toString(i)).build(); - assertThatExceptionOfType(ElasticsearchException.class) - .isThrownBy(() -> elasticsearchClient.get(getRequest, JsonData.class)) - .withFailMessage("Expected index not found exception for message %d") - .withMessageContaining("index_not_found_exception"); - - MessageBuilder builder = MessageBuilder - .withPayload("{\"seq\":" + i + ",\"age\":10,\"dateOfBirth\":1471466076564," - + "\"fullName\":\"John Doe\"}") - .setHeader(ElasticsearchConsumerConfiguration.INDEX_ID_HEADER, Integer.toString(i)); - - if (i == 0) { - // set an invalid index name to make the first request fail - builder.setHeader(ElasticsearchConsumerConfiguration.INDEX_NAME_HEADER, "_" + properties.getIndex()); - } - - final Message message = builder.build(); - - if (i < properties.getBatchSize() - 1) { - log.info("elasticsearchConsumer.accept:{}", message); - elasticsearchConsumer.accept(message); - } - else { - // last invocation - assertThatIllegalStateException() - .isThrownBy(() -> elasticsearchConsumer.accept(message)) - .withMessageContaining("Bulk indexing operation completed with failures"); - } - } - }); - } - - @Test - public void testIndexFromMessageHeader() { - this.contextRunner - .withPropertyValues("elasticsearch.consumer.index=foo", - "spring.elasticsearch.rest.uris=http://" + elasticsearch.getHttpHostAddress()) - .run(context -> { - final Consumer> elasticsearchConsumer = context.getBean("elasticsearchConsumer", Consumer.class); - final ElasticsearchConsumerProperties properties = context.getBean(ElasticsearchConsumerProperties.class); - - final String dynamicIndex = properties.getIndex() + "-2"; - - final String jsonObject = "{\"age\":10,\"dateOfBirth\":1471466076564," - + "\"fullName\":\"John Doe\"}"; - final Message message = MessageBuilder.withPayload(jsonObject) - .setHeader(ElasticsearchConsumerConfiguration.INDEX_ID_HEADER, "2") - .setHeader(ElasticsearchConsumerConfiguration.INDEX_NAME_HEADER, dynamicIndex) - .build(); - log.info("elasticsearchConsumer.accept:{}", message); - elasticsearchConsumer.accept(message); - final ElasticsearchClient elasticsearchClient = context.getBean(ElasticsearchClient.class); - - GetRequest getRequest = new GetRequest.Builder().index(dynamicIndex).id("2").build(); - final GetResponse response = elasticsearchClient.get(getRequest, JsonData.class); - assertThat(response.found()).isTrue(); - assertThat(response.source()).isNotNull(); - assertThat(response.source().toJson()).isEqualTo(JsonData.fromJson(jsonObject).toJson()); - assertThat(response.id()).isEqualTo("2"); - }); - } - - @SpringBootApplication - static class ElasticsearchConsumerTestApplication { - } - - @Configuration - static class Config extends ElasticsearchConfiguration { - @NonNull - @Override - public ClientConfiguration clientConfiguration() { - return ClientConfiguration.builder() - .connectedTo(elasticsearch.getHttpHostAddress()) - .build(); - } - } -} diff --git a/functions/consumer/elasticsearch-consumer/src/test/resources/logback.xml b/functions/consumer/elasticsearch-consumer/src/test/resources/logback.xml deleted file mode 100644 index ae8b3d29e..000000000 --- a/functions/consumer/elasticsearch-consumer/src/test/resources/logback.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - diff --git a/functions/consumer/file-consumer/README.adoc b/functions/consumer/file-consumer/README.adoc deleted file mode 100644 index f85a64039..000000000 --- a/functions/consumer/file-consumer/README.adoc +++ /dev/null @@ -1,28 +0,0 @@ -# File Consumer - -A consumer that allows you to write incoming messages into files. -The consumer uses the `FileWritingMessageHandler` from Spring Integration. - -## Beans for injection - -You can import `FileConsumerConfiguration` in the application and then inject the following bean. - -`Consumer> fileConsumer` - -You can use `fileConsumer` as a qualifier when injecting. - -## Configuration Options - -All configuration properties are prefixed with `file.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerProperties.java[FileConsumerProperties]. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `FileWritingMessageHandler` configuration used by the `fileConsumer`. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/file[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/file-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a File sink. diff --git a/functions/consumer/file-consumer/pom.xml b/functions/consumer/file-consumer/pom.xml deleted file mode 100644 index 256b64f71..000000000 --- a/functions/consumer/file-consumer/pom.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - file-consumer - file-consumer - file consumer - - - - org.springframework.integration - spring-integration-file - - - - diff --git a/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerConfiguration.java b/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerConfiguration.java deleted file mode 100644 index 784418f36..000000000 --- a/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerConfiguration.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.file; - -import java.util.function.Consumer; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.expression.ExpressionParser; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.integration.file.DefaultFileNameGenerator; -import org.springframework.integration.file.FileNameGenerator; -import org.springframework.integration.file.FileWritingMessageHandler; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; - -/** - * @author Mark Fisher - * @author Artem Bilan - * @author Soby Chacko - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties(FileConsumerProperties.class) -public class FileConsumerConfiguration { - - private static final ExpressionParser EXPRESSION_PARSER = new SpelExpressionParser(); - - private final FileConsumerProperties properties; - - public FileConsumerConfiguration(FileConsumerProperties properties) { - this.properties = properties; - } - - @Bean - public Consumer> fileConsumer(FileWritingMessageHandler fileWritingMessageHandler) { - return fileWritingMessageHandler::handleMessage; - } - - @Bean - public FileWritingMessageHandler fileWritingMessageHandler(FileNameGenerator fileNameGenerator, - @Nullable ComponentCustomizer fileWritingMessageHandlerCustomizer) { - - FileWritingMessageHandler handler = - this.properties.getDirectoryExpression() != null - ? new FileWritingMessageHandler( - EXPRESSION_PARSER.parseExpression(this.properties.getDirectoryExpression())) - : new FileWritingMessageHandler(this.properties.getDirectory()); - handler.setAutoCreateDirectory(true); - handler.setAppendNewLine(!properties.isBinary()); - handler.setCharset(properties.getCharset()); - handler.setExpectReply(false); - handler.setFileExistsMode(properties.getMode()); - handler.setFileNameGenerator(fileNameGenerator); - - if (fileWritingMessageHandlerCustomizer != null) { - fileWritingMessageHandlerCustomizer.customize(handler); - } - return handler; - } - - @Bean - public FileNameGenerator fileNameGenerator() { - DefaultFileNameGenerator fileNameGenerator = new DefaultFileNameGenerator(); - fileNameGenerator.setExpression(properties.getNameExpression()); - return fileNameGenerator; - } - -} diff --git a/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerProperties.java b/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerProperties.java deleted file mode 100644 index 30ce5886b..000000000 --- a/functions/consumer/file-consumer/src/main/java/org/springframework/cloud/fn/consumer/file/FileConsumerProperties.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.file; - -import java.io.File; - -import jakarta.validation.constraints.AssertTrue; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.integration.file.support.FileExistsMode; -import org.springframework.util.StringUtils; -import org.springframework.validation.annotation.Validated; - -/** - * Properties for the file sink. - * - * @author Mark Fisher - * @author Gary Russell - */ -@ConfigurationProperties("file.consumer") -@Validated -public class FileConsumerProperties { - - static final String DEFAULT_DIR = System.getProperty("java.io.tmpdir") + "file-consumer"; - - private static final String DEFAULT_NAME = "file-consumer"; - - /** - * A flag to indicate whether adding a newline after the write should be suppressed. - */ - private boolean binary = false; - - /** - * The charset to use when writing text content. - */ - private String charset = "UTF-8"; - - /** - * The parent directory of the target file. - */ - private File directory = new File(DEFAULT_DIR); - - /** - * The expression to evaluate for the parent directory of the target file. - */ - private String directoryExpression; - - /** - * The FileExistsMode to use if the target file already exists. - */ - private FileExistsMode mode = FileExistsMode.APPEND; - - /** - * The name of the target file. - */ - private String name = DEFAULT_NAME; - - /** - * The expression to evaluate for the name of the target file. - */ - private String nameExpression; - - /** - * The suffix to append to file name. - */ - private String suffix = ""; - - public boolean isBinary() { - return binary; - } - - public void setBinary(boolean binary) { - this.binary = binary; - } - - public String getCharset() { - return charset; - } - - public void setCharset(String charset) { - this.charset = charset; - } - - public File getDirectory() { - return directory; - } - - public void setDirectory(File directory) { - this.directory = directory; - } - - public String getDirectoryExpression() { - return directoryExpression; - } - - public void setDirectoryExpression(String directoryExpression) { - this.directoryExpression = directoryExpression; - } - - public FileExistsMode getMode() { - return mode; - } - - public void setMode(FileExistsMode mode) { - this.mode = mode; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getNameExpression() { - return (nameExpression != null) - ? nameExpression + " + '" + getSuffix() + "'" - : "'" + name + getSuffix() + "'"; - } - - public void setNameExpression(String nameExpression) { - this.nameExpression = nameExpression; - } - - public String getSuffix() { - String suffixWithDotIfNecessary = ""; - if (StringUtils.hasText(suffix)) { - suffixWithDotIfNecessary = suffix.startsWith(".") ? suffix : "." + suffix; - } - return suffixWithDotIfNecessary; - } - - public void setSuffix(String suffix) { - this.suffix = suffix; - } - - @AssertTrue(message = "Exactly one of 'name' or 'nameExpression' must be set") - public boolean isMutuallyExclusiveNameAndNameExpression() { - return DEFAULT_NAME.equals(name) || nameExpression == null; - } - - @AssertTrue(message = "Exactly one of 'directory' or 'directoryExpression' must be set") - public boolean isMutuallyExclusiveDirectoryAndDirectoryExpression() { - return new File(DEFAULT_DIR).equals(directory) || directoryExpression == null; - } - -} diff --git a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/AbstractFileConsumerTests.java b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/AbstractFileConsumerTests.java deleted file mode 100644 index 0a26929dc..000000000 --- a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/AbstractFileConsumerTests.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.file; - -import java.nio.file.Path; -import java.util.function.Consumer; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.io.TempDir; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; - -/** - * @author Soby Chacko - */ -@SpringBootTest -@DirtiesContext -public class AbstractFileConsumerTests { - - @TempDir - static Path tempDir; - - @Autowired - Consumer> fileConsumer; - - @BeforeAll - public static void beforeAll() { - System.setProperty("file.consumer.directory", tempDir.toAbsolutePath().toString()); - } - - @AfterAll - public static void afterAll() { - System.clearProperty("file.consumer.directory"); - } - - @SpringBootApplication - static class FileConsumerTestApplication { - } -} diff --git a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/BinaryFileTests.java b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/BinaryFileTests.java deleted file mode 100644 index ef6b2bdca..000000000 --- a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/BinaryFileTests.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.file; - -import java.io.File; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.test.context.TestPropertySource; -import org.springframework.util.FileCopyUtils; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Mark Fisher - * @author Artem Bilan - * @author Soby Chacko - */ -@TestPropertySource(properties = "file.consumer.binary = true") -public class BinaryFileTests extends AbstractFileConsumerTests { - - @Test - public void test() throws Exception { - fileConsumer.accept(MessageBuilder.withPayload("hello file-consumer".getBytes()).build()); - File file = new File(tempDir.toFile(), "file-consumer"); - assertThat(file.exists()).isTrue(); - byte[] results = FileCopyUtils.copyToByteArray(file); - assertThat("hello file-consumer".getBytes()).isEqualTo(results); - } - -} diff --git a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/ExpressionTests.java b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/ExpressionTests.java deleted file mode 100644 index 91138ff5b..000000000 --- a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/ExpressionTests.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.file; - -import java.io.File; -import java.io.FileReader; -import java.nio.file.Path; -import java.util.function.Consumer; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.util.FileCopyUtils; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Mark Fisher - * @author Artem Bilan - * @author Soby Chacko - *

- * We don't need a separate SpringBootApplication for this test as there is already one available in this package. - * {@link AbstractFileConsumerTests}. - */ -@SpringBootTest(properties = {"file.consumer.nameExpression = payload.substring(0, 4)", - "file.consumer.directoryExpression = '${java.io.tmpdir}'+'/'+headers.dir", - "file.consumer.suffix=out"}) -@DirtiesContext -public class ExpressionTests { - - @TempDir - static Path tempDir; - - @Autowired - Consumer> fileConsumer; - - @Test - public void test() throws Exception { - fileConsumer.accept(MessageBuilder.withPayload("this is something").setHeader("dir", "expression").build()); - File file = new File(System.getProperty("java.io.tmpdir") + File.separator + "expression", "this.out"); - file.deleteOnExit(); - assertThat(file.exists()).isTrue(); - assertThat("this is something" + System.lineSeparator()) - .isEqualTo(FileCopyUtils.copyToString(new FileReader(file))); - } -} diff --git a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/TextFileTests.java b/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/TextFileTests.java deleted file mode 100644 index 60a9b218d..000000000 --- a/functions/consumer/file-consumer/src/test/java/org/springframework/cloud/fn/consumer/file/TextFileTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.file; - -import java.io.File; -import java.io.FileReader; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.test.context.TestPropertySource; -import org.springframework.util.FileCopyUtils; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Mark Fisher - * @author Artem Bilan - * @author Soby Chacko - */ -@TestPropertySource(properties = {"file.consumer.name = test", "file.consumer.suffix=txt"}) -public class TextFileTests extends AbstractFileConsumerTests { - - @Test - public void test() throws Exception { - fileConsumer.accept(MessageBuilder.withPayload("hello file-consumer").build()); - File file = new File(tempDir.toFile(), "test.txt"); - assertThat(file.exists()).isTrue(); - assertThat("hello file-consumer" + System.lineSeparator()) - .isEqualTo(FileCopyUtils.copyToString(new FileReader(file))); - } - -} diff --git a/functions/consumer/ftp-consumer/README.adoc b/functions/consumer/ftp-consumer/README.adoc deleted file mode 100644 index baa6efd51..000000000 --- a/functions/consumer/ftp-consumer/README.adoc +++ /dev/null @@ -1,28 +0,0 @@ -# Ftp Consumer - -A consumer that allows you to ftp files. -The consumer uses the `FtpMessageHandler` from Spring Integration. - -## Beans for injection - -You can import `FtpConsumerConfiguration` in the application and then inject the following bean. - -`Consumer> ftpConsumer` - -You can use `ftpConsumer` as a qualifier when injecting. - -## Configuration Options - -All configuration properties are prefixed with `ftp.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerProperties.java[FtpConsumerProperties]. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `FtpMessageHandlerSpec` configuration used by the `ftpConsumer`. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/ftp[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/ftp-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a Ftp sink. diff --git a/functions/consumer/ftp-consumer/pom.xml b/functions/consumer/ftp-consumer/pom.xml deleted file mode 100644 index 084393b33..000000000 --- a/functions/consumer/ftp-consumer/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - ftp-consumer - ftp-consumer - file consumer - - - - org.springframework.integration - spring-integration-ftp - - - org.springframework.cloud.fn - ftp-common - ${project.version} - - - org.springframework.cloud.fn - function-test-support - ${project.version} - test - - - - diff --git a/functions/consumer/ftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerConfiguration.java b/functions/consumer/ftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerConfiguration.java deleted file mode 100644 index 2aba1b60f..000000000 --- a/functions/consumer/ftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerConfiguration.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2015-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.ftp; - -import java.util.function.Consumer; - -import org.apache.commons.net.ftp.FTPFile; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.cloud.fn.common.ftp.FtpSessionFactoryConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.expression.ExpressionParser; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.dsl.IntegrationFlowBuilder; -import org.springframework.integration.file.remote.session.SessionFactory; -import org.springframework.integration.ftp.dsl.Ftp; -import org.springframework.integration.ftp.dsl.FtpMessageHandlerSpec; -import org.springframework.integration.ftp.session.FtpRemoteFileTemplate; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; - -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties(FtpConsumerProperties.class) -@Import(FtpSessionFactoryConfiguration.class) -public class FtpConsumerConfiguration { - - private static final ExpressionParser EXPRESSION_PARSER = new SpelExpressionParser(); - - @Autowired - FtpConsumerProperties ftpConsumerProperties; - - @Bean - public IntegrationFlow ftpInboundFlow(FtpConsumerProperties properties, SessionFactory ftpSessionFactory, - @Nullable ComponentCustomizer ftpMessageHandlerSpecCustomizer) { - - IntegrationFlowBuilder integrationFlowBuilder = - IntegrationFlow.from(MessageConsumer.class, (gateway) -> gateway.beanName("ftpConsumer")); - - FtpMessageHandlerSpec handlerSpec = - Ftp.outboundAdapter(new FtpRemoteFileTemplate(ftpSessionFactory), properties.getMode()) - .remoteDirectory(properties.getRemoteDir()) - .remoteFileSeparator(properties.getRemoteFileSeparator()) - .autoCreateDirectory(properties.isAutoCreateDir()) - .temporaryFileSuffix(properties.getTmpFileSuffix()); - if (properties.getFilenameExpression() != null) { - handlerSpec.fileNameExpression( - EXPRESSION_PARSER.parseExpression(properties.getFilenameExpression()).getExpressionString()); - } - - if (ftpMessageHandlerSpecCustomizer != null) { - ftpMessageHandlerSpecCustomizer.customize(handlerSpec); - } - - return integrationFlowBuilder - .handle(handlerSpec) - .get(); - } - - private interface MessageConsumer extends Consumer> { - - } - -} diff --git a/functions/consumer/ftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerProperties.java b/functions/consumer/ftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerProperties.java deleted file mode 100644 index b99ff3eb7..000000000 --- a/functions/consumer/ftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerProperties.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.ftp; - -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.integration.file.support.FileExistsMode; -import org.springframework.validation.annotation.Validated; - -@ConfigurationProperties("ftp.consumer") -@Validated -public class FtpConsumerProperties { - - /** - * The remote FTP directory. - */ - private String remoteDir = "/"; - - /** - * The suffix to use while the transfer is in progress. - */ - private String tmpFileSuffix = ".tmp"; - - /** - * The remote file separator. - */ - private String remoteFileSeparator = "/"; - - /** - * A temporary directory where the file will be written if '#isUseTemporaryFilename()' - * is true. - */ - private String temporaryRemoteDir = "/"; - - /** - * Whether or not to create the remote directory. - */ - private boolean autoCreateDir = true; - - /** - * Action to take if the remote file already exists. - */ - private FileExistsMode mode = FileExistsMode.REPLACE; - - /** - * Whether or not to write to a temporary file and rename. - */ - private boolean useTemporaryFilename = true; - - /** - * A SpEL expression to generate the remote file name. - */ - private String filenameExpression; - - @NotBlank - public String getTemporaryRemoteDir() { - return this.temporaryRemoteDir; - } - - public void setTemporaryRemoteDir(String temporaryRemoteDir) { - this.temporaryRemoteDir = temporaryRemoteDir; - } - - public boolean isAutoCreateDir() { - return this.autoCreateDir; - } - - public void setAutoCreateDir(boolean autoCreateDir) { - this.autoCreateDir = autoCreateDir; - } - - @NotNull - public FileExistsMode getMode() { - return this.mode; - } - - public void setMode(FileExistsMode mode) { - this.mode = mode; - } - - public boolean isUseTemporaryFilename() { - return this.useTemporaryFilename; - } - - public void setUseTemporaryFilename(boolean useTemporaryFilename) { - this.useTemporaryFilename = useTemporaryFilename; - } - - public String getFilenameExpression() { - return this.filenameExpression; - } - - public void setFilenameExpression(String filenameExpression) { - this.filenameExpression = filenameExpression; - } - - @NotBlank - public String getRemoteDir() { - return this.remoteDir; - } - - public final void setRemoteDir(String remoteDir) { - this.remoteDir = remoteDir; - } - - @NotBlank - public String getTmpFileSuffix() { - return this.tmpFileSuffix; - } - - public void setTmpFileSuffix(String tmpFileSuffix) { - this.tmpFileSuffix = tmpFileSuffix; - } - - @NotBlank - public String getRemoteFileSeparator() { - return this.remoteFileSeparator; - } - - public void setRemoteFileSeparator(String remoteFileSeparator) { - this.remoteFileSeparator = remoteFileSeparator; - } -} diff --git a/functions/consumer/ftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerPropertiesTests.java b/functions/consumer/ftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerPropertiesTests.java deleted file mode 100644 index 4d449a9cb..000000000 --- a/functions/consumer/ftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerPropertiesTests.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.ftp; - -import org.junit.jupiter.api.Test; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.boot.test.util.TestPropertyValues; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.file.support.FileExistsMode; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author David Turanski - * @author Gary Russell - * @author Artem Bilan - */ -public class FtpConsumerPropertiesTests { - - @Test - public void remoteDirCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.consumer.remoteDir:/remote") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class); - assertThat(properties.getRemoteDir()).isEqualTo("/remote"); - context.close(); - } - - @Test - public void autoCreateDirCanBeDisabled() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.consumer.autoCreateDir:false") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class); - assertThat(!properties.isAutoCreateDir()).isTrue(); - context.close(); - } - - @Test - public void tmpFileSuffixCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.consumer.tmpFileSuffix:.foo") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class); - assertThat(properties.getTmpFileSuffix()).isEqualTo(".foo"); - context.close(); - } - - @Test - public void tmpFileRemoteDirCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.consumer.temporaryRemoteDir:/foo") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class); - assertThat(properties.getTemporaryRemoteDir()).isEqualTo("/foo"); - context.close(); - } - - @Test - public void remoteFileSeparatorCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.consumer.remoteFileSeparator:\\") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class); - assertThat(properties.getRemoteFileSeparator()).isEqualTo("\\"); - context.close(); - } - - @Test - public void useTemporaryFileNameCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.consumer.useTemporaryFilename:false") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class); - assertThat(properties.isUseTemporaryFilename()).isFalse(); - context.close(); - } - - @Test - public void fileExistsModeCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.consumer.mode:FAIL") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpConsumerProperties properties = context.getBean(FtpConsumerProperties.class); - assertThat(properties.getMode()).isEqualTo(FileExistsMode.FAIL); - context.close(); - } - - @Configuration - @EnableConfigurationProperties(FtpConsumerProperties.class) - static class Conf { - - } -} diff --git a/functions/consumer/ftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerTests.java b/functions/consumer/ftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerTests.java deleted file mode 100644 index e7815d310..000000000 --- a/functions/consumer/ftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/ftp/FtpConsumerTests.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.ftp; - -import java.io.File; -import java.util.function.Consumer; - -import org.junit.jupiter.api.Test; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.test.support.ftp.FtpTestSupport; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - -@DirtiesContext -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { - "ftp.consumer.remoteDir = ftpTarget", - "ftp.factory.username = foo", - "ftp.factory.password = foo", - "ftp.consumer.mode = FAIL", - "ftp.consumer.filenameExpression = payload.name.toUpperCase()" - }) -public class FtpConsumerTests extends FtpTestSupport { - - @Autowired - Consumer> ftpConsumer; - - @Test - public void sendFiles() { - for (int i = 1; i <= 2; i++) { - String pathname = "/localSource" + i + ".txt"; - String upperPathname = pathname.toUpperCase(); - new File(getTargetRemoteDirectory() + upperPathname).delete(); - assertThat(new File(getTargetRemoteDirectory() + upperPathname).exists()).isFalse(); - ftpConsumer.accept(new GenericMessage<>(new File(getSourceLocalDirectory() + pathname))); - File expected = new File(getTargetRemoteDirectory() + upperPathname); - assertThat(expected.exists()).isTrue(); - // verify the uppercase on a case-insensitive file system - File[] files = getTargetRemoteDirectory().listFiles(); - for (File file : files) { - assertThat(file.getName().startsWith("LOCALSOURCE")).isTrue(); - } - } - } - - @Test - public void serverRefreshed() { // noop test to test the dirs are refreshed properly - String pathname = "/LOCALSOURCE1.TXT"; - assertThat(getTargetRemoteDirectory().exists()).isTrue(); - assertThat(new File(getTargetRemoteDirectory() + pathname).exists()).isFalse(); - } - - @SpringBootApplication - static class FtpConsumerTestApplication { - } -} diff --git a/functions/consumer/jdbc-consumer/README.adoc b/functions/consumer/jdbc-consumer/README.adoc deleted file mode 100644 index 14a3decf4..000000000 --- a/functions/consumer/jdbc-consumer/README.adoc +++ /dev/null @@ -1,26 +0,0 @@ -# Jdbc Consumer - -A consumer that allows you to insert records into a relational database. -The consumer uses the `JdbcMessageHandler` from Spring Integration. - -## Beans for injection - -You can import `JdbcConsumerConfiguration` in the application and then inject the following bean. - -`Consumer> jdbcConsumer` - -You can use `jdbcConsumer` as a qualifier when injecting. - -## Configuration Options - -All configuration properties are prefixed with `jdbc.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerProperties.java[JdbcConsumerProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/jdbc[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/jdbc-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a Jdbc sink. \ No newline at end of file diff --git a/functions/consumer/jdbc-consumer/pom.xml b/functions/consumer/jdbc-consumer/pom.xml deleted file mode 100644 index 837712cde..000000000 --- a/functions/consumer/jdbc-consumer/pom.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - jdbc-consumer - jdbc-consumer - jdbc consumer - - - - org.springframework.integration - spring-integration-jdbc - - - org.springframework.boot - spring-boot-starter-jdbc - - - org.hsqldb - hsqldb - runtime - - - com.h2database - h2 - runtime - - - org.mariadb.jdbc - mariadb-java-client - runtime - - - org.postgresql - postgresql - runtime - - - com.microsoft.sqlserver - mssql-jdbc - runtime - - - com.microsoft.azure - azure-keyvault - - - - - - diff --git a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/DefaultInitializationScriptResource.java b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/DefaultInitializationScriptResource.java deleted file mode 100644 index df838db37..000000000 --- a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/DefaultInitializationScriptResource.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import java.nio.charset.StandardCharsets; -import java.util.Collection; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.core.io.ByteArrayResource; - -/** - * An in-memory script crafted for dropping-creating the table we're working with. - * All columns are created as VARCHAR(2000). - * - * @author Eric Bottard - * @author Thomas Risberg - */ -public class DefaultInitializationScriptResource extends ByteArrayResource { - - private static final Log logger = LogFactory.getLog(DefaultInitializationScriptResource.class); - - public DefaultInitializationScriptResource(String tableName, Collection columns) { - super(scriptFor(tableName, columns).getBytes(StandardCharsets.UTF_8)); - } - - private static String scriptFor(String tableName, Collection columns) { - StringBuilder result = new StringBuilder("DROP TABLE "); - result.append(tableName).append(";\n\n"); - - result.append("CREATE TABLE ").append(tableName).append('('); - int i = 0; - for (String column : columns) { - if (i++ > 0) { - result.append(", "); - } - result.append(column).append(" VARCHAR(2000)"); - } - result.append(");\n"); - logger.debug(String.format("Generated the following initializing script for table %s:\n%s", tableName, - result.toString())); - return result.toString(); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerConfiguration.java b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerConfiguration.java deleted file mode 100644 index 80ed10463..000000000 --- a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerConfiguration.java +++ /dev/null @@ -1,288 +0,0 @@ -/* - * Copyright 2020-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -import javax.sql.DataSource; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.FactoryBean; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.io.ResourceLoader; -import org.springframework.expression.EvaluationContext; -import org.springframework.expression.EvaluationException; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.SpelParseException; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.expression.spel.support.StandardEvaluationContext; -import org.springframework.integration.aggregator.DefaultAggregatingMessageGroupProcessor; -import org.springframework.integration.aggregator.MessageCountReleaseStrategy; -import org.springframework.integration.config.AggregatorFactoryBean; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.dsl.IntegrationFlowBuilder; -import org.springframework.integration.expression.ExpressionUtils; -import org.springframework.integration.expression.ValueExpression; -import org.springframework.integration.jdbc.JdbcMessageHandler; -import org.springframework.integration.jdbc.SqlParameterSourceFactory; -import org.springframework.integration.json.JsonPropertyAccessor; -import org.springframework.integration.store.MessageGroupStore; -import org.springframework.integration.store.SimpleMessageStore; -import org.springframework.integration.support.MutableMessage; -import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; -import org.springframework.jdbc.core.namedparam.SqlParameterSource; -import org.springframework.jdbc.datasource.init.DataSourceInitializer; -import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHandler; -import org.springframework.messaging.MessageHeaders; -import org.springframework.util.LinkedMultiValueMap; -import org.springframework.util.MimeTypeUtils; -import org.springframework.util.MultiValueMap; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Robert St. John - * @author Oliver Flasch - * @author Artem Bilan - * @author Soby Chacko - * @author Szabolcs Stremler - */ -@Configuration -@EnableConfigurationProperties(JdbcConsumerProperties.class) -public class JdbcConsumerConfiguration { - - private static final Log logger = LogFactory.getLog(JdbcConsumerConfiguration.class); - - private static final Object NOT_SET = new Object(); - - private final JdbcConsumerProperties properties; - - private SpelExpressionParser spelExpressionParser = new SpelExpressionParser(); - - private EvaluationContext evaluationContext; - - public JdbcConsumerConfiguration(JdbcConsumerProperties properties, BeanFactory beanFactory) { - this.properties = properties; - this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(beanFactory); - StandardEvaluationContext standardEvaluationContext = (StandardEvaluationContext) this.evaluationContext; - standardEvaluationContext.addPropertyAccessor(new JsonPropertyAccessor()); - } - - @Bean - public static ShorthandMapConverter shorthandMapConverter() { - return new ShorthandMapConverter(); - } - - private static boolean convertibleContentType(String contentType) { - return contentType.contains("text") || contentType.contains("json") || contentType.contains("x-spring-tuple"); - } - - private static String generateSql(String tableName, Set columns) { - StringBuilder builder = new StringBuilder("INSERT INTO "); - StringBuilder questionMarks = new StringBuilder(") VALUES ("); - builder.append(tableName).append("("); - int i = 0; - - for (String column : columns) { - if (i++ > 0) { - builder.append(", "); - questionMarks.append(", "); - } - builder.append(column); - questionMarks.append(':').append(column); - } - builder.append(questionMarks).append(")"); - return builder.toString(); - } - - @Bean - IntegrationFlow jdbcConsumerFlow(@Qualifier("aggregator") MessageHandler aggregator, - JdbcMessageHandler jdbcMessageHandler) { - - final IntegrationFlowBuilder builder = - IntegrationFlow.from(Consumer.class, gateway -> gateway.beanName("jdbcConsumer")); - if (properties.getBatchSize() > 1 || properties.getIdleTimeout() > 0) { - builder.handle(aggregator); - } - return builder.handle(jdbcMessageHandler).get(); - } - - @Bean - FactoryBean aggregator(MessageGroupStore messageGroupStore) { - AggregatorFactoryBean aggregatorFactoryBean = new AggregatorFactoryBean(); - aggregatorFactoryBean.setCorrelationStrategy(message -> message.getPayload().getClass().getName()); - aggregatorFactoryBean.setReleaseStrategy(new MessageCountReleaseStrategy(this.properties.getBatchSize())); - if (this.properties.getIdleTimeout() >= 0) { - aggregatorFactoryBean.setGroupTimeoutExpression(new ValueExpression<>(this.properties.getIdleTimeout())); - } - aggregatorFactoryBean.setMessageStore(messageGroupStore); - aggregatorFactoryBean.setProcessorBean(new DefaultAggregatingMessageGroupProcessor()); - aggregatorFactoryBean.setExpireGroupsUponCompletion(true); - aggregatorFactoryBean.setSendPartialResultOnExpiry(true); - return aggregatorFactoryBean; - } - - @Bean - MessageGroupStore messageGroupStore() { - SimpleMessageStore messageGroupStore = new SimpleMessageStore(); - messageGroupStore.setTimeoutOnIdle(true); - messageGroupStore.setCopyOnGet(false); - return messageGroupStore; - } - - @Bean - public JdbcMessageHandler jdbcMessageHandler(DataSource dataSource) { - final MultiValueMap columnExpressionVariations = new LinkedMultiValueMap<>(); - for (Map.Entry entry : this.properties.getColumnsMap().entrySet()) { - String value = entry.getValue(); - columnExpressionVariations.add(entry.getKey(), this.spelExpressionParser.parseExpression(value)); - if (!value.startsWith("payload")) { - String qualified = "payload." + value; - try { - columnExpressionVariations.add(entry.getKey(), - this.spelExpressionParser.parseExpression(qualified)); - } - catch (SpelParseException e) { - logger.info("failed to parse qualified fallback expression " + qualified + - "; be sure your expression uses the 'payload.' prefix where necessary"); - } - } - } - JdbcMessageHandler jdbcMessageHandler = new JdbcMessageHandler(dataSource, - generateSql(this.properties.getTableName(), columnExpressionVariations.keySet())) { - - @Override - protected void handleMessageInternal(final Message message) { - Message convertedMessage = message; - if (message.getPayload() instanceof byte[] || message.getPayload() instanceof Iterable) { - - final String contentType = message.getHeaders().containsKey(MessageHeaders.CONTENT_TYPE) - ? message.getHeaders().get(MessageHeaders.CONTENT_TYPE).toString() - : MimeTypeUtils.APPLICATION_JSON_VALUE; - if (message.getPayload() instanceof Iterable) { - Stream messageStream = - StreamSupport.stream(((Iterable) message.getPayload()).spliterator(), false) - .map(payload -> { - if (payload instanceof byte[]) { - return convertibleContentType(contentType) ? - new String(((byte[]) payload)) : payload; - } - else { - return payload; - } - }); - convertedMessage = new MutableMessage<>(messageStream.collect(Collectors.toList()), - message.getHeaders()); - } - else { - if (convertibleContentType(contentType)) { - convertedMessage = new MutableMessage<>(new String(((byte[]) message.getPayload())), - message.getHeaders()); - } - } - } - super.handleMessageInternal(convertedMessage); - } - }; - SqlParameterSourceFactory parameterSourceFactory = - new ParameterFactory(columnExpressionVariations, this.evaluationContext); - jdbcMessageHandler.setSqlParameterSourceFactory(parameterSourceFactory); - return jdbcMessageHandler; - } - - @ConditionalOnProperty("jdbc.consumer.initialize") - @Bean - public DataSourceInitializer nonBootDataSourceInitializer(DataSource dataSource, ResourceLoader resourceLoader) { - DataSourceInitializer dataSourceInitializer = new DataSourceInitializer(); - dataSourceInitializer.setDataSource(dataSource); - ResourceDatabasePopulator databasePopulator = new ResourceDatabasePopulator(); - databasePopulator.setIgnoreFailedDrops(true); - dataSourceInitializer.setDatabasePopulator(databasePopulator); - if ("true".equals(properties.getInitialize())) { - databasePopulator.addScript( - new DefaultInitializationScriptResource(this.properties.getTableName(), - this.properties.getColumnsMap().keySet())); - } - else { - databasePopulator.addScript(resourceLoader.getResource(this.properties.getInitialize())); - } - return dataSourceInitializer; - } - - private static final class ParameterFactory implements SqlParameterSourceFactory { - - private final MultiValueMap columnExpressions; - - private final EvaluationContext context; - - ParameterFactory(MultiValueMap columnExpressions, EvaluationContext context) { - this.columnExpressions = columnExpressions; - this.context = context; - } - - @Override - public SqlParameterSource createParameterSource(Object o) { - if (!(o instanceof Message)) { - throw new IllegalArgumentException("Unable to handle type " + o.getClass().getName()); - } - Message message = (Message) o; - MapSqlParameterSource parameterSource = new MapSqlParameterSource(); - for (Map.Entry> entry : this.columnExpressions.entrySet()) { - String key = entry.getKey(); - List spels = entry.getValue(); - Object value = NOT_SET; - EvaluationException lastException = null; - for (Expression spel : spels) { - try { - value = spel.getValue(context, message); - break; - } - catch (EvaluationException e) { - lastException = e; - } - } - if (value == NOT_SET) { - if (lastException != null) { - logger.info("Could not find value for column '" + key + "': " + lastException.getMessage()); - } - parameterSource.addValue(key, null); - } - else { - parameterSource.addValue(key, value); - } - } - return parameterSource; - } - - } - -} diff --git a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerProperties.java b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerProperties.java deleted file mode 100644 index a3c5d3f10..000000000 --- a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerProperties.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import java.util.Map; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.ConfigurationProperties; - -/** - * @author Eric Bottard - * @author Artem Bilan - * @author Oliver Flasch - */ -@ConfigurationProperties("jdbc.consumer") -public class JdbcConsumerProperties { - - @Autowired - private ShorthandMapConverter shorthandMapConverter; - - /** - * The name of the table to write into. - */ - private String tableName = "messages"; - - /** - * The comma separated colon-based pairs of column names and SpEL expressions for values to insert/update. - * Names are used at initialization time to issue the DDL. - */ - private String columns = "payload:payload.toString()"; - - /** - * 'true', 'false' or the location of a custom initialization script for the table. - */ - private String initialize = "false"; - - /** - * Threshold in number of messages when data will be flushed to database table. - */ - private int batchSize = 1; - - /** - * Idle timeout in milliseconds when data is automatically flushed to database table. - */ - private long idleTimeout = -1L; - - private Map columnsMap; - - public String getTableName() { - return this.tableName; - } - - public void setTableName(String tableName) { - this.tableName = tableName; - } - - public String getColumns() { - return this.columns; - } - - public void setColumns(String columns) { - this.columns = columns; - } - - public String getInitialize() { - return this.initialize; - } - - public void setInitialize(String initialize) { - this.initialize = initialize; - } - - public int getBatchSize() { - return this.batchSize; - } - - public void setBatchSize(int batchSize) { - this.batchSize = batchSize; - } - - public long getIdleTimeout() { - return this.idleTimeout; - } - - public void setIdleTimeout(long idleTimeout) { - this.idleTimeout = idleTimeout; - } - - Map getColumnsMap() { - if (this.columnsMap == null) { - this.columnsMap = this.shorthandMapConverter.convert(this.columns); - } - return this.columnsMap; - } -} diff --git a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/ShorthandMapConverter.java b/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/ShorthandMapConverter.java deleted file mode 100644 index 78534e993..000000000 --- a/functions/consumer/jdbc-consumer/src/main/java/org/springframework/cloud/fn/consumer/jdbc/ShorthandMapConverter.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import java.util.LinkedHashMap; -import java.util.Map; - -import org.springframework.core.convert.converter.Converter; -import org.springframework.util.Assert; - -/** - * A Converter from String to Map that accepts csv {@literal key:value} pairs - * (similar to what comes out of the box in Spring Core) but also simple - * {@literal key} items, in which case the value is assumed to be equal to the key. - *

- *

Additionally, commas and colons can be escaped by using a backslash, which is - * useful if said mappings are to be used for SpEL for example.

- * - * @author Eric Bottard - * @author Artem Bilan - */ -public class ShorthandMapConverter implements Converter> { - - @Override - public Map convert(String source) { - Map result = new LinkedHashMap<>(); - - // Split on comma if not preceded by backslash - String[] mappings = source.split("(? message = MessageBuilder.withPayload(sent).build(); - jdbcConsumer.accept(message); - } - Awaitility.await().until(() -> jdbcOperations - .queryForObject("select count(*) from messages", Integer.class), value -> value == numberOfInserts); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/DataReceivedAsByteArrayTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/DataReceivedAsByteArrayTests.java deleted file mode 100644 index e4e9b9df8..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/DataReceivedAsByteArrayTests.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -@TestPropertySource(properties = "jdbc.consumer.columns=a") -public class DataReceivedAsByteArrayTests extends JdbcConsumerApplicationTests { - - @Test - public void testInsertionWhenDataReceivedAsByteArray() { - String hello = "{\"a\": \"hello\"}"; - final Message message = MessageBuilder.withPayload(hello.getBytes()).build(); - jdbcConsumer.accept(message); - final Integer count = - jdbcOperations.queryForObject("select count(*) from messages where a = ?", Integer.class, "hello"); - assertThat(count).isEqualTo(1); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ExplicitTableCreationTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ExplicitTableCreationTests.java deleted file mode 100644 index bba4fb4e1..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ExplicitTableCreationTests.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.jdbc.core.BeanPropertyRowMapper; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -@TestPropertySource(properties = {"jdbc.consumer.tableName=foobar", - "jdbc.consumer.initialize=classpath:explicit-script.sql", - "jdbc.consumer.columns=a,b"}) -public class ExplicitTableCreationTests extends JdbcConsumerApplicationTests { - - @Test - public void testInsertion() { - Payload sent = new Payload("hello", 42); - final Message message = MessageBuilder.withPayload(sent).build(); - jdbcConsumer.accept(message); - Payload result = - jdbcOperations.query("select a, b from foobar", new BeanPropertyRowMapper<>(Payload.class)) - .get(0); - assertThat(result).isEqualToComparingFieldByField(sent); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/HeaderInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/HeaderInsertTests.java deleted file mode 100644 index b8c1ceaf7..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/HeaderInsertTests.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -@TestPropertySource(properties = "jdbc.consumer.columns=a: headers[foo]") -public class HeaderInsertTests extends JdbcConsumerApplicationTests { - - @Test - public void testHeaderInsertion() { - Payload sent = new Payload("hello", 42); - final Message message = MessageBuilder.withPayload(sent) - .setHeader("foo", "bar").build(); - jdbcConsumer.accept(message); - assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ?", - Integer.class, "bar")).isEqualTo(1); - } -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ImplicitTableCreationTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ImplicitTableCreationTests.java deleted file mode 100644 index 0d79c476c..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/ImplicitTableCreationTests.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.jdbc.core.BeanPropertyRowMapper; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -@TestPropertySource(properties = { - "jdbc.consumer.tableName=no_script", - "jdbc.consumer.initialize=true", - "jdbc.consumer.columns=a,b"}) -public class ImplicitTableCreationTests extends JdbcConsumerApplicationTests { - - @Test - public void testInsertion() { - Payload sent = new Payload("hello", 42); - final Message message = MessageBuilder.withPayload(sent).build(); - jdbcConsumer.accept(message); - Payload result = jdbcOperations - .query("select a, b from no_script", new BeanPropertyRowMapper<>(Payload.class)).get(0); - assertThat(result).isEqualToComparingFieldByField(sent); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerApplicationTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerApplicationTests.java deleted file mode 100644 index c069ca3b6..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JdbcConsumerApplicationTests.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import java.util.function.Consumer; - -import org.junit.jupiter.api.AfterEach; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.jdbc.core.JdbcOperations; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; - -/** - * @author Soby Chacko - */ -@SpringBootTest -@DirtiesContext -public class JdbcConsumerApplicationTests { - - @Autowired - Consumer> jdbcConsumer; - - @Autowired - JdbcOperations jdbcOperations; - - @Autowired - JdbcTemplate jdbcTemplate; - - @AfterEach - public void cleanup() { - jdbcOperations.execute("DROP TABLE MESSAGES IF EXISTS"); - } - - static class Payload { - - private String a; - - private Integer b; - - Payload() { - } - - Payload(String a, Integer b) { - this.a = a; - this.b = b; - } - - public String getA() { - return a; - } - - public void setA(String a) { - this.a = a; - } - - public Integer getB() { - return b; - } - - public void setB(Integer b) { - this.b = b; - } - - @Override - public String toString() { - return a + b; - } - - } - - @SpringBootApplication - static class JdbcConsumerTestApplication { - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JsonStringPayloadInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JsonStringPayloadInsertTests.java deleted file mode 100644 index c167e70e8..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/JsonStringPayloadInsertTests.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -@TestPropertySource(properties = "jdbc.consumer.columns=a,b") -public class JsonStringPayloadInsertTests extends JdbcConsumerApplicationTests { - - @Test - public void testInsertion() { - String stringA = "{\"a\": \"hello1\", \"b\": 42}"; - String stringB = "{\"a\": \"hello2\", \"b\": null}"; - String stringC = "{\"a\": \"hello3\"}"; - final Message message1 = MessageBuilder.withPayload(stringA).build(); - jdbcConsumer.accept(message1); - final Message message2 = MessageBuilder.withPayload(stringB).build(); - jdbcConsumer.accept(message2); - final Message message3 = MessageBuilder.withPayload(stringC).build(); - jdbcConsumer.accept(message3); - assertThat(jdbcOperations.queryForObject( - "select count(*) from messages where a = ? and b = ?", - Integer.class, "hello1", 42)).isEqualTo(1); - assertThat(jdbcOperations.queryForObject( - "select count(*) from messages where a = ? and b IS NULL", - Integer.class, "hello2")).isEqualTo(1); - assertThat(jdbcOperations.queryForObject( - "select count(*) from messages where a = ? and b IS NULL", - Integer.class, "hello3")).isEqualTo(1); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/MapPayloadInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/MapPayloadInsertTests.java deleted file mode 100644 index 9ecce9cfb..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/MapPayloadInsertTests.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import java.util.HashMap; -import java.util.Map; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.jdbc.core.namedparam.NamedParameterJdbcOperations; -import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -@TestPropertySource(properties = "jdbc.consumer.columns=a,b") -public class MapPayloadInsertTests extends JdbcConsumerApplicationTests { - - @Test - public void testInsertion() { - NamedParameterJdbcOperations namedParameterJdbcOperations = new NamedParameterJdbcTemplate(jdbcOperations); - Map mapA = new HashMap<>(); - mapA.put("a", "hello1"); - mapA.put("b", 42); - Map mapB = new HashMap<>(); - mapB.put("a", "hello2"); - mapB.put("b", null); - Map mapC = new HashMap<>(); - mapC.put("a", "hello3"); - final Message> message1 = MessageBuilder.withPayload(mapA).build(); - jdbcConsumer.accept(message1); - final Message> message2 = MessageBuilder.withPayload(mapB).build(); - jdbcConsumer.accept(message2); - final Message> message3 = MessageBuilder.withPayload(mapC).build(); - jdbcConsumer.accept(message3); - assertThat(namedParameterJdbcOperations.queryForObject( - "select count(*) from messages where a = :a and b = :b", mapA, Integer.class)).isEqualTo(1); - assertThat(namedParameterJdbcOperations.queryForObject( - "select count(*) from messages where a = :a and b IS NULL", mapB, Integer.class)).isEqualTo(1); - assertThat(namedParameterJdbcOperations.queryForObject( - "select count(*) from messages where a = :a and b IS NULL", mapC, Integer.class)).isEqualTo(1); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleBatchInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleBatchInsertTests.java deleted file mode 100644 index 29d52b6f8..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleBatchInsertTests.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -@TestPropertySource(properties = "jdbc.consumer.batchSize=1000") -public class SimpleBatchInsertTests extends JdbcConsumerApplicationTests { - - @Test - public void testBatchInsertion() { - final int numberOfInserts = 5000; - Payload sent = new Payload("hello", 42); - for (int i = 0; i < numberOfInserts; i++) { - final Message message = MessageBuilder.withPayload(sent).build(); - jdbcConsumer.accept(message); - } - int result = jdbcOperations.queryForObject("select count(*) from messages", Integer.class); - assertThat(result).isEqualTo(numberOfInserts); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleInsertTests.java deleted file mode 100644 index 66566f968..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleInsertTests.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -public class SimpleInsertTests extends JdbcConsumerApplicationTests { - - @Test - public void testSimpleInsert() { - Payload sent = new Payload("hello", 42); - final Message message = MessageBuilder.withPayload(sent).build(); - jdbcConsumer.accept(message); - String result = jdbcOperations.queryForObject("select payload from messages", String.class); - assertThat(result).isEqualTo(("hello42")); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleMappingTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleMappingTests.java deleted file mode 100644 index 3efbe4b4e..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SimpleMappingTests.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.jdbc.core.BeanPropertyRowMapper; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -@TestPropertySource(properties = "jdbc.consumer.columns=a,b") -public class SimpleMappingTests extends JdbcConsumerApplicationTests { - - @Test - public void testInsertion() { - Payload sent = new Payload("hello", 42); - final Message message = MessageBuilder.withPayload(sent).build(); - jdbcConsumer.accept(message); - Payload result = jdbcOperations - .query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class)).get(0); - assertThat(result).isEqualToComparingFieldByField(sent); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SpELTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SpELTests.java deleted file mode 100644 index 8ec1ddf01..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/SpELTests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.jdbc.core.BeanPropertyRowMapper; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -// annotation below relies on java.util.Properties so backslash needs to be doubled -@TestPropertySource(properties = "jdbc.consumer.columns=a: a.substring(0\\\\, 4), b: b + 624") -public class SpELTests extends JdbcConsumerApplicationTests { - - @Test - public void testInsertion() { - Payload sent = new Payload("hello", 42); - final Message message = MessageBuilder.withPayload(sent).build(); - jdbcConsumer.accept(message); - Payload expected = new Payload("hell", 666); - Payload result = jdbcOperations - .query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class)).get(0); - assertThat(result).isEqualToComparingFieldByField(expected); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/UnqualifiableColumnExpressionTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/UnqualifiableColumnExpressionTests.java deleted file mode 100644 index 3f5f829af..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/UnqualifiableColumnExpressionTests.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -@TestPropertySource(properties = "jdbc.consumer.columns=a: new StringBuilder(payload.a).reverse().toString(), b") -public class UnqualifiableColumnExpressionTests extends JdbcConsumerApplicationTests { - - @Test - public void doesNotFailParsingUnqualifiableExpression() { - // if the app initializes, the test condition passes, but go ahead and apply the column expression anyway - jdbcConsumer.accept(MessageBuilder.withPayload(new Payload("desrever", 123)).build()); - assertThat(jdbcOperations.queryForObject("select count(*) from messages where a = ? and b = ?", - Integer.class, "reversed", 123)).isEqualTo(1); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/VaryingInsertTests.java b/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/VaryingInsertTests.java deleted file mode 100644 index 8fafda2b5..000000000 --- a/functions/consumer/jdbc-consumer/src/test/java/org/springframework/cloud/fn/consumer/jdbc/VaryingInsertTests.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.jdbc; - -import java.util.List; - -import org.assertj.core.api.Assertions; -import org.junit.jupiter.api.Test; - -import org.springframework.integration.support.MessageBuilder; -import org.springframework.jdbc.core.BeanPropertyRowMapper; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -/** - * @author Eric Bottard - * @author Thomas Risberg - * @author Artem Bilan - * @author Robert St. John - * @author Oliver Flasch - * @author Soby Chacko - * @author Szabolcs Stremler - */ -@TestPropertySource(properties = "jdbc.consumer.columns=a,b") -public class VaryingInsertTests extends JdbcConsumerApplicationTests { - - @Test - public void testInsertion() { - Payload a = new Payload("hello", 42); - Payload b = new Payload("world", 12); - Payload c = new Payload("bonjour", null); - Payload d = new Payload(null, 22); - final Message message1 = MessageBuilder.withPayload(a).build(); - jdbcConsumer.accept(message1); - final Message message2 = MessageBuilder.withPayload(b).build(); - jdbcConsumer.accept(message2); - final Message message3 = MessageBuilder.withPayload(c).build(); - jdbcConsumer.accept(message3); - final Message message4 = MessageBuilder.withPayload(d).build(); - jdbcConsumer.accept(message4); - List result = jdbcOperations - .query("select a, b from messages", new BeanPropertyRowMapper<>(Payload.class)); - Assertions.assertThat(result).extracting("a").containsExactly("hello", "world", "bonjour", null); - Assertions.assertThat(result).extracting("b").contains(42, 12, 22, null); - } - -} diff --git a/functions/consumer/jdbc-consumer/src/test/resources/explicit-script.sql b/functions/consumer/jdbc-consumer/src/test/resources/explicit-script.sql deleted file mode 100644 index ac07d2d01..000000000 --- a/functions/consumer/jdbc-consumer/src/test/resources/explicit-script.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Used in test for explicit script - -create table foobar( - a varchar(2000), - b VARCHAR (2000) -); diff --git a/functions/consumer/jdbc-consumer/src/test/resources/schema.sql b/functions/consumer/jdbc-consumer/src/test/resources/schema.sql deleted file mode 100644 index 65e85283f..000000000 --- a/functions/consumer/jdbc-consumer/src/test/resources/schema.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Run by default by Boot infrastructure - -create table messages( - a varchar(2000), - b VARCHAR (2000), - payload VARCHAR (2000) -); diff --git a/functions/consumer/kafka-publisher/README.adoc b/functions/consumer/kafka-publisher/README.adoc deleted file mode 100644 index de8027cdd..000000000 --- a/functions/consumer/kafka-publisher/README.adoc +++ /dev/null @@ -1,31 +0,0 @@ -# Apache Kafka Publisher (Consumer function) - -A `Consumer>` that allows to publish messages to Apache Kafka topic. - - -## Beans for injection - -The `KafkaPublisherConfiguration` is an auto-configuration, so no need to import anything. - -The `Consumer> kafkaPublisher` bean can be injected into the target service for producing data into Kafka topic. - -## Configuration Options - -All configuration properties are prefixed with `kafka.publisher`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/kafka/KafkaPublisherProperties.java[KafkaPublisherProperties]. -Also, this artifact fully depends on Spring for Apache Kafka auto-configuration and injects a `KafkaTemplate` from there. - -A `ComponentCustomizer>` bean can be added in the target project to provide any custom options for the `KafkaProducerMessageHandlerSpec` configuration used by the `kafkaPublisher`. - -The `KafkaPublisherConfiguration` also exposes 3 `PublishSubscribeChannel`: `kafkaPublisherSuccessChannel`, `kafkaPublisherFailureChannel`, `kafkaPublisherFuturesChannel`. -They are mapped to respective options of the `KafkaProducerMessageHandler`. -They may be subscribed in the target project any possible Spring Integration way. -See more information about `KafkaProducerMessageHandler` configuration and behavior in Spring Integration https://docs.spring.io/spring-integration/docs/current/reference/html/kafka.html#kafka-outbound[documentation]. - -## Tests - - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/kafka-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes an Apache Kafka sink. diff --git a/functions/consumer/kafka-publisher/pom.xml b/functions/consumer/kafka-publisher/pom.xml deleted file mode 100644 index fa3c2a742..000000000 --- a/functions/consumer/kafka-publisher/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - 4.0.0 - - - spring-functions-parent - org.springframework.cloud.fn - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - kafka-publisher - kafka-publisher - Apache Kafka Publisher(Consumer Function) - - - - org.springframework.integration - spring-integration-kafka - - - - org.springframework.kafka - spring-kafka-test - test - - - org.springframework.integration - spring-integration-test - test - - - - diff --git a/functions/consumer/kafka-publisher/src/main/java/org/springframework/cloud/fn/consumer/kafka/KafkaPublisherConfiguration.java b/functions/consumer/kafka-publisher/src/main/java/org/springframework/cloud/fn/consumer/kafka/KafkaPublisherConfiguration.java deleted file mode 100644 index 6138f10bb..000000000 --- a/functions/consumer/kafka-publisher/src/main/java/org/springframework/cloud/fn/consumer/kafka/KafkaPublisherConfiguration.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.kafka; - -import java.time.Duration; -import java.util.function.Consumer; - -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.boot.context.properties.PropertyMapper; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.integration.channel.PublishSubscribeChannel; -import org.springframework.integration.expression.ValueExpression; -import org.springframework.integration.kafka.dsl.Kafka; -import org.springframework.integration.kafka.dsl.KafkaProducerMessageHandlerSpec; -import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.support.DefaultKafkaHeaderMapper; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageChannel; - -/** - * A configuration for Apache Kafka Publisher (Consumer function). - * Uses a {@link KafkaProducerMessageHandlerSpec} to publish a message to Kafka topic. - * - * @author Artem Bilan - * - * @since 4.0 - */ -@AutoConfiguration(after = KafkaAutoConfiguration.class) -@EnableConfigurationProperties(KafkaPublisherProperties.class) -public class KafkaPublisherConfiguration { - - /** - * The function to produce messages to the Kafka topic. - * @param kafkaProducerMessageHandler the handler to publish messages to Kafka. - * @return the consumer for accepting message for producing to Kafka. - */ - @Bean - public Consumer> kafkaPublisher(KafkaProducerMessageHandler kafkaProducerMessageHandler) { - return kafkaProducerMessageHandler::handleMessage; - } - - @Bean - public KafkaProducerMessageHandler kafkaProducerMessageHandlerSpec(KafkaTemplate kafkaTemplate, - KafkaPublisherProperties kafkaPublisherProperties, - PublishSubscribeChannel kafkaPublisherSuccessChannel, - PublishSubscribeChannel kafkaPublisherFailureChannel, - PublishSubscribeChannel kafkaPublisherFuturesChannel, - @Nullable ComponentCustomizer> kafkaProducerSpecComponentCustomizer) { - - var kafkaProducerMessageHandlerSpec = Kafka.outboundChannelAdapter(kafkaTemplate); - - PropertyMapper mapper = PropertyMapper.get().alwaysApplyingWhenNonNull(); - - mapper.from(kafkaPublisherProperties.getTopic()).to(kafkaProducerMessageHandlerSpec::topic); - mapper.from(kafkaPublisherProperties.getTopicExpression()).to(kafkaProducerMessageHandlerSpec::topicExpression); - mapper.from(kafkaPublisherProperties.getKey()).to(kafkaProducerMessageHandlerSpec::messageKey); - mapper.from(kafkaPublisherProperties.getKeyExpression()).to(kafkaProducerMessageHandlerSpec::messageKeyExpression); - mapper.from(kafkaPublisherProperties.getPartition()).to(kafkaProducerMessageHandlerSpec::partitionId); - mapper.from(kafkaPublisherProperties.getPartitionExpression()).to(kafkaProducerMessageHandlerSpec::partitionIdExpression); - mapper.from(kafkaPublisherProperties.getTimestamp()).as(ValueExpression::new).to(kafkaProducerMessageHandlerSpec::timestampExpression); - mapper.from(kafkaPublisherProperties.getTimestampExpression()).to(kafkaProducerMessageHandlerSpec::timestampExpression); - mapper.from(kafkaPublisherProperties.getSendTimeout()).as(Duration::toMillis).to(kafkaProducerMessageHandlerSpec::sendTimeout); - mapper.from(kafkaPublisherProperties.isUseTemplateConverter()).to(kafkaProducerMessageHandlerSpec::useTemplateConverter); - - kafkaProducerMessageHandlerSpec.headerMapper(new DefaultKafkaHeaderMapper(kafkaPublisherProperties.getMappedHeaders())); - - kafkaProducerMessageHandlerSpec.sendSuccessChannel(kafkaPublisherSuccessChannel); - kafkaProducerMessageHandlerSpec.sendFailureChannel(kafkaPublisherFailureChannel); - kafkaProducerMessageHandlerSpec.futuresChannel(kafkaPublisherFuturesChannel); - - if (kafkaProducerSpecComponentCustomizer != null) { - kafkaProducerSpecComponentCustomizer.customize(kafkaProducerMessageHandlerSpec); - } - - return kafkaProducerMessageHandlerSpec.get(); - } - - /** - * @see KafkaProducerMessageHandler#setSendSuccessChannel(MessageChannel) - */ - @Bean - public PublishSubscribeChannel kafkaPublisherSuccessChannel() { - return new PublishSubscribeChannel(); - } - - /** - * @see KafkaProducerMessageHandler#setSendFailureChannel(MessageChannel) - */ - @Bean - public PublishSubscribeChannel kafkaPublisherFailureChannel() { - return new PublishSubscribeChannel(); - } - - /** - * @see KafkaProducerMessageHandler#setFuturesChannel(MessageChannel) - */ - @Bean - public PublishSubscribeChannel kafkaPublisherFuturesChannel() { - return new PublishSubscribeChannel(); - } - -} diff --git a/functions/consumer/kafka-publisher/src/main/java/org/springframework/cloud/fn/consumer/kafka/KafkaPublisherProperties.java b/functions/consumer/kafka-publisher/src/main/java/org/springframework/cloud/fn/consumer/kafka/KafkaPublisherProperties.java deleted file mode 100644 index 4994cab6e..000000000 --- a/functions/consumer/kafka-publisher/src/main/java/org/springframework/cloud/fn/consumer/kafka/KafkaPublisherProperties.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.kafka; - -import java.time.Duration; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; - -/** - * Properties for the Kafka Publisher (Consumer function). - * - * @author Artem Bilan - * - * @since 4.0 - */ -@ConfigurationProperties("kafka.publisher") -public class KafkaPublisherProperties { - - /** - * Kafka topic - overridden by topicExpression, if supplied. Defaults to KafkaTemplate.getDefaultTopic() - */ - private String topic; - - /** - * A SpEL expression that evaluates to a Kafka topic. - */ - private Expression topicExpression; - - /** - * Kafka record key - overridden by keyExpression, if supplied. - */ - private String key; - - /** - * A SpEL expression that evaluates to a Kafka record key. - */ - private Expression keyExpression; - - /** - * Kafka topic partition - overridden by partitionExpression, if supplied. - */ - private Integer partition; - - /** - * A SpEL expression that evaluates to a Kafka topic partition. - */ - private Expression partitionExpression; - - /** - * Kafka record timestamp - overridden by timestampExpression, if supplied. - */ - private Long timestamp; - - /** - * A SpEL expression that evaluates to a Kafka record timestamp. - */ - private Expression timestampExpression; - - /** - * True if Kafka producer handler should operation in a sync mode. - */ - private boolean sync; - - /** - * How long Kafka producer handler should wait for send operation results. Defaults to 10 seconds. - */ - private Duration sendTimeout = Duration.ofSeconds(10); - - /** - * Headers that will be mapped. - */ - private String[] mappedHeaders = { "*" }; - - /** - * Whether to use the template's message converter to create a Kafka record. - */ - private boolean useTemplateConverter; - - public String getTopic() { - return this.topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public Expression getTopicExpression() { - return this.topicExpression; - } - - public void setTopicExpression(Expression topicExpression) { - this.topicExpression = topicExpression; - } - - public String getKey() { - return this.key; - } - - public void setKey(String key) { - this.key = key; - } - - public Expression getKeyExpression() { - return this.keyExpression; - } - - public void setKeyExpression(Expression keyExpression) { - this.keyExpression = keyExpression; - } - - public Integer getPartition() { - return this.partition; - } - - public void setPartition(Integer partition) { - this.partition = partition; - } - - public Expression getPartitionExpression() { - return this.partitionExpression; - } - - public void setPartitionExpression(Expression partitionExpression) { - this.partitionExpression = partitionExpression; - } - - public Long getTimestamp() { - return this.timestamp; - } - - public void setTimestamp(Long timestamp) { - this.timestamp = timestamp; - } - - public Expression getTimestampExpression() { - return this.timestampExpression; - } - - public void setTimestampExpression(Expression timestampExpression) { - this.timestampExpression = timestampExpression; - } - - public boolean isSync() { - return this.sync; - } - - public void setSync(boolean sync) { - this.sync = sync; - } - - public Duration getSendTimeout() { - return this.sendTimeout; - } - - public void setSendTimeout(Duration sendTimeout) { - this.sendTimeout = sendTimeout; - } - - public String[] getMappedHeaders() { - return this.mappedHeaders; - } - - public void setMappedHeaders(String[] mappedHeaders) { - this.mappedHeaders = mappedHeaders; - } - - public boolean isUseTemplateConverter() { - return this.useTemplateConverter; - } - - public void setUseTemplateConverter(boolean useTemplateConverter) { - this.useTemplateConverter = useTemplateConverter; - } - -} diff --git a/functions/consumer/kafka-publisher/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/consumer/kafka-publisher/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index 9be4c43d4..000000000 --- a/functions/consumer/kafka-publisher/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.consumer.kafka.KafkaPublisherConfiguration diff --git a/functions/consumer/kafka-publisher/src/test/java/org/springframework/cloud/fn/consumer/kafka/KafkaPublisherConfigurationTests.java b/functions/consumer/kafka-publisher/src/test/java/org/springframework/cloud/fn/consumer/kafka/KafkaPublisherConfigurationTests.java deleted file mode 100644 index 124af4779..000000000 --- a/functions/consumer/kafka-publisher/src/test/java/org/springframework/cloud/fn/consumer/kafka/KafkaPublisherConfigurationTests.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.kafka; - -import java.time.Duration; -import java.util.Arrays; -import java.util.Map; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.TopicPartition; -import org.apache.kafka.common.header.Header; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Sinks; - -import org.springframework.boot.autoconfigure.AutoConfigurations; -import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.cloud.fn.common.config.SpelExpressionConverterConfiguration; -import org.springframework.context.ApplicationContext; -import org.springframework.integration.channel.PublishSubscribeChannel; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.kafka.KafkaException; -import org.springframework.kafka.core.ConsumerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.support.KafkaHeaders; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHandlingException; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.support.GenericMessage; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; - -/** - * @author Artem Bilan - * - * @since 4.0 - */ -public class KafkaPublisherConfigurationTests { - - static final EmbeddedKafkaBroker EMBEDDED_KAFKA = - new EmbeddedKafkaBroker(1, true, 1) - .brokerListProperty("spring.kafka.bootstrap-servers"); - - final ApplicationContextRunner contextRunner = new ApplicationContextRunner() - .withConfiguration(AutoConfigurations.of( - KafkaAutoConfiguration.class, - KafkaPublisherConfiguration.class, - SpelExpressionConverterConfiguration.class)); - - @BeforeAll - static void initializeEmbeddedKafka() { - EMBEDDED_KAFKA.afterPropertiesSet(); - } - - @Test - void defaultTopicReceivesTheRecord() { - String defaultTopic = "DEFAULT_TOPIC"; - this.contextRunner.withPropertyValues("spring.kafka.template.defaultTopic=" + defaultTopic) - .run((context) -> { - KafkaTemplate kafkaTemplate = obtainKafkaTemplate(context); - Consumer> kafkaPublisher = getKafkaPublisher(context); - String testData = "test data"; - kafkaPublisher.accept(new GenericMessage<>(testData)); - ConsumerRecord receive = kafkaTemplate.receive(defaultTopic, 0, 0, Duration.ofSeconds(10)); - assertThat(receive).extracting(ConsumerRecord::value).isEqualTo(testData); - }); - } - - @Test - void wrongPartitionViaProperties() { - this.contextRunner.withPropertyValues( - "spring.kafka.producer.properties[max.block.ms]=1000", - "kafka.publisher.topic=topic1", - "kafka.publisher.partition=1", // Our broker allows only one partition for auto-created topic - "kafka.publisher.sync=true") - .run((context) -> { - Consumer> kafkaConsumer = getKafkaPublisher(context); - assertThatExceptionOfType(MessageHandlingException.class) - .isThrownBy(() -> kafkaConsumer.accept(new GenericMessage<>("test data"))) - .withCauseInstanceOf(KafkaException.class) - .withStackTraceContaining("Topic topic1 not present in metadata after 1000 ms."); - }); - } - - @Test - void successChannelInteractionAndMappedHeaders() { - this.contextRunner.withPropertyValues("kafka.publisher.topicExpression=headers.topic", - "kafka.publisher.mappedHeaders=mapped") - .run((context) -> { - KafkaTemplate kafkaTemplate = obtainKafkaTemplate(context); - Consumer> kafkaConsumer = getKafkaPublisher(context); - - PublishSubscribeChannel kafkaConsumerSuccessChannel = - context.getBean("kafkaPublisherSuccessChannel", PublishSubscribeChannel.class); - - Sinks.One> successSend = Sinks.one(); - - kafkaConsumerSuccessChannel.subscribe(successSend::tryEmitValue); - - String testTopic = "topic2"; - String testData = "some other data"; - Message testMessage = - MessageBuilder.withPayload(testData) - .setHeader("topic", testTopic) - .setHeader("mapped", "mapped value") - .setHeader("not mapped", "not mapped") - .build(); - - kafkaConsumer.accept(testMessage); - - ConsumerRecord receive = kafkaTemplate.receive(testTopic, 0, 0, Duration.ofSeconds(10)); - assertThat(receive).extracting(ConsumerRecord::value).isEqualTo(testData); - Map headers = - Arrays.stream(receive.headers().toArray()) - .collect(Collectors.toMap(Header::key, (header) -> new String(header.value()))); - assertThat(headers) - .containsEntry("mapped", "mapped value") - .doesNotContainKeys("topic", "not mapped"); - - Message successMessage = successSend.asMono().block(Duration.ofSeconds(10)); - - assertThat(successMessage) - .satisfies(message -> { - assertThat(message.getPayload()).isEqualTo(testData); - MessageHeaders messageHeaders = message.getHeaders(); - assertThat(messageHeaders) - .containsKeys("topic", "mapped", "not mapped", KafkaHeaders.RECORD_METADATA); - assertThat(messageHeaders.get(KafkaHeaders.RECORD_METADATA)) - .isInstanceOf(RecordMetadata.class) - .extracting("topicPartition") - .isEqualTo(new TopicPartition(testTopic, 0)); - }); - }); - } - - @SuppressWarnings("unchecked") - private static KafkaTemplate obtainKafkaTemplate(ApplicationContext applicationContext) { - KafkaTemplate kafkaTemplate = applicationContext.getBean(KafkaTemplate.class); - kafkaTemplate.setConsumerFactory(applicationContext.getBean(ConsumerFactory.class)); - return kafkaTemplate; - } - - @SuppressWarnings("unchecked") - private static Consumer> getKafkaPublisher(ApplicationContext applicationContext) { - return (Consumer>) applicationContext.getBean("kafkaPublisher"); - } - -} diff --git a/functions/consumer/log-consumer/README.adoc b/functions/consumer/log-consumer/README.adoc deleted file mode 100644 index 2f3bd9ec5..000000000 --- a/functions/consumer/log-consumer/README.adoc +++ /dev/null @@ -1,26 +0,0 @@ -# Log Consumer - -A consumer that allows you to log the data -The consumer uses the `LoggingMessageHandler` from Spring Integration. - -## Beans for injection - -You can import `LogConsumerConfiguration` in the application and then inject the following bean. - -`Consumer> logConsumer` - -You can use `logConsumer` as a qualifier when injecting. - -## Configuration Options - -All configuration properties are prefixed with `log`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerProperties.java[LogConsumerProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/log/LogConsumerApplicationTests.java[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/log-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a Log sink. \ No newline at end of file diff --git a/functions/consumer/log-consumer/pom.xml b/functions/consumer/log-consumer/pom.xml deleted file mode 100644 index e2346ab52..000000000 --- a/functions/consumer/log-consumer/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - log-consumer - log-consumer - Log Consumer - - - - org.springframework.cloud.fn - payload-converter-function - ${project.version} - - - org.hibernate.validator - hibernate-validator - true - - - - diff --git a/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerConfiguration.java b/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerConfiguration.java deleted file mode 100644 index ede6abaf0..000000000 --- a/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerConfiguration.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2020-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.log; - -import java.util.function.Consumer; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.messaging.Message; - -/** - * The Configuration class for {@link Consumer} which logs incoming data. - * For the logging logic a Spring Integration {@link org.springframework.integration.handler.LoggingHandler} - * is used. - * If incoming payload is a {@code byte[]} and incoming message {@code contentType} header is text-compatible - * (e.g. {@code application/json}), it is converted into a {@link String}. - * Otherwise the payload is passed to logger as is. - * - * @author Artem Bilan - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties(LogConsumerProperties.class) -public class LogConsumerConfiguration { - - @Bean - IntegrationFlow logConsumerFlow(LogConsumerProperties logSinkProperties) { - return IntegrationFlow.from(MessageConsumer.class, (gateway) -> gateway.beanName("logConsumer")) - .log(logSinkProperties.getLevel(), logSinkProperties.getName(), logSinkProperties.getExpression()) - .nullChannel(); - } - - private interface MessageConsumer extends Consumer> { - - } - -} diff --git a/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerProperties.java b/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerProperties.java deleted file mode 100644 index ffb549290..000000000 --- a/functions/consumer/log-consumer/src/main/java/org/springframework/cloud/fn/consumer/log/LogConsumerProperties.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.log; - -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.integration.handler.LoggingHandler; -import org.springframework.validation.annotation.Validated; - -import static org.springframework.integration.handler.LoggingHandler.Level.INFO; - -/** - * Configuration properties for the Log Sink app. - * - * @author Gary Russell - * @author Eric Bottard - * @author Chris Schaefer - * @author Artem Bilan - */ -@ConfigurationProperties("log") -@Validated -public class LogConsumerProperties { - - /** - * The name of the logger to use. - */ - @Value("${spring.application.name:log.consumer}") - private String name; - - /** - * A SpEL expression (against the incoming message) to evaluate as the logged message. - */ - private String expression = "payload"; - - /** - * The level at which to log messages. - */ - private LoggingHandler.Level level = INFO; - - @NotBlank - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - @NotBlank - public String getExpression() { - return expression; - } - - public void setExpression(String expression) { - this.expression = expression; - } - - @NotNull - public LoggingHandler.Level getLevel() { - return level; - } - - public void setLevel(LoggingHandler.Level level) { - this.level = level; - } - -} diff --git a/functions/consumer/log-consumer/src/test/java/org/springframework/cloud/fn/consumer/log/LogConsumerApplicationTests.java b/functions/consumer/log-consumer/src/test/java/org/springframework/cloud/fn/consumer/log/LogConsumerApplicationTests.java deleted file mode 100644 index 5b990a017..000000000 --- a/functions/consumer/log-consumer/src/test/java/org/springframework/cloud/fn/consumer/log/LogConsumerApplicationTests.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2020-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.log; - -import java.util.function.Consumer; -import java.util.function.Supplier; - -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -import org.springframework.beans.DirectFieldAccessor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.core.log.LogAccessor; -import org.springframework.integration.handler.LoggingHandler; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.util.MimeType; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -/** - * @author Artem Bilan - */ -@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) -@SpringBootTest({ "log.name=foo", "log.level=warn", "log.expression=payload.toUpperCase()" }) -class LogConsumerApplicationTests { - - @Autowired - private Consumer> logConsumer; - - @Autowired - @Qualifier("logConsumerFlow.logging-channel-adapter#0") - private LoggingHandler loggingHandler; - - @Test - public void testJsonContentType() { - Message message = MessageBuilder.withPayload("{\"foo\":\"bar\"}") - .setHeader("contentType", new MimeType("json")) - .build(); - testMessage(message, "{\"foo\":\"bar\"}"); - } - - private void testMessage(Message message, String expectedPayload) { - assertThat(this.loggingHandler.getLevel()).isEqualTo(LoggingHandler.Level.WARN); - LogAccessor logger = TestUtils.getPropertyValue(this.loggingHandler, "messageLogger", LogAccessor.class); - assertThat(TestUtils.getPropertyValue(logger.getLog(), "logger.name")).isEqualTo("foo"); - logger = spy(logger); - new DirectFieldAccessor(this.loggingHandler).setPropertyValue("messageLogger", logger); - this.logConsumer.accept(message); - ArgumentCaptor> captor = ArgumentCaptor.forClass(Supplier.class); - verify(logger).warn(captor.capture()); - assertThat(captor.getValue().get()).isEqualTo(expectedPayload.toUpperCase()); - this.loggingHandler.setLogExpressionString("#this"); - this.logConsumer.accept(message); - verify(logger, times(2)).warn(captor.capture()); - -// Message captorMessage = (Message) captor.getAllValues().get(2); -// assertThat(captorMessage.getPayload()).isEqualTo(expectedPayload); -// -// MessageHeaders messageHeaders = captorMessage.getHeaders(); -// assertThat(messageHeaders).hasSize(4); -// -// assertThat(messageHeaders) -// .containsEntry(MessageHeaders.CONTENT_TYPE, message.getHeaders().get(MessageHeaders.CONTENT_TYPE)); - } - - @SpringBootApplication - static class LogConsumerTestApplication { - } -} diff --git a/functions/consumer/mongodb-consumer/README.adoc b/functions/consumer/mongodb-consumer/README.adoc deleted file mode 100644 index 236cf009d..000000000 --- a/functions/consumer/mongodb-consumer/README.adoc +++ /dev/null @@ -1,31 +0,0 @@ -# MongoDB Consumer - -A consumer that allows you to insert records into MongoDB. - -## Beans for injection - -You can import `MongoDbConsumerConfiguration` in the application and then inject one of the following beans. - -`Function, Mono> mongodbConsumerFunction` - Allows you to subscribe. - -`Consumer> mongodbConsumer` - Wraps the function as a Consumer with no-op subscriber. - -You can use `mongodbConsumer` or `mongodbConsumerFunction` as a qualifier when injecting. - -The return value from the function can be ignored as this is used as a consumer to send records to MongoDB. - -## Configuration Options - -All configuration properties are prefixed with `mongodb.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDBConsumerProperties.java[MongoDBConsumerProperties]. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `ReactiveMongoDbStoringMessageHandler` configuration used by the `mongodbConsumer`. - -## Examples - -See this link:src/test/java/org/springframework/cloud/fn/consumer/mongo/MongoDBConsumerApplicationTests.java[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/mongodb-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a MongoDB sink. diff --git a/functions/consumer/mongodb-consumer/pom.xml b/functions/consumer/mongodb-consumer/pom.xml deleted file mode 100644 index 3e663799c..000000000 --- a/functions/consumer/mongodb-consumer/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - mongodb-consumer - mongodb-consumer - Mongo DB consumer - - - - org.springframework.integration - spring-integration-mongodb - - - org.mongodb - mongodb-driver-reactivestreams - - - org.testcontainers - mongodb - ${testcontainers.version} - test - - - org.springframework.cloud.fn - mongodb-common - ${project.version} - test-jar - test - - - - diff --git a/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerConfiguration.java b/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerConfiguration.java deleted file mode 100644 index e50d3528b..000000000 --- a/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerConfiguration.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2017-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.mongo; - -import java.util.function.Consumer; -import java.util.function.Function; - -import reactor.core.publisher.Mono; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.ReactiveMongoTemplate; -import org.springframework.expression.Expression; -import org.springframework.expression.common.LiteralExpression; -import org.springframework.integration.mongodb.outbound.ReactiveMongoDbStoringMessageHandler; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.messaging.ReactiveMessageHandler; - -/** - * A configuration for MongoDB Consumer function. Uses a - * {@link ReactiveMongoDbStoringMessageHandler} to save payload contents to Mongo DB. - * - * @author Artem Bilan - * @author David Turanski - * - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties({ MongoDbConsumerProperties.class }) -public class MongoDbConsumerConfiguration { - - private final MongoDbConsumerProperties properties; - - private final ReactiveMongoTemplate mongoTemplate; - - public MongoDbConsumerConfiguration(MongoDbConsumerProperties properties, ReactiveMongoTemplate mongoTemplate) { - this.properties = properties; - this.mongoTemplate = mongoTemplate; - } - - @Bean - public Consumer> mongodbConsumer(Function, Mono> mongodbConsumerFunction) { - return message -> mongodbConsumerFunction.apply(message).subscribe(); - } - - @Bean - public Function, Mono> mongodbConsumerFunction( - ReactiveMessageHandler mongoConsumerMessageHandler) { - - return mongoConsumerMessageHandler::handleMessage; - } - - @Bean - public ReactiveMessageHandler mongoConsumerMessageHandler( - @Nullable ComponentCustomizer mongoDbMessageHandlerCustomizer) { - - ReactiveMongoDbStoringMessageHandler mongoDbMessageHandler = new ReactiveMongoDbStoringMessageHandler( - this.mongoTemplate); - Expression collectionExpression = this.properties.getCollectionExpression(); - if (collectionExpression == null) { - collectionExpression = new LiteralExpression(this.properties.getCollection()); - } - mongoDbMessageHandler.setCollectionNameExpression(collectionExpression); - if (mongoDbMessageHandlerCustomizer != null) { - mongoDbMessageHandlerCustomizer.customize(mongoDbMessageHandler); - } - return mongoDbMessageHandler; - } - -} diff --git a/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerProperties.java b/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerProperties.java deleted file mode 100644 index 8986ce923..000000000 --- a/functions/consumer/mongodb-consumer/src/main/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerProperties.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2019-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.mongo; - -import jakarta.validation.constraints.AssertTrue; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.util.StringUtils; -import org.springframework.validation.annotation.Validated; -/** - * @author Artem Bilan - * @author David Turanski - * - */ -@ConfigurationProperties("mongodb.consumer") -@Validated -public class MongoDbConsumerProperties { - - /** - * The MongoDB collection to store data. - */ - private String collection; - - /** - * The SpEL expression to evaluate MongoDB collection. - */ - private Expression collectionExpression; - - public void setCollection(String collection) { - this.collection = collection; - } - - public String getCollection() { - return this.collection; - } - - public void setCollectionExpression(Expression collectionExpression) { - this.collectionExpression = collectionExpression; - } - - public Expression getCollectionExpression() { - return collectionExpression; - } - - @AssertTrue(message = "One of 'collection' or 'collectionExpression' is required") - private boolean isValid() { - return StringUtils.hasText(this.collection) || this.collectionExpression != null; - } -} diff --git a/functions/consumer/mongodb-consumer/src/test/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerApplicationTests.java b/functions/consumer/mongodb-consumer/src/test/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerApplicationTests.java deleted file mode 100644 index b5f43bfc0..000000000 --- a/functions/consumer/mongodb-consumer/src/test/java/org/springframework/cloud/fn/consumer/mongo/MongoDbConsumerApplicationTests.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2019-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.mongo; - -import java.time.Duration; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Consumer; - -import org.bson.Document; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.common.mongo.MongoDbTestContainerSupport; -import org.springframework.data.mongodb.core.ReactiveMongoTemplate; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; - -/** - * @author David Turanski - * @author Chris Bono - */ -@SpringBootTest(properties = { - "mongodb.consumer.collection=testing" -}) -class MongoDbConsumerApplicationTests implements MongoDbTestContainerSupport { - - @DynamicPropertySource - static void mongoDbProperties(DynamicPropertyRegistry registry) { - registry.add("spring.data.mongodb.port", MONGO_CONTAINER::getFirstMappedPort); - registry.add("spring.data.mongodb.database", () -> "test"); - } - - @Autowired - private MongoDbConsumerProperties properties; - - @Autowired - private Consumer> mongodbConsumer; - - @Autowired - private ReactiveMongoTemplate mongoTemplate; - - @Test - void testMongodbConsumer() { - Map data1 = new HashMap<>(); - data1.put("foo", "bar"); - - Map data2 = new HashMap<>(); - data2.put("firstName", "Foo"); - data2.put("lastName", "Bar"); - - Flux> messages = Flux.just( - new GenericMessage<>(data1), - new GenericMessage<>(data2), - new GenericMessage<>("{\"my_data\": \"THE DATA\"}")); - - messages.map(message -> { - mongodbConsumer.accept(message); - return message; - - }).subscribe(); - - await().timeout(Duration.ofSeconds(10)) - .until(() -> mongoTemplate.findAll(Document.class, properties.getCollection()).count().block() == 3L); - - StepVerifier.create(this.mongoTemplate.findAll(Document.class, properties.getCollection()) - .sort(Comparator.comparing(d -> d.get("_id").toString()))) - .assertNext(document -> { - assertThat(document.get("foo")).isEqualTo("bar"); - }) - .assertNext(document -> { - assertThat(document.get("firstName")).isEqualTo("Foo"); - assertThat(document.get("lastName")).isEqualTo("Bar"); - }) - .assertNext(document -> { - assertThat(document.get("my_data")).isEqualTo("THE DATA"); - }) - .verifyComplete(); - } - - @SpringBootApplication - static class MongoDbConsumerTestApplication { - } -} diff --git a/functions/consumer/mongodb-consumer/src/test/resources/logback-test.xml b/functions/consumer/mongodb-consumer/src/test/resources/logback-test.xml deleted file mode 100644 index 1378a823a..000000000 --- a/functions/consumer/mongodb-consumer/src/test/resources/logback-test.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n - - - - - - - - - - diff --git a/functions/consumer/mqtt-consumer/README.adoc b/functions/consumer/mqtt-consumer/README.adoc deleted file mode 100644 index 2bc79d883..000000000 --- a/functions/consumer/mqtt-consumer/README.adoc +++ /dev/null @@ -1,33 +0,0 @@ -# MQTT Consumer - -A consumer that allows you to send messages using the MQTT protocol. - -## Beans for injection - -You can import `MqttConsumerConfiguration` in the application and then inject the following bean. - -`Consumer> mqttConsumer` - -You can use `mqttConsumer` as a qualifier when injecting. - -## Configuration Options - -All configuration properties are prefixed with `mqtt.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/mqtt/MqttConsumerProperties.java[MqttConsumerProperties]. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `MqttPahoMessageHandler` configuration used by the `mqttConsumer`. - -## SSL Configuration - -The MQTT Paho client can accept an SSL configuration via `MqttConnectOptions.setSSLProperties()`. -These properties are exposed on the `MqttProperties.sslProperties` map. -The keys for these SSL properties should be taken from the `org.eclipse.paho.client.mqttv3.internal.security.SSLSocketFactoryFactory` constants, which all start with the `com.ibm.ssl.` prefix. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/mqtt/MqttConsumerTests.java[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/mqtt-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a MQTT sink. diff --git a/functions/consumer/mqtt-consumer/pom.xml b/functions/consumer/mqtt-consumer/pom.xml deleted file mode 100644 index eb38c25ec..000000000 --- a/functions/consumer/mqtt-consumer/pom.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - mqtt-consumer - mqtt-consumer - mqtt consumer - - - - org.springframework.cloud.fn - mqtt-common - ${project.version} - - - diff --git a/functions/consumer/mqtt-consumer/src/main/java/org/springframework/cloud/fn/consumer/mqtt/MqttConsumerConfiguration.java b/functions/consumer/mqtt-consumer/src/main/java/org/springframework/cloud/fn/consumer/mqtt/MqttConsumerConfiguration.java deleted file mode 100644 index cd8d8a0d1..000000000 --- a/functions/consumer/mqtt-consumer/src/main/java/org/springframework/cloud/fn/consumer/mqtt/MqttConsumerConfiguration.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2017-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.mqtt; - -import java.util.function.Consumer; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.cloud.fn.common.mqtt.MqttConfiguration; -import org.springframework.cloud.fn.common.mqtt.MqttProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.integration.mqtt.core.MqttPahoClientFactory; -import org.springframework.integration.mqtt.outbound.MqttPahoMessageHandler; -import org.springframework.integration.mqtt.support.DefaultPahoMessageConverter; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHandler; - -/** - * A consumer that sends data to Mqtt. - * - * @author Janne Valkealahti - * - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties({ MqttProperties.class, MqttConsumerProperties.class }) -@Import(MqttConfiguration.class) -public class MqttConsumerConfiguration { - - @Autowired - private MqttConsumerProperties properties; - - @Autowired - private MqttPahoClientFactory mqttClientFactory; - - @Autowired - private BeanFactory beanFactory; - - @Bean - public Consumer> mqttConsumer(MessageHandler mqttOutbound) { - return mqttOutbound::handleMessage; - } - - @Bean - public MessageHandler mqttOutbound( - @Nullable ComponentCustomizer mqttMessageHandlerCustomizer) { - - MqttPahoMessageHandler messageHandler = new MqttPahoMessageHandler(properties.getClientId(), mqttClientFactory); - messageHandler.setAsync(properties.isAsync()); - messageHandler.setDefaultTopic(properties.getTopic()); - messageHandler.setConverter(pahoMessageConverter()); - if (mqttMessageHandlerCustomizer != null) { - mqttMessageHandlerCustomizer.customize(messageHandler); - } - return messageHandler; - } - - public DefaultPahoMessageConverter pahoMessageConverter() { - DefaultPahoMessageConverter converter = new DefaultPahoMessageConverter(properties.getQos(), - properties.isRetained(), properties.getCharset()); - converter.setBeanFactory(beanFactory); - return converter; - } - -} diff --git a/functions/consumer/mqtt-consumer/src/main/java/org/springframework/cloud/fn/consumer/mqtt/MqttConsumerProperties.java b/functions/consumer/mqtt-consumer/src/main/java/org/springframework/cloud/fn/consumer/mqtt/MqttConsumerProperties.java deleted file mode 100644 index 439bd89e7..000000000 --- a/functions/consumer/mqtt-consumer/src/main/java/org/springframework/cloud/fn/consumer/mqtt/MqttConsumerProperties.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2017-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.mqtt; - -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.Size; -import org.hibernate.validator.constraints.Range; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * Properties for the Mqtt Consumer. - * - * @author Janne Valkealahti - * - */ -@Validated -@ConfigurationProperties("mqtt.consumer") -public class MqttConsumerProperties { - - /** - * identifies the client. - */ - private String clientId = "stream.client.id.sink"; - - /** - * the topic to which the sink will publish. - */ - private String topic = "stream.mqtt"; - - /** - * the quality of service to use. - */ - private int qos = 1; - - /** - * whether to set the 'retained' flag. - */ - private boolean retained = false; - - /** - * the charset used to convert a String payload to byte[]. - */ - private String charset = "UTF-8"; - - /** - * whether or not to use async sends. - */ - private boolean async = false; - - @Range(min = 0, max = 2) - public int getQos() { - return this.qos; - } - - public boolean isRetained() { - return this.retained; - } - - public void setQos(int qos) { - this.qos = qos; - } - - public void setRetained(boolean retained) { - this.retained = retained; - } - - @NotBlank - @Size(min = 1, max = 23) - public String getClientId() { - return this.clientId; - } - - public void setClientId(String clientId) { - this.clientId = clientId; - } - - @NotBlank - public String getTopic() { - return this.topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public String getCharset() { - return this.charset; - } - - public void setCharset(String charset) { - this.charset = charset; - } - - public boolean isAsync() { - return this.async; - } - - public void setAsync(boolean async) { - this.async = async; - } -} diff --git a/functions/consumer/mqtt-consumer/src/test/java/org/springframework/cloud/fn/consumer/mqtt/MqttConsumerTests.java b/functions/consumer/mqtt-consumer/src/test/java/org/springframework/cloud/fn/consumer/mqtt/MqttConsumerTests.java deleted file mode 100644 index c572a0452..000000000 --- a/functions/consumer/mqtt-consumer/src/test/java/org/springframework/cloud/fn/consumer/mqtt/MqttConsumerTests.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2017-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.mqtt; - -import java.time.Duration; -import java.util.Properties; -import java.util.function.Consumer; - -import org.eclipse.paho.client.mqttv3.MqttConnectOptions; -import org.eclipse.paho.client.mqttv3.internal.security.SSLSocketFactoryFactory; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.GenericContainer; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.integration.channel.QueueChannel; -import org.springframework.integration.mqtt.core.MqttPahoClientFactory; -import org.springframework.integration.mqtt.inbound.MqttPahoMessageDrivenChannelAdapter; -import org.springframework.integration.mqtt.support.DefaultPahoMessageConverter; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - -@SpringBootTest(properties = { - "mqtt.consumer.topic=test", - "mqtt.ssl-properties.com.ibm.ssl.protocol=TLS", - "mqtt.ssl-properties.com.ibm.ssl.keyStoreType=TEST" }) -@DirtiesContext -@Tag("integration") -public class MqttConsumerTests { - - static { - GenericContainer mosquitto = - new GenericContainer<>("eclipse-mosquitto:2.0.13") - .withCommand("mosquitto -c /mosquitto-no-auth.conf") - .withReuse(true) - .withExposedPorts(1883) - .withStartupTimeout(Duration.ofSeconds(120)) - .withStartupAttempts(3); - mosquitto.start(); - final Integer mappedPort = mosquitto.getMappedPort(1883); - System.setProperty("mqtt.url", "tcp://localhost:" + mappedPort); - } - - @Autowired - private MqttPahoMessageDrivenChannelAdapter mqttPahoMessageDrivenChannelAdapter; - - @Autowired - private Consumer> mqttConsumer; - - @Autowired - protected QueueChannel queue; - - @AfterAll - public static void cleanup() { - System.clearProperty("mqtt.url"); - } - - @Test - public void testMqttConsumer() { - MqttConnectOptions connectionInfo = this.mqttPahoMessageDrivenChannelAdapter.getConnectionInfo(); - Properties sslProperties = connectionInfo.getSSLProperties(); - assertThat(sslProperties) - .containsEntry(SSLSocketFactoryFactory.SSLPROTOCOL, SSLSocketFactoryFactory.DEFAULT_PROTOCOL) - .containsEntry(SSLSocketFactoryFactory.KEYSTORETYPE, "TEST"); - - this.mqttConsumer.accept(MessageBuilder.withPayload("hello").build()); - Message in = this.queue.receive(10000); - assertThat(in).isNotNull(); - assertThat(in.getPayload()).isEqualTo("hello"); - } - - @SpringBootApplication - static class MqttConsumerTestApplication { - - @Autowired - private MqttPahoClientFactory mqttClientFactory; - - @Bean - public MqttPahoMessageDrivenChannelAdapter mqttInbound(BeanFactory beanFactory) { - MqttPahoMessageDrivenChannelAdapter adapter = new MqttPahoMessageDrivenChannelAdapter("test", - mqttClientFactory, "test"); - adapter.setQos(0); - adapter.setConverter(pahoMessageConverter(beanFactory)); - adapter.setOutputChannelName("queue"); - return adapter; - } - - public DefaultPahoMessageConverter pahoMessageConverter(BeanFactory beanFactory) { - DefaultPahoMessageConverter converter = new DefaultPahoMessageConverter(1, true, "UTF-8"); - converter.setPayloadAsBytes(false); - converter.setBeanFactory(beanFactory); - return converter; - } - - @Bean - public QueueChannel queue() { - return new QueueChannel(); - } - - } - -} diff --git a/functions/consumer/mqtt-consumer/src/test/resources/logback.xml b/functions/consumer/mqtt-consumer/src/test/resources/logback.xml deleted file mode 100644 index 984967b8a..000000000 --- a/functions/consumer/mqtt-consumer/src/test/resources/logback.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - diff --git a/functions/consumer/pom.xml b/functions/consumer/pom.xml deleted file mode 100644 index bd58ca318..000000000 --- a/functions/consumer/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - 4.0.0 - - org.springframework.cloud.fn - java-functions-consumer - 5.0.0-SNAPSHOT - java-functions-consumer - Java Functions Consumer - pom - - - - cassandra-consumer - elasticsearch-consumer - analytics-consumer - file-consumer - ftp-consumer - jdbc-consumer - kafka-publisher - log-consumer - mongodb-consumer - mqtt-consumer - rabbit-consumer - redis-consumer - sftp-consumer - tcp-consumer - websocket-consumer - s3-consumer - twitter-consumer - wavefront-consumer - rsocket-consumer - xmpp-consumer - zeromq-consumer - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.0.0 - - true - - - - - diff --git a/functions/consumer/rabbit-consumer/README.adoc b/functions/consumer/rabbit-consumer/README.adoc deleted file mode 100644 index c0531b362..000000000 --- a/functions/consumer/rabbit-consumer/README.adoc +++ /dev/null @@ -1,26 +0,0 @@ -# RabbitMQ Consumer - -A consumer that allows you to send messages to RabbitMQ. - -## Beans for injection - -You can import `RabbitConsumerConfiguration` in the application and then inject the following bean. - -`Function, Object> rabbitConsumer` - -You can use `rabbitConsumer` as a qualifier when injecting. - -## Configuration Options - -All configuration properties are prefixed with `rabbit`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerProperties.java[RabbitConsumerProperties]. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `AmqpOutboundChannelAdapterSpec` configuration used by the `rabbitConsumer`. - -## Tests - - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/rabbit-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a RabbitMQ sink. diff --git a/functions/consumer/rabbit-consumer/pom.xml b/functions/consumer/rabbit-consumer/pom.xml deleted file mode 100644 index 900612a05..000000000 --- a/functions/consumer/rabbit-consumer/pom.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - rabbit-consumer - rabbit-consumer - Rabbit consumer - - - - org.springframework.integration - spring-integration-amqp - - - org.springframework.boot - spring-boot-starter-amqp - - - org.springframework.boot - spring-boot-starter-web - - - - diff --git a/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerConfiguration.java b/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerConfiguration.java deleted file mode 100644 index ad92019c1..000000000 --- a/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerConfiguration.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright 2019-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.rabbit; - -import java.util.function.Function; - -import com.rabbitmq.client.impl.CredentialsProvider; -import com.rabbitmq.client.impl.CredentialsRefreshService; - -import org.springframework.amqp.core.MessageDeliveryMode; -import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; -import org.springframework.amqp.rabbit.connection.ConnectionFactory; -import org.springframework.amqp.rabbit.connection.RabbitConnectionFactoryBean; -import org.springframework.amqp.rabbit.core.RabbitTemplate; -import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter; -import org.springframework.amqp.support.converter.MessageConverter; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.amqp.CachingConnectionFactoryConfigurer; -import org.springframework.boot.autoconfigure.amqp.ConnectionFactoryCustomizer; -import org.springframework.boot.autoconfigure.amqp.RabbitConnectionFactoryBeanConfigurer; -import org.springframework.boot.autoconfigure.amqp.RabbitProperties; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.io.ResourceLoader; -import org.springframework.expression.Expression; -import org.springframework.integration.amqp.dsl.Amqp; -import org.springframework.integration.amqp.dsl.AmqpOutboundChannelAdapterSpec; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHandler; - -/** - * A configuration for RabbitMQ Consumer function. Uses a - * {@link AmqpOutboundChannelAdapterSpec} to save payload contents to RabbitMQ. - * - * @author Soby Chako - * @author Nicolas Labrot - * @author Chris Bono - */ -@EnableConfigurationProperties(RabbitConsumerProperties.class) -@Configuration -public class RabbitConsumerConfiguration implements DisposableBean { - - @Autowired - private RabbitProperties bootProperties; - - @Autowired - private ResourceLoader resourceLoader; - - @Autowired - private ObjectProvider credentialsProvider; - - @Autowired - private ObjectProvider credentialsRefreshService; - - @Autowired - private ObjectProvider connectionFactoryCustomizers; - - @Autowired - private RabbitConsumerProperties properties; - - @Value("#{${rabbit.converterBeanName:null}}") - private MessageConverter messageConverter; - - private CachingConnectionFactory ownConnectionFactory; - - @Bean - public Function, Object> rabbitConsumer(@Qualifier("amqpChannelAdapter") MessageHandler messageHandler) { - return o -> { - messageHandler.handleMessage(o); - return ""; - }; - } - - @Bean - public AmqpOutboundChannelAdapterSpec amqpChannelAdapter(ConnectionFactory rabbitConnectionFactory, - @Nullable ComponentCustomizer amqpOutboundChannelAdapterSpecCustomizer) - throws Exception { - - AmqpOutboundChannelAdapterSpec handler = Amqp - .outboundAdapter(rabbitTemplate(this.properties.isOwnConnection() - ? buildLocalConnectionFactory() : rabbitConnectionFactory)) - .mappedRequestHeaders(properties.getMappedRequestHeaders()) - .defaultDeliveryMode(properties.getPersistentDeliveryMode() - ? MessageDeliveryMode.PERSISTENT - : MessageDeliveryMode.NON_PERSISTENT) - .headersMappedLast(this.properties.isHeadersMappedLast()); - - Expression exchangeExpression = this.properties.getExchangeExpression(); - if (exchangeExpression != null) { - handler.exchangeNameExpression(exchangeExpression); - } - else { - handler.exchangeName(this.properties.getExchange()); - } - - Expression routingKeyExpression = this.properties.getRoutingKeyExpression(); - if (routingKeyExpression != null) { - handler.routingKeyExpression(routingKeyExpression); - } - else { - handler.routingKey(this.properties.getRoutingKey()); - } - - if (amqpOutboundChannelAdapterSpecCustomizer != null) { - amqpOutboundChannelAdapterSpecCustomizer.customize(handler); - } - - return handler; - } - - @Bean - public RabbitTemplate rabbitTemplate(ConnectionFactory rabbitConnectionFactory) { - RabbitTemplate rabbitTemplate = new RabbitTemplate(rabbitConnectionFactory); - if (this.messageConverter != null) { - rabbitTemplate.setMessageConverter(this.messageConverter); - } - return rabbitTemplate; - } - - - @Bean - @ConditionalOnProperty(name = "rabbit.converterBeanName", - havingValue = RabbitConsumerProperties.JSON_CONVERTER) - public Jackson2JsonMessageConverter jsonConverter() { - return new Jackson2JsonMessageConverter(); - } - - @Override - public void destroy() { - if (this.ownConnectionFactory != null) { - this.ownConnectionFactory.destroy(); - } - } - - private ConnectionFactory buildLocalConnectionFactory() throws Exception { - this.ownConnectionFactory = rabbitConnectionFactory(this.bootProperties, this.resourceLoader, - this.credentialsProvider, this.credentialsRefreshService, this.connectionFactoryCustomizers); - return this.ownConnectionFactory; - } - - private CachingConnectionFactory rabbitConnectionFactory(RabbitProperties properties, ResourceLoader resourceLoader, - ObjectProvider credentialsProvider, - ObjectProvider credentialsRefreshService, - ObjectProvider connectionFactoryCustomizers) throws Exception { - - /* NOTE: This is based on RabbitAutoConfiguration.RabbitConnectionFactoryCreator - * https://github.com/spring-projects/spring-boot/blob/c820ad01a108d419d8548265b8a34ed7c5591f7c/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/amqp/RabbitAutoConfiguration.java#L95 - * [UPGRADE_CONSIDERATION] this should stay somewhat in sync w/ the functionality provided by its original source. - */ - RabbitConnectionFactoryBean connectionFactoryBean = new RabbitConnectionFactoryBean(); - RabbitConnectionFactoryBeanConfigurer connectionFactoryBeanConfigurer = - new RabbitConnectionFactoryBeanConfigurer(resourceLoader, properties); - connectionFactoryBeanConfigurer.setCredentialsProvider(credentialsProvider.getIfUnique()); - connectionFactoryBeanConfigurer.setCredentialsRefreshService(credentialsRefreshService.getIfUnique()); - connectionFactoryBeanConfigurer.configure(connectionFactoryBean); - connectionFactoryBean.afterPropertiesSet(); - - com.rabbitmq.client.ConnectionFactory connectionFactory = connectionFactoryBean.getObject(); - connectionFactoryCustomizers.orderedStream() - .forEach((customizer) -> customizer.customize(connectionFactory)); - - CachingConnectionFactory cachingConnectionFactory = new CachingConnectionFactory(connectionFactory); - CachingConnectionFactoryConfigurer cachingConnectionFactoryConfigurer = - new CachingConnectionFactoryConfigurer(properties); - cachingConnectionFactoryConfigurer.setConnectionNameStrategy(cf -> "rabbit.sink.own.connection"); - cachingConnectionFactoryConfigurer.configure(cachingConnectionFactory); - cachingConnectionFactory.afterPropertiesSet(); - - return cachingConnectionFactory; - } - -} diff --git a/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerProperties.java b/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerProperties.java deleted file mode 100644 index 914fdc838..000000000 --- a/functions/consumer/rabbit-consumer/src/main/java/org/springframework/cloud/fn/consumer/rabbit/RabbitConsumerProperties.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright 2019-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.rabbit; - -import jakarta.validation.constraints.AssertTrue; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.validation.annotation.Validated; - -@ConfigurationProperties("rabbit") -@Validated -public class RabbitConsumerProperties { - - /** - * JSON Converter. - */ - public static final String JSON_CONVERTER = "jsonConverter"; - - /** - * Exchange name - overridden by exchangeNameExpression, if supplied. - */ - private String exchange = ""; - - /** - * A SpEL expression that evaluates to an exchange name. - */ - private Expression exchangeExpression; - - /** - * Routing key - overridden by routingKeyExpression, if supplied. - */ - private String routingKey; - - /** - * A SpEL expression that evaluates to a routing key. - */ - private Expression routingKeyExpression; - - /** - * Default delivery mode when 'amqp_deliveryMode' header is not present, - * true for PERSISTENT. - */ - private boolean persistentDeliveryMode; - - /** - * Headers that will be mapped. - */ - private String[] mappedRequestHeaders = { "*" }; - - /** - * The bean name for a custom message converter; if omitted, a SimpleMessageConverter is used. - * If 'jsonConverter', a Jackson2JsonMessageConverter bean will be created for you. - */ - private String converterBeanName; - - /** - * When true, use a separate connection based on the boot properties. - */ - private boolean ownConnection; - - /** - * When mapping headers for the outbound message, determine whether the headers are - * mapped before the message is converted, or afterwards. - */ - private boolean headersMappedLast = true; - - public String getExchange() { - return this.exchange; - } - - public void setExchange(String exchange) { - this.exchange = exchange; - } - - public Expression getExchangeExpression() { - return this.exchangeExpression; - } - - public void setExchangeExpression(Expression exchangeExpression) { - this.exchangeExpression = exchangeExpression; - } - - public String getRoutingKey() { - return this.routingKey; - } - - public void setRoutingKey(String routingKey) { - this.routingKey = routingKey; - } - - public Expression getRoutingKeyExpression() { - return this.routingKeyExpression; - } - - public void setRoutingKeyExpression(Expression routingKeyExpression) { - this.routingKeyExpression = routingKeyExpression; - } - - public boolean getPersistentDeliveryMode() { - return this.persistentDeliveryMode; - } - - public void setPersistentDeliveryMode(boolean persistentDeliveryMode) { - this.persistentDeliveryMode = persistentDeliveryMode; - } - - public String[] getMappedRequestHeaders() { - return this.mappedRequestHeaders; - } - - public void setMappedRequestHeaders(String[] mappedRequestHeaders) { - this.mappedRequestHeaders = mappedRequestHeaders; - } - - public String getConverterBeanName() { - return this.converterBeanName; - } - - public void setConverterBeanName(String converterBeanName) { - this.converterBeanName = converterBeanName; - } - - @AssertTrue(message = "routingKey or routingKeyExpression is required") - public boolean isRoutingKeyProvided() { - return this.routingKey != null || this.routingKeyExpression != null; - } - - public boolean isOwnConnection() { - return this.ownConnection; - } - - public void setOwnConnection(boolean ownConnection) { - this.ownConnection = ownConnection; - } - - public boolean isHeadersMappedLast() { - return this.headersMappedLast; - } - - public void setHeadersMappedLast(boolean headersMappedLast) { - this.headersMappedLast = headersMappedLast; - } -} diff --git a/functions/consumer/redis-consumer/README.adoc b/functions/consumer/redis-consumer/README.adoc deleted file mode 100644 index 4d3e3355f..000000000 --- a/functions/consumer/redis-consumer/README.adoc +++ /dev/null @@ -1,25 +0,0 @@ -# Redis Consumer - -A consumer that allows you to write incoming messages into Redis. - -## Beans for injection - -You can import `RedisConsumerConfiguration` in the application and then inject the following bean. - -`Consumer> redisConsumer` - -You can use `redisConsumer` as a qualifier when injecting. - -## Configuration Options - -All configuration properties are prefixed with `redis.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerProperties.java[RedisConsumerProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/redis[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/redis-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a Redis sink. \ No newline at end of file diff --git a/functions/consumer/redis-consumer/pom.xml b/functions/consumer/redis-consumer/pom.xml deleted file mode 100644 index ea448835f..000000000 --- a/functions/consumer/redis-consumer/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - redis-consumer - redis-consumer - Redis Consumer - - - - org.springframework.boot - spring-boot-starter-data-redis - - - org.springframework.integration - spring-integration-redis - - - org.springframework.cloud - spring-cloud-starter-bootstrap - ${spring-cloud-starters.version} - test - - - org.springframework.cloud.fn - redis-common - ${project.version} - test-jar - test - - - - diff --git a/functions/consumer/redis-consumer/src/main/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerConfiguration.java b/functions/consumer/redis-consumer/src/main/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerConfiguration.java deleted file mode 100644 index a3ab67b00..000000000 --- a/functions/consumer/redis-consumer/src/main/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerConfiguration.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.redis; - -import java.util.function.Consumer; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.integration.redis.outbound.RedisPublishingMessageHandler; -import org.springframework.integration.redis.outbound.RedisQueueOutboundChannelAdapter; -import org.springframework.integration.redis.outbound.RedisStoreWritingMessageHandler; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHandler; - -/** - * @author Eric Bottard - * @author Mark Pollack - * @author Gary Russell - * @author Soby Chacko - * @author Artem Bilan - */ -@Configuration -@EnableConfigurationProperties(RedisConsumerProperties.class) -public class RedisConsumerConfiguration { - - @Bean - public Consumer> redisConsumer(MessageHandler redisConsumerMessageHandler) { - return redisConsumerMessageHandler::handleMessage; - } - - @Bean - public MessageHandler redisConsumerMessageHandler(RedisConnectionFactory redisConnectionFactory, - RedisConsumerProperties redisConsumerProperties) { - if (redisConsumerProperties.isKeyPresent()) { - RedisStoreWritingMessageHandler redisStoreWritingMessageHandler = new RedisStoreWritingMessageHandler( - redisConnectionFactory); - redisStoreWritingMessageHandler.setKeyExpression(redisConsumerProperties.keyExpression()); - return redisStoreWritingMessageHandler; - } - else if (redisConsumerProperties.isQueuePresent()) { - return new RedisQueueOutboundChannelAdapter(redisConsumerProperties.queueExpression(), - redisConnectionFactory); - } - else { // must be topic - RedisPublishingMessageHandler redisPublishingMessageHandler = new RedisPublishingMessageHandler( - redisConnectionFactory); - redisPublishingMessageHandler.setTopicExpression(redisConsumerProperties.topicExpression()); - return redisPublishingMessageHandler; - } - } - -} diff --git a/functions/consumer/redis-consumer/src/main/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerProperties.java b/functions/consumer/redis-consumer/src/main/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerProperties.java deleted file mode 100644 index 7a91e0aa4..000000000 --- a/functions/consumer/redis-consumer/src/main/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerProperties.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.redis; - -import java.util.Arrays; -import java.util.Collections; - -import jakarta.validation.constraints.AssertTrue; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.expression.ExpressionParser; -import org.springframework.expression.common.LiteralExpression; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.util.StringUtils; -import org.springframework.validation.annotation.Validated; - -/** - * Used to configure those Redis Sink module options that are not related to connecting to Redis. - * - * @author Eric Bottard - * @author Mark Pollack - * @author Artem Bilan - * @author Soby Chacko - */ -@ConfigurationProperties("redis.consumer") -@Validated -public class RedisConsumerProperties { - - private static final ExpressionParser EXPRESSION_PARSER = new SpelExpressionParser(); - - /** - * A SpEL expression to use for topic. - */ - private String topicExpression; - - /** - * A SpEL expression to use for queue. - */ - private String queueExpression; - - /** - * A SpEL expression to use for storing to a key. - */ - private String keyExpression; - - /** - * A literal key name to use when storing to a key. - */ - private String key; - - /** - * A literal queue name to use when storing in a queue. - */ - private String queue; - - /** - * A literal topic name to use when publishing to a topic. - */ - private String topic; - - public Expression keyExpression() { - return key != null ? new LiteralExpression(key) : EXPRESSION_PARSER.parseExpression(keyExpression); - } - - public Expression queueExpression() { - return queue != null ? new LiteralExpression(queue) : EXPRESSION_PARSER.parseExpression(queueExpression); - } - - public Expression topicExpression() { - return topic != null ? new LiteralExpression(topic) : EXPRESSION_PARSER.parseExpression(topicExpression); - } - - boolean isKeyPresent() { - return StringUtils.hasText(key) || keyExpression != null; - } - - boolean isQueuePresent() { - return StringUtils.hasText(queue) || queueExpression != null; - } - - boolean isTopicPresent() { - return StringUtils.hasText(topic) || topicExpression != null; - } - - public String getTopicExpression() { - return topicExpression; - } - - public void setTopicExpression(String topicExpression) { - this.topicExpression = topicExpression; - } - - public String getQueueExpression() { - return queueExpression; - } - - public void setQueueExpression(String queueExpression) { - this.queueExpression = queueExpression; - } - - public String getKeyExpression() { - return keyExpression; - } - - public void setKeyExpression(String keyExpression) { - this.keyExpression = keyExpression; - } - - public String getKey() { - return key; - } - - public void setKey(String key) { - this.key = key; - } - - public String getQueue() { - return queue; - } - - public void setQueue(String queue) { - this.queue = queue; - } - - public String getTopic() { - return topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - // The javabean property name is what will be reported in case of violation. Make it meaningful - @AssertTrue(message = "Exactly one of 'queue', 'queueExpression', 'key', 'keyExpression', " - + "'topic' and 'topicExpression' must be set") - public boolean isMutuallyExclusive() { - Object[] props = new Object[] { queue, queueExpression, key, keyExpression, topic, topicExpression }; - return (props.length - 1) == Collections.frequency(Arrays.asList(props), null); - } - -} diff --git a/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/AbstractRedisConsumerTests.java b/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/AbstractRedisConsumerTests.java deleted file mode 100644 index d7af01531..000000000 --- a/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/AbstractRedisConsumerTests.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.redis; - -import java.util.function.Consumer; - -import org.junit.jupiter.api.Tag; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.core.StringRedisTemplate; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; - -/** - * @author Soby Chacko - * @author Artem Bilan - */ -@SpringBootTest -@DirtiesContext -@Tag("integration") -public class AbstractRedisConsumerTests implements RedisTestContainerSupport { - - @DynamicPropertySource - static void redisProperties(DynamicPropertyRegistry registry) { - registry.add("spring.data.redis.url", RedisTestContainerSupport::getUri); - } - - @Autowired - Consumer> redisConsumer; - - @Autowired - RedisConnectionFactory redisConnectionFactory; - - @Autowired - StringRedisTemplate redisTemplate; - - @SpringBootApplication - static class RedisConsumerTestApplication { - - } -} diff --git a/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerKeyTests.java b/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerKeyTests.java deleted file mode 100644 index b6fc741d2..000000000 --- a/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerKeyTests.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.redis; - -import java.util.ArrayList; -import java.util.List; - -import org.junit.jupiter.api.Test; - -import org.springframework.data.redis.support.collections.DefaultRedisList; -import org.springframework.data.redis.support.collections.RedisList; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Mark Pollack - * @author Marius Bogoevici - * @author Gary Russell - */ -@TestPropertySource(properties = "redis.consumer.key = foo") -public class RedisConsumerKeyTests extends AbstractRedisConsumerTests { - - @Test - public void testWithKey() { - //Setup - String key = "foo"; - redisTemplate.delete(key); - - RedisList redisList = new DefaultRedisList<>(key, redisTemplate); - List list = new ArrayList<>(); - list.add("Manny"); - list.add("Moe"); - list.add("Jack"); - - //Execute - Message> message = new GenericMessage<>(list); - - redisConsumer.accept(message); - - //Assert - assertThat(redisList.size()).isEqualTo(3); - assertThat(redisList.get(0)).isEqualTo("Manny"); - assertThat(redisList.get(1)).isEqualTo("Moe"); - assertThat(redisList.get(2)).isEqualTo("Jack"); - - //Cleanup - redisTemplate.delete(key); - } -} diff --git a/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerQueueTests.java b/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerQueueTests.java deleted file mode 100644 index eecf8ca29..000000000 --- a/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerQueueTests.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.redis; - -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.Test; - -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Soby Chacko - */ -@TestPropertySource(properties = "redis.consumer.queue = test-queue") -public class RedisConsumerQueueTests extends AbstractRedisConsumerTests { - - @Test - public void testWithQueue() { - Message message = MessageBuilder.withPayload("hello").build(); - - redisConsumer.accept(message); - - Object result = redisTemplate.boundListOps("test-queue").rightPop(5000, TimeUnit.MILLISECONDS); - assertThat(result).isEqualTo("hello"); - } -} diff --git a/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerTopicTests.java b/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerTopicTests.java deleted file mode 100644 index d2af40dc9..000000000 --- a/functions/consumer/redis-consumer/src/test/java/org/springframework/cloud/fn/consumer/redis/RedisConsumerTopicTests.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.redis; - -import java.util.Collections; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.awaitility.Awaitility; -import org.junit.jupiter.api.Test; - -import org.springframework.data.redis.listener.ChannelTopic; -import org.springframework.data.redis.listener.RedisMessageListenerContainer; -import org.springframework.data.redis.listener.Topic; -import org.springframework.data.redis.listener.adapter.MessageListenerAdapter; -import org.springframework.data.redis.serializer.StringRedisSerializer; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Soby Chacko - * @author Corneil du Plessis - */ -@TestPropertySource(properties = "redis.consumer.topic = foo-topic") -public class RedisConsumerTopicTests extends AbstractRedisConsumerTests { - - @Test - public void testWithTopic() throws Exception { - - int numToTest = 10; - String topic = "foo-topic"; - final CountDownLatch latch = new CountDownLatch(numToTest); - - MessageListenerAdapter listener = new MessageListenerAdapter(); - listener.setDelegate(new Listener(latch)); - listener.setSerializer(new StringRedisSerializer()); - listener.afterPropertiesSet(); - - RedisMessageListenerContainer container = new RedisMessageListenerContainer(); - container.setConnectionFactory(redisTemplate.getConnectionFactory()); - container.afterPropertiesSet(); - container.addMessageListener(listener, Collections.singletonList(new ChannelTopic(topic))); - container.start(); - - Awaitility.await().until(container::isListening); - - Message message = MessageBuilder.withPayload("hello").build(); - for (int i = 0; i < numToTest; i++) { - redisConsumer.accept(message); - } - - assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue(); - container.stop(); - } - - private static class Listener { - - private final CountDownLatch latch; - - Listener(CountDownLatch latch) { - this.latch = latch; - } - - @SuppressWarnings("unused") - public void handleMessage(String s) { - this.latch.countDown(); - } - } -} diff --git a/functions/consumer/redis-consumer/src/test/resources/application.properties b/functions/consumer/redis-consumer/src/test/resources/application.properties deleted file mode 100644 index b7db25411..000000000 --- a/functions/consumer/redis-consumer/src/test/resources/application.properties +++ /dev/null @@ -1 +0,0 @@ -# Empty diff --git a/functions/consumer/rsocket-consumer/README.adoc b/functions/consumer/rsocket-consumer/README.adoc deleted file mode 100644 index 1aba1e70a..000000000 --- a/functions/consumer/rsocket-consumer/README.adoc +++ /dev/null @@ -1,26 +0,0 @@ -# RSocket Consumer - -A consumer that allows you to communicate to an RSocket route using its fire and forget strategy of execution. -The consumer uses the RSocket support from https://docs.spring.io/spring/docs/current/spring-framework-reference/web-reactive.html#rsocket-requester[Spring Framework]. - -## Beans for injection - -You can import `RSocketConsumerConfiguration` in the application and then inject the following bean. - -`Function>, Mono> rsocketConsumer` - -You can use `rsocketConsumer` as a qualifier when injecting. - -## Configuration Options - -All configuration properties are prefixed with `rsocket.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/rsocket/RsocketConsumerProperties.java[RsocketConsumerProperties]. - -## Examples - -See this link:src/test/java/org/springframework/cloud/fn/consumer/rsocket/RsocketConsumerTests.java[test suite] for learning more about this consumer. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/rsocket-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream based RSocket Sink application. \ No newline at end of file diff --git a/functions/consumer/rsocket-consumer/pom.xml b/functions/consumer/rsocket-consumer/pom.xml deleted file mode 100644 index 003f787c7..000000000 --- a/functions/consumer/rsocket-consumer/pom.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - rsocket-consumer - rsocket-consumer - - - - org.springframework.boot - spring-boot-starter-rsocket - - - - diff --git a/functions/consumer/rsocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/rsocket/RsocketConsumerConfiguration.java b/functions/consumer/rsocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/rsocket/RsocketConsumerConfiguration.java deleted file mode 100644 index 029c21efc..000000000 --- a/functions/consumer/rsocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/rsocket/RsocketConsumerConfiguration.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.rsocket; - -import java.util.function.Function; - -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.messaging.Message; -import org.springframework.messaging.rsocket.RSocketRequester; - -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties(RsocketConsumerProperties.class) -public class RsocketConsumerConfiguration { - - @Bean - public Function>, Mono> rsocketConsumer(RSocketRequester.Builder builder, - RsocketConsumerProperties rsocketConsumerProperties) { - RSocketRequester rSocketRequester = - rsocketConsumerProperties.getUri() != null - ? builder.websocket(rsocketConsumerProperties.getUri()) - : builder.tcp(rsocketConsumerProperties.getHost(), rsocketConsumerProperties.getPort()); - - String route = rsocketConsumerProperties.getRoute(); - - return input -> - input.flatMap(message -> - rSocketRequester.route(route) - .data(message.getPayload()) - .send()) - .ignoreElements(); - } - -} diff --git a/functions/consumer/rsocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/rsocket/RsocketConsumerProperties.java b/functions/consumer/rsocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/rsocket/RsocketConsumerProperties.java deleted file mode 100644 index f2f36fc71..000000000 --- a/functions/consumer/rsocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/rsocket/RsocketConsumerProperties.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.rsocket; - -import java.net.URI; - -import org.springframework.boot.context.properties.ConfigurationProperties; - -@ConfigurationProperties("rsocket.consumer") -public class RsocketConsumerProperties { - - /** - * RSocket host. - */ - private String host = "localhost"; - - /** - * RSocket port. - */ - private int port = 7000; - - /** - * URI that can be used for websocket based transport. - */ - private URI uri; - - /** - * Route used for RSocket. - */ - private String route; - - public String getHost() { - return this.host; - } - - public void setHost(String host) { - this.host = host; - } - - public int getPort() { - return this.port; - } - - public void setPort(int port) { - this.port = port; - } - - public String getRoute() { - return this.route; - } - - public void setRoute(String route) { - this.route = route; - } - - public URI getUri() { - return this.uri; - } - - public void setUri(URI uri) { - this.uri = uri; - } -} diff --git a/functions/consumer/rsocket-consumer/src/test/java/org/springframework/cloud/fn/consumer/rsocket/RsocketConsumerTests.java b/functions/consumer/rsocket-consumer/src/test/java/org/springframework/cloud/fn/consumer/rsocket/RsocketConsumerTests.java deleted file mode 100644 index 561f71342..000000000 --- a/functions/consumer/rsocket-consumer/src/test/java/org/springframework/cloud/fn/consumer/rsocket/RsocketConsumerTests.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.rsocket; - -import java.util.function.Function; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.core.publisher.ReplayProcessor; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.autoconfigure.rsocket.RSocketRequesterAutoConfiguration; -import org.springframework.boot.autoconfigure.rsocket.RSocketStrategiesAutoConfiguration; -import org.springframework.boot.rsocket.context.RSocketServerBootstrap; -import org.springframework.boot.rsocket.server.RSocketServer; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.context.ApplicationContext; -import org.springframework.messaging.Message; -import org.springframework.messaging.handler.annotation.MessageMapping; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.stereotype.Controller; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.util.ReflectionTestUtils; - -@SpringBootTest(properties = {"spring.rsocket.server.port=0"}) -@DirtiesContext -public class RsocketConsumerTests { - - private static ApplicationContextRunner applicationContextRunner = new ApplicationContextRunner() - .withUserConfiguration(RsocketConsumerConfiguration.class, RSocketRequesterAutoConfiguration.class, - RSocketStrategiesAutoConfiguration.class); - - @Autowired - ApplicationContext applicationContext; - - @Test - void testRsocketConsumer() { - - RSocketServerBootstrap serverBootstrap = applicationContext.getBean(RSocketServerBootstrap.class); - RSocketServer server = (RSocketServer) ReflectionTestUtils.getField(serverBootstrap, "server"); - final int port = server.address().getPort(); - - applicationContextRunner.withPropertyValues( - "rsocket.consumer.port=" + port, - "rsocket.consumer.route=test-route") - .run(context -> { - Function>, Mono> rsocketConsumer = context.getBean("rsocketConsumer", Function.class); - rsocketConsumer.apply(Flux.just(new GenericMessage<>("Hello RSocket"))) - .subscribe(); - - StepVerifier.create(RSocketserverApplication.fireForgetPayloads) - .expectNext("Hello RSocket") - .thenCancel() - .verify(); - }); - - } - - @EnableAutoConfiguration - @SpringBootConfiguration - @Controller - static class RSocketserverApplication { - static final ReplayProcessor fireForgetPayloads = ReplayProcessor.create(); - - @MessageMapping("test-route") - void someMethod(String payload) { - this.fireForgetPayloads.onNext(payload); - } - } -} - - diff --git a/functions/consumer/s3-consumer/README.adoc b/functions/consumer/s3-consumer/README.adoc deleted file mode 100644 index 073c5b594..000000000 --- a/functions/consumer/s3-consumer/README.adoc +++ /dev/null @@ -1,27 +0,0 @@ -= AWS S3 Consumer - -A consumer that allows you to upload files to AWS S3. -The consumer uses the AWS S3 support from Spring Integration and Spring Cloud AWS. - -== Beans for injection - -You can import `AwsS3ConsumerConfiguration` in the application and then inject the following bean. - -`Consumer> s3Consumer` - -You can use `s3Consumer` as a qualifier when injecting. - -== Configuration Options - -All configuration properties are prefixed with `s3.consumer`. -There are also properties that need to be used with the prefix `s3.common`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/s3/AwsS3ConsumerProperties.java[AwsS3ConsumerProperties] and `io.awspring.cloud.autoconfigure.s3.properties.S3Properties` & `io.awspring.cloud.autoconfigure.s3.properties.S3CrtClientProperties` from Spring Cloud AWS auto-configuration. - -== Examples - -See this link:src/test/java/org/springframework/cloud/fn/consumer/s3[test suite] for the various ways, this consumer is used. - -== Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/s3-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream based S3 Sink application. diff --git a/functions/consumer/s3-consumer/pom.xml b/functions/consumer/s3-consumer/pom.xml deleted file mode 100644 index 8fb0a38e2..000000000 --- a/functions/consumer/s3-consumer/pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - s3-consumer - s3-consumer - s3 consumer - - - - org.springframework.cloud.fn - aws-s3-common - ${project.version} - - - org.springframework.cloud.fn - file-common - ${project.version} - - - software.amazon.awssdk - s3-transfer-manager - - - org.springframework - spring-web - test - - - - diff --git a/functions/consumer/s3-consumer/src/main/java/org/springframework/cloud/fn/consumer/s3/AwsS3ConsumerConfiguration.java b/functions/consumer/s3-consumer/src/main/java/org/springframework/cloud/fn/consumer/s3/AwsS3ConsumerConfiguration.java deleted file mode 100644 index 8eceaab85..000000000 --- a/functions/consumer/s3-consumer/src/main/java/org/springframework/cloud/fn/consumer/s3/AwsS3ConsumerConfiguration.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.s3; - -import java.util.function.BiConsumer; -import java.util.function.Consumer; - -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.transfer.s3.S3TransferManager; -import software.amazon.awssdk.transfer.s3.progress.TransferListener; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.expression.EvaluationContext; -import org.springframework.expression.Expression; -import org.springframework.integration.aws.outbound.S3MessageHandler; -import org.springframework.integration.aws.support.AwsHeaders; -import org.springframework.integration.context.IntegrationContextUtils; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.expression.ValueExpression; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHandler; -import org.springframework.util.Assert; - -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties(AwsS3ConsumerProperties.class) -public class AwsS3ConsumerConfiguration { - - @Bean - public Consumer> s3Consumer(IntegrationFlow s3ConsumerFlow) { - return s3ConsumerFlow.getInputChannel()::send; - } - - @Bean - public IntegrationFlow s3ConsumerFlow(@Nullable TransferListener transferListener, - MessageHandler amazonS3MessageHandler) { - - return flow -> flow - .enrichHeaders(headers -> headers.header(AwsHeaders.TRANSFER_LISTENER, transferListener)) - .handle(amazonS3MessageHandler); - } - - @Bean - public MessageHandler amazonS3MessageHandler(S3TransferManager s3TransferManager, - AwsS3ConsumerProperties s3ConsumerProperties, - BeanFactory beanFactory, - @Nullable BiConsumer> uploadMetadataProvider) { - - Expression bucketExpression = s3ConsumerProperties.getBucketExpression(); - if (s3ConsumerProperties.getBucket() != null) { - bucketExpression = new ValueExpression<>(s3ConsumerProperties.getBucket()); - } - - S3MessageHandler s3MessageHandler = new S3MessageHandler(s3TransferManager, bucketExpression); - s3MessageHandler.setKeyExpression(s3ConsumerProperties.getKeyExpression()); - - Expression aclExpression; - - if (s3ConsumerProperties.getAcl() != null) { - aclExpression = new ValueExpression<>(s3ConsumerProperties.getAcl()); - } - else { - aclExpression = s3ConsumerProperties.getAclExpression(); - } - - BiConsumer> metadataProviderToUse = uploadMetadataProvider; - - if (aclExpression != null) { - EvaluationContext evaluationContext = IntegrationContextUtils.getEvaluationContext(beanFactory); - - metadataProviderToUse = - (builder, message) -> { - Object aclValue = aclExpression.getValue(evaluationContext, message); - Assert.notNull(aclValue, - () -> String.format("The expression '%s' for message '%s' returned null", - aclExpression, message)); - - builder.acl(aclValue.toString()); - - if (uploadMetadataProvider != null) { - uploadMetadataProvider.accept(builder, message); - } - }; - } - - if (metadataProviderToUse != null) { - s3MessageHandler.setUploadMetadataProvider(metadataProviderToUse); - } - return s3MessageHandler; - } - -} diff --git a/functions/consumer/s3-consumer/src/main/java/org/springframework/cloud/fn/consumer/s3/AwsS3ConsumerProperties.java b/functions/consumer/s3-consumer/src/main/java/org/springframework/cloud/fn/consumer/s3/AwsS3ConsumerProperties.java deleted file mode 100644 index 04dd3388e..000000000 --- a/functions/consumer/s3-consumer/src/main/java/org/springframework/cloud/fn/consumer/s3/AwsS3ConsumerProperties.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.s3; - -import jakarta.validation.constraints.AssertTrue; -import org.hibernate.validator.constraints.Length; -import software.amazon.awssdk.services.s3.model.ObjectCannedACL; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.validation.annotation.Validated; - -/** - * @author Artem Bilan - */ -@ConfigurationProperties("s3.consumer") -@Validated -public class AwsS3ConsumerProperties { - - /** - * AWS bucket for target file(s) to store. - */ - private String bucket; - - /** - * Expression to evaluate AWS bucket name. - */ - private Expression bucketExpression; - - /** - * Expression to evaluate S3 Object key. - */ - private Expression keyExpression; - - /** - * S3 Object access control list. - */ - private ObjectCannedACL acl; - - /** - * Expression to evaluate S3 Object access control list. - */ - private Expression aclExpression; - - @Length(min = 3) - public String getBucket() { - return this.bucket; - } - - public void setBucket(String bucket) { - this.bucket = bucket; - } - - public Expression getBucketExpression() { - return this.bucketExpression; - } - - public void setBucketExpression(Expression bucketExpression) { - this.bucketExpression = bucketExpression; - } - - public Expression getKeyExpression() { - return this.keyExpression; - } - - public void setKeyExpression(Expression keyExpression) { - this.keyExpression = keyExpression; - } - - public ObjectCannedACL getAcl() { - return this.acl; - } - - public void setAcl(ObjectCannedACL acl) { - this.acl = acl; - } - - public Expression getAclExpression() { - return this.aclExpression; - } - - public void setAclExpression(Expression aclExpression) { - this.aclExpression = aclExpression; - } - - @AssertTrue(message = "Exactly one of 'bucket' or 'bucketExpression' must be set") - public boolean isMutuallyExclusiveBucketAndBucketExpression() { - return (this.bucket != null && this.bucketExpression == null) || - (this.bucket == null && this.bucketExpression != null); - } - - @AssertTrue(message = "Only one of 'acl' or 'aclExpression' must be set") - public boolean isMutuallyExclusiveAclAndAclExpression() { - return this.acl == null || this.aclExpression == null; - } -} diff --git a/functions/consumer/s3-consumer/src/test/java/org/springframework/cloud/fn/consumer/s3/AbstractAwsS3ConsumerMockTests.java b/functions/consumer/s3-consumer/src/test/java/org/springframework/cloud/fn/consumer/s3/AbstractAwsS3ConsumerMockTests.java deleted file mode 100644 index c209f9411..000000000 --- a/functions/consumer/s3-consumer/src/test/java/org/springframework/cloud/fn/consumer/s3/AbstractAwsS3ConsumerMockTests.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.s3; - -import java.io.InputStream; -import java.nio.file.Path; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; -import java.util.function.BiConsumer; -import java.util.function.Consumer; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.io.TempDir; -import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.services.s3.model.PutObjectResponse; -import software.amazon.awssdk.transfer.s3.S3TransferManager; -import software.amazon.awssdk.transfer.s3.progress.TransferListener; - -import org.springframework.beans.DirectFieldAccessor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.http.MediaType; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.BDDMockito.willReturn; -import static org.mockito.Mockito.spy; - -@DirtiesContext -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { - "spring.cloud.aws.credentials.accessKey=" + AbstractAwsS3ConsumerMockTests.AWS_ACCESS_KEY, - "spring.cloud.aws.credentials.secretKey=" + AbstractAwsS3ConsumerMockTests.AWS_SECRET_KEY, - "spring.cloud.aws.region.static=" + AbstractAwsS3ConsumerMockTests.AWS_REGION, - "spring.cloud.aws.s3.endpoint=s3://foo", - "s3.consumer.bucket=" + AbstractAwsS3ConsumerMockTests.S3_BUCKET}) -public abstract class AbstractAwsS3ConsumerMockTests { - - protected static final String AWS_ACCESS_KEY = "test.accessKey"; - - protected static final String AWS_SECRET_KEY = "test.secretKey"; - - protected static final String AWS_REGION = "us-gov-west-1"; - - protected static final String S3_BUCKET = "S3_BUCKET"; - - @TempDir - protected static Path temporaryRemoteFolder; - - @Autowired - private S3AsyncClient amazonS3; - - @Autowired - protected S3TransferManager s3TransferManager; - - @Autowired - protected CountDownLatch transferCompletedLatch; - - @Autowired - protected Consumer> s3Consumer; - - @BeforeEach - public void setupTest() { - S3AsyncClient amazonS3 = spy(this.amazonS3); - - willReturn(CompletableFuture.completedFuture(PutObjectResponse.builder().build())) - .given(amazonS3) - .putObject(any(PutObjectRequest.class), any(AsyncRequestBody.class)); - - new DirectFieldAccessor(this.s3TransferManager).setPropertyValue("s3AsyncClient", amazonS3); - } - - @SpringBootApplication - public static class S3ConsumerTestApplication { - - @Bean - public CountDownLatch transferCompletedLatch() { - return new CountDownLatch(1); - } - - @Bean - public TransferListener transferListener() { - return new TransferListener() { - - - @Override - public void transferComplete(Context.TransferComplete context) { - transferCompletedLatch().countDown(); - } - - }; - } - - @Bean - public BiConsumer> uploadMetadataProvider() { - return (builder, message) -> { - if (message.getPayload() instanceof InputStream) { - builder.contentLength(1L) - .contentType(MediaType.APPLICATION_JSON_VALUE) - .contentDisposition("test.json"); - } - }; - } - - } - -} diff --git a/functions/consumer/s3-consumer/src/test/java/org/springframework/cloud/fn/consumer/s3/AmazonS3UploadFileTests.java b/functions/consumer/s3-consumer/src/test/java/org/springframework/cloud/fn/consumer/s3/AmazonS3UploadFileTests.java deleted file mode 100644 index a04a8a828..000000000 --- a/functions/consumer/s3-consumer/src/test/java/org/springframework/cloud/fn/consumer/s3/AmazonS3UploadFileTests.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.s3; - -import java.io.File; -import java.util.concurrent.TimeUnit; - -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import reactor.test.StepVerifier; -import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.model.ObjectCannedACL; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.utils.Md5Utils; - -import org.springframework.integration.test.util.TestUtils; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.verify; - -@TestPropertySource(properties = "s3.consumer.acl=PUBLIC_READ_WRITE") -public class AmazonS3UploadFileTests extends AbstractAwsS3ConsumerMockTests { - - @Test - public void test() throws Exception { - S3AsyncClient amazonS3Client = - TestUtils.getPropertyValue(this.s3TransferManager, "s3AsyncClient", S3AsyncClient.class); - - File file = new File(temporaryRemoteFolder.toFile(), "foo.mp3"); - file.createNewFile(); - Message message = MessageBuilder.withPayload(file) - .build(); - - this.s3Consumer.accept(message); - - ArgumentCaptor putObjectRequestArgumentCaptor = - ArgumentCaptor.forClass(PutObjectRequest.class); - ArgumentCaptor asyncRequestBodyArgumentCaptor = - ArgumentCaptor.forClass(AsyncRequestBody.class); - verify(amazonS3Client, atLeastOnce()) - .putObject(putObjectRequestArgumentCaptor.capture(), asyncRequestBodyArgumentCaptor.capture()); - - PutObjectRequest putObjectRequest = putObjectRequestArgumentCaptor.getValue(); - assertThat(putObjectRequest.bucket()).isEqualTo(S3_BUCKET); - assertThat(putObjectRequest.key()).isEqualTo("foo.mp3"); - assertThat(putObjectRequest.contentMD5()).isEqualTo(Md5Utils.md5AsBase64(file)); - assertThat(putObjectRequest.contentLength()).isEqualTo(0L); - assertThat(putObjectRequest.contentType()).isEqualTo("audio/mpeg"); - assertThat(putObjectRequest.acl()).isEqualTo(ObjectCannedACL.PUBLIC_READ_WRITE); - - AsyncRequestBody asyncRequestBody = asyncRequestBodyArgumentCaptor.getValue(); - StepVerifier.create(asyncRequestBody) - .assertNext(buffer -> assertThat(buffer.array()).isEmpty()) - .expectComplete() - .verify(); - - assertThat(this.transferCompletedLatch.await(10, TimeUnit.SECONDS)).isTrue(); - } - -} diff --git a/functions/consumer/s3-consumer/src/test/java/org/springframework/cloud/fn/consumer/s3/AmazonS3UploadInputStreamTests.java b/functions/consumer/s3-consumer/src/test/java/org/springframework/cloud/fn/consumer/s3/AmazonS3UploadInputStreamTests.java deleted file mode 100644 index 8c1799656..000000000 --- a/functions/consumer/s3-consumer/src/test/java/org/springframework/cloud/fn/consumer/s3/AmazonS3UploadInputStreamTests.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.s3; - -import java.io.InputStream; - -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import reactor.test.StepVerifier; -import software.amazon.awssdk.core.async.AsyncRequestBody; -import software.amazon.awssdk.services.s3.S3AsyncClient; -import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import software.amazon.awssdk.utils.Md5Utils; -import software.amazon.awssdk.utils.StringInputStream; - -import org.springframework.http.MediaType; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.Mockito.atLeastOnce; -import static org.mockito.Mockito.verify; - -@TestPropertySource(properties = "s3.consumer.key-expression=headers.key") -public class AmazonS3UploadInputStreamTests extends AbstractAwsS3ConsumerMockTests { - - @Test - public void test() throws Exception { - S3AsyncClient amazonS3Client = - TestUtils.getPropertyValue(this.s3TransferManager, "s3AsyncClient", S3AsyncClient.class); - - InputStream payload = new StringInputStream("a"); - Message message = MessageBuilder.withPayload(payload) - .setHeader("key", "myInputStream") - .build(); - - this.s3Consumer.accept(message); - - ArgumentCaptor putObjectRequestArgumentCaptor = - ArgumentCaptor.forClass(PutObjectRequest.class); - ArgumentCaptor asyncRequestBodyArgumentCaptor = - ArgumentCaptor.forClass(AsyncRequestBody.class); - verify(amazonS3Client, atLeastOnce()) - .putObject(putObjectRequestArgumentCaptor.capture(), asyncRequestBodyArgumentCaptor.capture()); - - PutObjectRequest putObjectRequest = putObjectRequestArgumentCaptor.getValue(); - assertThat(putObjectRequest.bucket()).isEqualTo(S3_BUCKET); - assertThat(putObjectRequest.key()).isEqualTo("myInputStream"); - assertThat(putObjectRequest.contentMD5()).isEqualTo(Md5Utils.md5AsBase64(payload)); - assertThat(putObjectRequest.contentLength()).isEqualTo(1L); - assertThat(putObjectRequest.contentType()).isEqualTo(MediaType.APPLICATION_JSON_VALUE); - assertThat(putObjectRequest.contentDisposition()).isEqualTo("test.json"); - - - AsyncRequestBody asyncRequestBody = asyncRequestBodyArgumentCaptor.getValue(); - StepVerifier.create(asyncRequestBody.map(buffer -> new String(buffer.array()))) - .expectNext("a") - .expectComplete() - .verify(); - } - -} diff --git a/functions/consumer/sftp-consumer/README.adoc b/functions/consumer/sftp-consumer/README.adoc deleted file mode 100644 index 0baa33fbf..000000000 --- a/functions/consumer/sftp-consumer/README.adoc +++ /dev/null @@ -1,27 +0,0 @@ -# SFTP Consumer - -A consumer that allows you to SFTP files. - -## Beans for injection - -You can import `SftpConsumerConfiguration` in the application and then inject the following bean. - -`Consumer> sftpConsumer` - -You can use `sftpConsumer` as a qualifier when injecting. - -## Configuration Options - -All configuration properties are prefixed with `sftp.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerProperties.java[SftpConsumerProperties]. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `SftpMessageHandlerSpec` configuration used by the `sftpConsumer`. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/sftp[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/sftp-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a SFTP sink. diff --git a/functions/consumer/sftp-consumer/pom.xml b/functions/consumer/sftp-consumer/pom.xml deleted file mode 100644 index 98bb73fd4..000000000 --- a/functions/consumer/sftp-consumer/pom.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - sftp-consumer - sftp-consumer - file consumer - - - - org.springframework.integration - spring-integration-sftp - - - org.springframework.cloud.fn - function-test-support - ${project.version} - test - - - - diff --git a/functions/consumer/sftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerConfiguration.java b/functions/consumer/sftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerConfiguration.java deleted file mode 100644 index 58a4018f3..000000000 --- a/functions/consumer/sftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerConfiguration.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2015-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.sftp; - -import java.util.function.Consumer; - -import org.apache.sshd.sftp.client.SftpClient; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.dsl.IntegrationFlowBuilder; -import org.springframework.integration.file.remote.session.SessionFactory; -import org.springframework.integration.sftp.dsl.Sftp; -import org.springframework.integration.sftp.dsl.SftpMessageHandlerSpec; -import org.springframework.integration.sftp.session.SftpRemoteFileTemplate; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; - -/** - * Configuration for SFTP Consumer. - * @author Soby Chacko - * @author Corneil du Plessis - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties(SftpConsumerProperties.class) -@Import(SftpConsumerSessionFactoryConfiguration.class) -public class SftpConsumerConfiguration { - - @Bean - public IntegrationFlow ftpOutboundFlow(SftpConsumerProperties properties, - SessionFactory ftpSessionFactory, - @Nullable ComponentCustomizer sftpMessageHandlerSpecCustomizer) { - - IntegrationFlowBuilder integrationFlowBuilder = - IntegrationFlow.from(MessageConsumer.class, (gateway) -> gateway.beanName("sftpConsumer")); - - SftpMessageHandlerSpec handlerSpec = - Sftp.outboundAdapter(new SftpRemoteFileTemplate(ftpSessionFactory), properties.getMode()) - .remoteDirectory(properties.getRemoteDir()) - .temporaryRemoteDirectory(properties.getTemporaryRemoteDir()) - .remoteFileSeparator(properties.getRemoteFileSeparator()) - .autoCreateDirectory(properties.isAutoCreateDir()) - .useTemporaryFileName(properties.isUseTemporaryFilename()) - .temporaryFileSuffix(properties.getTmpFileSuffix()); - if (properties.getFilenameExpression() != null) { - handlerSpec.fileNameExpression(properties.getFilenameExpression()); - } - - if (sftpMessageHandlerSpecCustomizer != null) { - sftpMessageHandlerSpecCustomizer.customize(handlerSpec); - } - - return integrationFlowBuilder - .handle(handlerSpec) - .get(); - } - - private interface MessageConsumer extends Consumer> { - - } - -} diff --git a/functions/consumer/sftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerProperties.java b/functions/consumer/sftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerProperties.java deleted file mode 100644 index 57ecdcd2a..000000000 --- a/functions/consumer/sftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerProperties.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.sftp; - -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import org.hibernate.validator.constraints.Range; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.core.io.Resource; -import org.springframework.expression.Expression; -import org.springframework.integration.file.support.FileExistsMode; -import org.springframework.validation.annotation.Validated; - -/** - * @author Gary Russell - * @author Artem Bilan - * @author Corneil du Plessis - */ -@ConfigurationProperties("sftp.consumer") -@Validated -public class SftpConsumerProperties { - - private final Factory factory = new Factory(); - - /** - * A temporary directory where the file will be written if 'isUseTemporaryFilename()' is true. - */ - private String temporaryRemoteDir = "/"; - - /** - * Whether to create the remote directory. - */ - private boolean autoCreateDir = true; - - /** - * Action to take if the remote file already exists. - */ - private FileExistsMode mode = FileExistsMode.REPLACE; - - /** - * Whether to write to a temporary file and rename. - */ - private boolean useTemporaryFilename = true; - - /** - * A SpEL expression to generate the remote file name. - */ - private String filenameExpression; - - /** - * The remote FTP directory. - */ - private String remoteDir = "/"; - - /** - * The suffix to use while the transfer is in progress. - */ - private String tmpFileSuffix = ".tmp"; - - /** - * The remote file separator. - */ - private String remoteFileSeparator = "/"; - - @NotBlank - public String getTemporaryRemoteDir() { - return this.temporaryRemoteDir; - } - - public void setTemporaryRemoteDir(String temporaryRemoteDir) { - this.temporaryRemoteDir = temporaryRemoteDir; - } - - public boolean isAutoCreateDir() { - return this.autoCreateDir; - } - - public void setAutoCreateDir(boolean autoCreateDir) { - this.autoCreateDir = autoCreateDir; - } - - @NotNull - public FileExistsMode getMode() { - return this.mode; - } - - public void setMode(FileExistsMode mode) { - this.mode = mode; - } - - public boolean isUseTemporaryFilename() { - return this.useTemporaryFilename; - } - - public void setUseTemporaryFilename(boolean useTemporaryFilename) { - this.useTemporaryFilename = useTemporaryFilename; - } - - public String getFilenameExpression() { - return this.filenameExpression; - } - - public void setFilenameExpression(String filenameExpression) { - this.filenameExpression = filenameExpression; - } - - @NotBlank - public String getRemoteDir() { - return remoteDir; - } - - public final void setRemoteDir(String remoteDir) { - this.remoteDir = remoteDir; - } - - @NotBlank - public String getTmpFileSuffix() { - return tmpFileSuffix; - } - - public void setTmpFileSuffix(String tmpFileSuffix) { - this.tmpFileSuffix = tmpFileSuffix; - } - - @NotBlank - public String getRemoteFileSeparator() { - return remoteFileSeparator; - } - - public void setRemoteFileSeparator(String remoteFileSeparator) { - this.remoteFileSeparator = remoteFileSeparator; - } - - public Factory getFactory() { - return this.factory; - } - - public static class Factory { - - /** - * The host name of the server. - */ - private String host = "localhost"; - - /** - * The username to use to connect to the server. - */ - - private String username; - - /** - * The password to use to connect to the server. - */ - private String password; - - /** - * Cache sessions. - */ - private Boolean cacheSessions; - - /** - * The port of the server. - */ - private int port = 22; - - /** - * Resource location of user's private key. - */ - private Resource privateKey; - - /** - * Passphrase for user's private key. - */ - private String passPhrase = ""; - - /** - * True to allow an unknown or changed key. - */ - private boolean allowUnknownKeys = false; - - /** - * A SpEL expression resolving to the location of the known hosts file. - */ - private Expression knownHostsExpression = null; - - - @NotBlank - public String getHost() { - return this.host; - } - - public void setHost(String host) { - this.host = host; - } - - @NotBlank - public String getUsername() { - return this.username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return this.password; - } - - public void setPassword(String password) { - this.password = password; - } - - public Boolean getCacheSessions() { - return this.cacheSessions; - } - - public void setCacheSessions(Boolean cacheSessions) { - this.cacheSessions = cacheSessions; - } - - @Range(min = 0, max = 65535) - public int getPort() { - return this.port; - } - - public void setPort(int port) { - this.port = port; - } - - public Resource getPrivateKey() { - return this.privateKey; - } - - public void setPrivateKey(Resource privateKey) { - this.privateKey = privateKey; - } - - public String getPassPhrase() { - return this.passPhrase; - } - - public void setPassPhrase(String passPhrase) { - this.passPhrase = passPhrase; - } - - public boolean isAllowUnknownKeys() { - return this.allowUnknownKeys; - } - - public void setAllowUnknownKeys(boolean allowUnknownKeys) { - this.allowUnknownKeys = allowUnknownKeys; - } - - public Expression getKnownHostsExpression() { - return this.knownHostsExpression; - } - - public void setKnownHostsExpression(Expression knownHosts) { - this.knownHostsExpression = knownHosts; - } - - } -} diff --git a/functions/consumer/sftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerSessionFactoryConfiguration.java b/functions/consumer/sftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerSessionFactoryConfiguration.java deleted file mode 100644 index 90b4995d1..000000000 --- a/functions/consumer/sftp-consumer/src/main/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerSessionFactoryConfiguration.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.sftp; - -import org.apache.sshd.sftp.client.SftpClient; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.core.io.Resource; -import org.springframework.integration.context.IntegrationContextUtils; -import org.springframework.integration.file.remote.session.CachingSessionFactory; -import org.springframework.integration.file.remote.session.SessionFactory; -import org.springframework.integration.sftp.session.DefaultSftpSessionFactory; - -/** - * Session factory configuration. - * - * @author Gary Russell - * @author Corneil du Plessis - * @author Chris Bono - * @author Artem Bilan - */ -public class SftpConsumerSessionFactoryConfiguration { - - @Bean - @ConditionalOnMissingBean - public SessionFactory sftpSessionFactory( - SftpConsumerProperties properties, - ApplicationContext applicationContext) { - - DefaultSftpSessionFactory sftpSessionFactory = new DefaultSftpSessionFactory(); - SftpConsumerProperties.Factory factory = properties.getFactory(); - sftpSessionFactory.setHost(factory.getHost()); - sftpSessionFactory.setPort(factory.getPort()); - sftpSessionFactory.setUser(factory.getUsername()); - sftpSessionFactory.setPassword(factory.getPassword()); - sftpSessionFactory.setPrivateKey(factory.getPrivateKey()); - sftpSessionFactory.setPrivateKeyPassphrase(factory.getPassPhrase()); - sftpSessionFactory.setAllowUnknownKeys(factory.isAllowUnknownKeys()); - if (factory.getKnownHostsExpression() != null) { - String knownHostsLocation = factory.getKnownHostsExpression() - .getValue(IntegrationContextUtils.getEvaluationContext(applicationContext), String.class); - Resource knownHostsResource = applicationContext.getResource(knownHostsLocation); - sftpSessionFactory.setKnownHostsResource(knownHostsResource); - } - if (factory.getCacheSessions() != null) { - return new CachingSessionFactory<>(sftpSessionFactory); - } - else { - return sftpSessionFactory; - } - } -} diff --git a/functions/consumer/sftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerPropertiesTests.java b/functions/consumer/sftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerPropertiesTests.java deleted file mode 100644 index c70912ace..000000000 --- a/functions/consumer/sftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerPropertiesTests.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.sftp; - -import java.io.File; - -import org.junit.jupiter.api.Test; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.boot.test.util.TestPropertyValues; -import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Lazy; -import org.springframework.core.convert.converter.Converter; -import org.springframework.expression.EvaluationContext; -import org.springframework.expression.Expression; -import org.springframework.expression.ParseException; -import org.springframework.expression.spel.standard.SpelExpression; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.integration.config.EnableIntegration; -import org.springframework.integration.config.IntegrationConverter; -import org.springframework.integration.context.IntegrationContextUtils; -import org.springframework.integration.file.remote.session.SessionFactory; -import org.springframework.integration.file.support.FileExistsMode; -import org.springframework.integration.test.util.TestUtils; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author David Turanski - * @author Gary Russell - * @author Artem Bilan - * @author Chris Schaefer - */ -public class SftpConsumerPropertiesTests { - - @Test - public void remoteDirCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - testPropertyValues(context, "sftp.consumer.remoteDir:/remote"); - context.register(Conf.class); - context.refresh(); - SftpConsumerProperties properties = context.getBean(SftpConsumerProperties.class); - assertThat(properties.getRemoteDir()).isEqualTo("/remote"); - context.close(); - } - - @Test - public void autoCreateDirCanBeDisabled() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - testPropertyValues(context, "sftp.consumer.autoCreateDir:false"); - context.register(Conf.class); - context.refresh(); - SftpConsumerProperties properties = context.getBean(SftpConsumerProperties.class); - assertThat(!properties.isAutoCreateDir()).isTrue(); - context.close(); - } - - @Test - public void tmpFileSuffixCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - testPropertyValues(context, "sftp.consumer.tmpFileSuffix:.foo"); - context.register(Conf.class); - context.refresh(); - SftpConsumerProperties properties = context.getBean(SftpConsumerProperties.class); - assertThat(properties.getTmpFileSuffix()).isEqualTo(".foo"); - context.close(); - } - - @Test - public void tmpFileRemoteDirCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - testPropertyValues(context, "sftp.consumer.temporaryRemoteDir:/foo"); - context.register(Conf.class); - context.refresh(); - SftpConsumerProperties properties = context.getBean(SftpConsumerProperties.class); - assertThat(properties.getTemporaryRemoteDir()).isEqualTo("/foo"); - context.close(); - } - - @Test - public void remoteFileSeparatorCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - testPropertyValues(context, "sftp.consumer.remoteFileSeparator:\\"); - context.register(Conf.class); - context.refresh(); - SftpConsumerProperties properties = context.getBean(SftpConsumerProperties.class); - assertThat(properties.getRemoteFileSeparator()).isEqualTo("\\"); - context.close(); - } - - @Test - public void useTemporaryFileNameCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - testPropertyValues(context, "sftp.consumer.useTemporaryFilename:false"); - context.register(Conf.class); - context.refresh(); - SftpConsumerProperties properties = context.getBean(SftpConsumerProperties.class); - assertThat(properties.isUseTemporaryFilename()).isFalse(); - context.close(); - } - - @Test - public void fileExistsModeCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - testPropertyValues(context, "sftp.consumer.mode:FAIL"); - context.register(Conf.class); - context.refresh(); - SftpConsumerProperties properties = context.getBean(SftpConsumerProperties.class); - assertThat(properties.getMode()).isEqualTo(FileExistsMode.FAIL); - context.close(); - } - - @Test - public void knownHostsExpression() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - testPropertyValues(context, - "sftp.consumer.factory.known-hosts-expression = @systemProperties[\"user.home\"] + \"/.ssh/known_hosts\"", - "sftp.consumer.factory.cache-sessions = true"); - context.register(Factory.class); - context.refresh(); - SessionFactory sessionFactory = context.getBean(SessionFactory.class); - assertThat(TestUtils.getPropertyValue(sessionFactory, "sessionFactory.knownHosts") - .toString().replaceAll(java.util.regex.Matcher.quoteReplacement(File.separator), "/") - .endsWith("/.ssh/known_hosts]")).isTrue(); - context.close(); - } - - private void testPropertyValues(ConfigurableApplicationContext context, String... props) { - TestPropertyValues.of("sftp.consumer.factory.username=foo").and(props).applyTo(context); - } - - @Configuration - @EnableConfigurationProperties(SftpConsumerProperties.class) - static class Conf { - - } - - @Configuration - @EnableConfigurationProperties(SftpConsumerProperties.class) - @EnableIntegration - @Import(SftpConsumerSessionFactoryConfiguration.class) - static class Factory { - - @Bean - @ConfigurationPropertiesBinding - @IntegrationConverter - public Converter spelConverter() { - return new SpelConverter(); - } - - /** - * TODO: This needs to be refactored into a generic place for any functions to use. - * - * A simple converter from String to Expression. - * - * @author Eric Bottard - */ - public static class SpelConverter implements Converter { - - private SpelExpressionParser parser = new SpelExpressionParser(); - - @Autowired - @Qualifier(IntegrationContextUtils.INTEGRATION_EVALUATION_CONTEXT_BEAN_NAME) - @Lazy - private EvaluationContext evaluationContext; - - @Override - public Expression convert(String source) { - try { - Expression expression = this.parser.parseExpression(source); - if (expression instanceof SpelExpression) { - ((SpelExpression) expression) - .setEvaluationContext(this.evaluationContext); - } - return expression; - } - catch (ParseException e) { - throw new IllegalArgumentException(String.format( - "Could not convert '%s' into a SpEL expression", source), e); - } - } - - } - - } - -} diff --git a/functions/consumer/sftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerTests.java b/functions/consumer/sftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerTests.java deleted file mode 100644 index 072257060..000000000 --- a/functions/consumer/sftp-consumer/src/test/java/org/springframework/cloud/fn/consumer/sftp/SftpConsumerTests.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.sftp; - -import java.io.File; -import java.util.function.Consumer; - -import org.junit.jupiter.api.Test; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.test.support.sftp.SftpTestSupport; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - -@DirtiesContext -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { - "sftp.consumer.remoteDir = sftpTarget", - "sftp.consumer.factory.username = foo", - "sftp.consumer.factory.password = foo", - "sftp.consumer.mode = FAIL", - "sftp.consumer.factory.port = ${sftp.factory.port}", - "sftp.consumer.filenameExpression = payload.name.toUpperCase()", - "sftp.consumer.factory.allowUnknownKeys = true" - }) -public class SftpConsumerTests extends SftpTestSupport { - - @Autowired - Consumer> sftpConsumer; - - @Test - public void sendFiles() { - for (int i = 1; i <= 2; i++) { - String pathname = "/localSource" + i + ".txt"; - String upperPathname = pathname.toUpperCase(); - new File(getTargetRemoteDirectory() + upperPathname).delete(); - assertThat(new File(getTargetRemoteDirectory() + upperPathname).exists()).isFalse(); - sftpConsumer.accept(new GenericMessage<>(new File(getSourceLocalDirectory() + pathname))); - File expected = new File(getTargetRemoteDirectory() + upperPathname); - assertThat(expected.exists()).isTrue(); - // verify the uppercase on a case-insensitive file system - File[] files = getTargetRemoteDirectory().listFiles(); - for (File file : files) { - assertThat(file.getName().startsWith("LOCALSOURCE")).isTrue(); - } - } - } - - @Test - public void serverRefreshed() { // noop test to test the dirs are refreshed properly - String pathname = "/LOCALSOURCE1.TXT"; - assertThat(getTargetRemoteDirectory().exists()).isTrue(); - assertThat(new File(getTargetRemoteDirectory() + pathname).exists()).isFalse(); - } - - @SpringBootApplication - static class SftpConsumerTestApplication { - } -} - diff --git a/functions/consumer/tcp-consumer/README.adoc b/functions/consumer/tcp-consumer/README.adoc deleted file mode 100644 index 4d8ff29e0..000000000 --- a/functions/consumer/tcp-consumer/README.adoc +++ /dev/null @@ -1,25 +0,0 @@ -# TCP Consumer - -A consumer that allows you to send TCP messages. - -## Beans for injection - -You can import `TcpConsumerConfiguration` in the application and then inject the following bean. - -`Consumer> tcpConsumer` - -You can use `tcpConsumer` as a qualifier when injecting. - -## Configuration Options - -All configuration properties are prefixed with `tcp.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/tcp/TCPConsumerProperties.java[TcpConsumerProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/tcp[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/tcp-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a TCP sink. \ No newline at end of file diff --git a/functions/consumer/tcp-consumer/pom.xml b/functions/consumer/tcp-consumer/pom.xml deleted file mode 100644 index 7d24ba654..000000000 --- a/functions/consumer/tcp-consumer/pom.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - tcp-consumer - tcp-consumer - tcp consumer - - - - org.springframework.cloud.fn - tcp-common - ${project.version} - - - - diff --git a/functions/consumer/tcp-consumer/src/main/java/org/springframework/cloud/fn/consumer/tcp/TcpConsumerConfiguration.java b/functions/consumer/tcp-consumer/src/main/java/org/springframework/cloud/fn/consumer/tcp/TcpConsumerConfiguration.java deleted file mode 100644 index 25caf7a8e..000000000 --- a/functions/consumer/tcp-consumer/src/main/java/org/springframework/cloud/fn/consumer/tcp/TcpConsumerConfiguration.java +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import java.util.function.Consumer; - -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.tcp.EncoderDecoderFactoryBean; -import org.springframework.cloud.fn.common.tcp.TcpConnectionFactoryProperties; -import org.springframework.context.SmartLifecycle; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.ip.config.TcpConnectionFactoryFactoryBean; -import org.springframework.integration.ip.tcp.TcpSendingMessageHandler; -import org.springframework.integration.ip.tcp.connection.AbstractConnectionFactory; -import org.springframework.integration.ip.tcp.connection.TcpMessageMapper; -import org.springframework.integration.ip.tcp.serializer.AbstractByteArraySerializer; -import org.springframework.messaging.Message; - -/** - * A consumer that sends data over TCP. - * - * @author Gary Russell - * @author Christian Tzolov - * @author Chris Bono - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties({TcpConsumerProperties.class, TcpConnectionFactoryProperties.class}) -public class TcpConsumerConfiguration { - - private TcpConsumerProperties properties; - private TcpConnectionFactoryProperties tcpConnectionProperties; - - public TcpConsumerConfiguration(TcpConsumerProperties properties, - TcpConnectionFactoryProperties tcpConnectionProperties) { - this.properties = properties; - this.tcpConnectionProperties = tcpConnectionProperties; - } - - @Bean - public Consumer> tcpConsumer(TcpSendingMessageHandlerSmartLifeCycle handler) { - return handler::handleMessage; - } - - @Bean - public TcpSendingMessageHandlerSmartLifeCycle handler(@Qualifier("tcpSinkConnectionFactory") AbstractConnectionFactory connectionFactory) { - TcpSendingMessageHandlerSmartLifeCycle tcpMessageHandler = new TcpSendingMessageHandlerSmartLifeCycle(); - tcpMessageHandler.setConnectionFactory(connectionFactory); - return tcpMessageHandler; - } - - @Bean - public TcpConnectionFactoryFactoryBean tcpSinkConnectionFactory( - @Qualifier("tcpSinkEncoder") AbstractByteArraySerializer encoder, - @Qualifier("tcpSinkMapper") TcpMessageMapper mapper) throws Exception { - TcpConnectionFactoryFactoryBean factoryBean = new TcpConnectionFactoryFactoryBean(); - factoryBean.setType("client"); - factoryBean.setHost(this.properties.getHost()); - factoryBean.setPort(this.tcpConnectionProperties.getPort()); - factoryBean.setUsingNio(this.tcpConnectionProperties.isNio()); - factoryBean.setUsingDirectBuffers(this.tcpConnectionProperties.isUseDirectBuffers()); - factoryBean.setLookupHost(this.tcpConnectionProperties.isReverseLookup()); - factoryBean.setSerializer(encoder); - factoryBean.setSoTimeout(this.tcpConnectionProperties.getSocketTimeout()); - factoryBean.setMapper(mapper); - factoryBean.setSingleUse(this.properties.isClose()); - return factoryBean; - } - - @Bean - public EncoderDecoderFactoryBean tcpSinkEncoder() { - return new EncoderDecoderFactoryBean(this.properties.getEncoder()); - } - - @Bean - public TcpMessageMapper tcpSinkMapper() { - TcpMessageMapper mapper = new TcpMessageMapper(); - mapper.setCharset(this.properties.getCharset()); - return mapper; - } - - static class TcpSendingMessageHandlerSmartLifeCycle extends TcpSendingMessageHandler implements SmartLifecycle { - - @Override - public boolean isAutoStartup() { - return true; - } - - @Override - public int getPhase() { - return Integer.MIN_VALUE; - } - } -} diff --git a/functions/consumer/tcp-consumer/src/main/java/org/springframework/cloud/fn/consumer/tcp/TcpConsumerProperties.java b/functions/consumer/tcp-consumer/src/main/java/org/springframework/cloud/fn/consumer/tcp/TcpConsumerProperties.java deleted file mode 100644 index 4ba112c2a..000000000 --- a/functions/consumer/tcp-consumer/src/main/java/org/springframework/cloud/fn/consumer/tcp/TcpConsumerProperties.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.cloud.fn.common.tcp.Encoding; -import org.springframework.validation.annotation.Validated; - -/** - * Properties for the TCP Consumer. - * - * @author Gary Russell - * @author Christian Tzolov - * - */ -@ConfigurationProperties("tcp.consumer") -@Validated -public class TcpConsumerProperties { - - /** - * The host to which this sink will connect. - */ - private String host; - - /** - * The encoder to use when sending messages. - */ - private Encoding encoder = Encoding.CRLF; - - /** - * The charset used when converting from bytes to String. - */ - private String charset = "UTF-8"; - - /** - * Whether to close the socket after each message. - */ - private boolean close; - - @NotNull - public String getHost() { - return host; - } - - public void setHost(String host) { - this.host = host; - } - - @NotNull - public Encoding getEncoder() { - return this.encoder; - } - - public void setEncoder(Encoding encoder) { - this.encoder = encoder; - } - - @NotNull - public String getCharset() { - return charset; - } - - public void setCharset(String charset) { - this.charset = charset; - } - - public boolean isClose() { - return close; - } - - public void setClose(boolean close) { - this.close = close; - } -} diff --git a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/AbstractTcpConsumerTests.java b/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/AbstractTcpConsumerTests.java deleted file mode 100644 index f1d92a922..000000000 --- a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/AbstractTcpConsumerTests.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import java.io.IOException; -import java.net.ServerSocket; -import java.net.Socket; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; - -import javax.net.ServerSocketFactory; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.integration.ip.tcp.connection.AbstractClientConnectionFactory; -import org.springframework.integration.ip.tcp.serializer.AbstractByteArraySerializer; -import org.springframework.integration.ip.tcp.serializer.ByteArrayCrLfSerializer; -import org.springframework.integration.ip.tcp.serializer.SoftEndOfStreamException; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Tests for TCP Consumer. - * - * @author Gary Russell - * @author Soby Chacko - */ -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { "tcp.consumer.host = localhost", "tcp.port = ${tcp.consumer.test.port}" }) -@DirtiesContext -public class AbstractTcpConsumerTests { - - private static TestTCPServer server; - - @Autowired - protected AbstractClientConnectionFactory connectionFactory; - - @Autowired - Consumer> tcpConsumer; - - @BeforeAll - public static void startup() { - server = new TestTCPServer(); - } - - @AfterAll - public static void shutDown() { - server.shutDown(); - } - - - /* - * Sends two messages and asserts they arrive as expected on the other side using - * the supplied decoder. - */ - protected void doTest(AbstractByteArraySerializer decoder) throws Exception { - server.setDecoder(decoder); - Message message = new GenericMessage<>("foo"); - tcpConsumer.accept(message); - String received = server.queue.poll(10, TimeUnit.SECONDS); - assertThat(received).isEqualTo("foo"); - - tcpConsumer.accept(message); - received = server.queue.poll(10, TimeUnit.SECONDS); - assertThat(received).isEqualTo("foo"); - } - - /** - * TCP server that uses the supplied {@link AbstractByteArraySerializer} - * to decode the input stream and put the resulting message in a queue. - * - */ - private static class TestTCPServer implements Runnable { - - private static final Log logger = LogFactory.getLog(TestTCPServer.class); - - private final ServerSocket serverSocket; - - private final ExecutorService executor; - - private volatile AbstractByteArraySerializer decoder; - - private final BlockingQueue queue = new LinkedBlockingQueue<>(); - - private volatile boolean stopped; - - TestTCPServer() { - ServerSocket serverSocket = null; - ExecutorService executor = null; - try { - serverSocket = ServerSocketFactory.getDefault().createServerSocket(0); - System.setProperty("tcp.consumer.test.port", Integer.toString(serverSocket.getLocalPort())); - executor = Executors.newSingleThreadExecutor(); - } - catch (IOException e) { - e.printStackTrace(); - } - this.serverSocket = serverSocket; - this.executor = executor; - this.decoder = new ByteArrayCrLfSerializer(); - executor.execute(this); - } - - private void setDecoder(AbstractByteArraySerializer decoder) { - this.decoder = decoder; - } - - @Override - public void run() { - while (true) { - Socket socket = null; - try { - logger.info("Server listening on " + this.serverSocket.getLocalPort()); - socket = this.serverSocket.accept(); - while (true) { - byte[] data = decoder.deserialize(socket.getInputStream()); - queue.offer(new String(data)); - } - } - catch (SoftEndOfStreamException e) { - // normal close - } - catch (IOException e) { - try { - if (socket != null) { - socket.close(); - } - } - catch (IOException e1) { - } - logger.error(e.getMessage()); - if (this.stopped) { - logger.info("Server stopped on " + this.serverSocket.getLocalPort()); - break; - } - } - } - } - - private void shutDown() { - try { - this.stopped = true; - this.serverSocket.close(); - this.executor.shutdownNow(); - } - catch (IOException e) { - } - } - } - - @SpringBootApplication - public static class TcpConsumerTestApplication { - - } -} diff --git a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/CRLFTests.java b/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/CRLFTests.java deleted file mode 100644 index 43dd6d42f..000000000 --- a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/CRLFTests.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.serializer.ByteArrayCrLfSerializer; - - -/** - * @author Gary Russell - */ -public class CRLFTests extends AbstractTcpConsumerTests { - - @Test - public void test() throws Exception { - doTest(new ByteArrayCrLfSerializer()); - } -} diff --git a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/L1Tests.java b/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/L1Tests.java deleted file mode 100644 index b55b9581d..000000000 --- a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/L1Tests.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.serializer.ByteArrayLengthHeaderSerializer; -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = {"tcp.consumer.encoder = L1"}) -public class L1Tests extends AbstractTcpConsumerTests { - - @Test - public void test() throws Exception { - doTest(new ByteArrayLengthHeaderSerializer(1)); - } -} diff --git a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/L2Tests.java b/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/L2Tests.java deleted file mode 100644 index b1454e787..000000000 --- a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/L2Tests.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.serializer.ByteArrayLengthHeaderSerializer; -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = {"tcp.consumer.encoder = L2"}) -public class L2Tests extends AbstractTcpConsumerTests { - - @Test - public void test() throws Exception { - doTest(new ByteArrayLengthHeaderSerializer(2)); - } -} diff --git a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/L4Tests.java b/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/L4Tests.java deleted file mode 100644 index a909d5b1b..000000000 --- a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/L4Tests.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.serializer.ByteArrayLengthHeaderSerializer; -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = {"tcp.consumer.encoder = L4"}) -public class L4Tests extends AbstractTcpConsumerTests { - - @Test - public void test() throws Exception { - doTest(new ByteArrayLengthHeaderSerializer(4)); - } -} diff --git a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/LFTests.java b/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/LFTests.java deleted file mode 100644 index f633f18e8..000000000 --- a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/LFTests.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.serializer.ByteArrayLfSerializer; -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = {"tcp.consumer.encoder = LF"}) -public class LFTests extends AbstractTcpConsumerTests { - - @Test - public void test() throws Exception { - doTest(new ByteArrayLfSerializer()); - } -} diff --git a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/NULLTests.java b/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/NULLTests.java deleted file mode 100644 index f22126bb6..000000000 --- a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/NULLTests.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.serializer.ByteArraySingleTerminatorSerializer; -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = {"tcp.consumer.encoder = NULL"}) -public class NULLTests extends AbstractTcpConsumerTests { - - @Test - public void test() throws Exception { - doTest(new ByteArraySingleTerminatorSerializer((byte) 0)); - } -} diff --git a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/NotNioTests.java b/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/NotNioTests.java deleted file mode 100644 index aee67b2cf..000000000 --- a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/NotNioTests.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.connection.TcpNetClientConnectionFactory; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = {"tcp.consumer.host = foo"}) -public class NotNioTests extends AbstractTcpConsumerTests { - - @Test - public void test() throws Exception { - assertThat(this.connectionFactory).isInstanceOf(TcpNetClientConnectionFactory.class); - assertThat(this.connectionFactory.getHost()).isEqualTo("foo"); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "lookupHost", Boolean.class)).isFalse(); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "soTimeout")).isEqualTo(120000); - } -} diff --git a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/PropertiesPopulatedTests.java b/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/PropertiesPopulatedTests.java deleted file mode 100644 index 19a92efe3..000000000 --- a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/PropertiesPopulatedTests.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.connection.TcpNioClientConnectionFactory; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = {"tcp.consumer.host = foo", "tcp.nio = true", "tcp.reverseLookup = true", - "tcp.useDirectBuffers = true", "tcp.socketTimeout = 123", "tcp.consumer.close = true", "tcp.consumer.charset = bar"}) -public class PropertiesPopulatedTests extends AbstractTcpConsumerTests { - - @Test - public void test() throws Exception { - assertThat(this.connectionFactory).isInstanceOf(TcpNioClientConnectionFactory.class); - assertThat(this.connectionFactory.getHost()).isEqualTo("foo"); - assertThat((TestUtils.getPropertyValue(this.connectionFactory, "lookupHost", Boolean.class))).isTrue(); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "usingDirectBuffers", Boolean.class)).isTrue(); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "soTimeout")).isEqualTo(123); - assertThat(this.connectionFactory.isSingleUse()).isTrue(); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "mapper.charset")).isEqualTo("bar"); - } -} diff --git a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/RAWTests.java b/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/RAWTests.java deleted file mode 100644 index c1a6efa59..000000000 --- a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/RAWTests.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.serializer.ByteArrayRawSerializer; -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = {"tcp.consumer.encoder = RAW", "tcp.consumer.close = true"}) -public class RAWTests extends AbstractTcpConsumerTests { - - @Test - public void test() throws Exception { - doTest(new ByteArrayRawSerializer()); - } -} diff --git a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/STXETXTests.java b/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/STXETXTests.java deleted file mode 100644 index 25cdddc19..000000000 --- a/functions/consumer/tcp-consumer/src/test/java/org/springframework/cloud/fn/consumer/tcp/STXETXTests.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.serializer.ByteArrayStxEtxSerializer; -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = { "tcp.consumer.encoder = STXETX" }) -public class STXETXTests extends AbstractTcpConsumerTests { - - @Test - public void test() throws Exception { - doTest(new ByteArrayStxEtxSerializer()); - } -} diff --git a/functions/consumer/twitter-consumer/README.adoc b/functions/consumer/twitter-consumer/README.adoc deleted file mode 100644 index 1f95e1845..000000000 --- a/functions/consumer/twitter-consumer/README.adoc +++ /dev/null @@ -1,110 +0,0 @@ -# Twitter Consumers - - -## 1. Twitter Status Update Consumer. - -Updates the authenticating user's current text (e.g Tweeting). - -NOTE: For each update attempt, the update text is compared with the authenticating user's recent Tweets. -Any attempt that would result in duplication will be blocked, resulting in a 403 error. -A user cannot submit the same text twice in a row. - -While not rate limited by the API, a user is limited in the number of Tweets they can create at a time. -The update limit for standard API is 300 in 3 hours windows. -If the number of updates posted by the user reaches the current allowed limit this method will return an HTTP 403 error. - -You can find details for the Update API here: https://developer.twitter.com/en/docs/tweets/post-and-engage/api-reference/post-statuses-update - - -### 1.1 Beans for injection - -You can import `TwitterUpdateConsumerConfiguration` in the application and then inject the following beans. - -- `Consumer updateStatus` - if you have an `StatusUpdate` instance you can use the `updateStatus` to apply it. - -- `Function, StatusUpdate> messageToStatusUpdateFunction` - function that converts a `Message` text into a `StatusUpdate` instance using the `TwitterUpdateConsumerProperties` properties. - -- `Consumer> twitterStatusUpdateConsumer` - composes `messageToStatusUpdateFunction` and `updateStatus` to update the twitter status from Message text. - -Note: the Message content is expected to be in text format. Consider using the `byteArrayTextToString` utility `Function`. - -You can use `twitterStatusUpdateConsumer` as a qualifier when injecting. - -### 1.2 Configuration Options - -All configuration properties are prefixed with `twitter.update`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/twitter/status/update/TwitterUpdateConsumerProperties.java[TwitterUpdateConsumerProperties]. - -The twitter function makes uses of link:../spel-function/README.adoc[SpEL function]. - -### 1.3 Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/twitter-update-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a Twitter Update sink. - -## 2. Twitter Message Consumer. - -Send Direct Messages to a specified user from the authenticating user. -Requires a JSON POST body and `Content-Type` header to be set to `application/json`. - -NOTE: When a message is received from a user you may send up to 5 messages in response within a 24 hour window. -Each message received resets the 24 hour window and the 5 allotted messages. -Sending a 6th message within a 24 hour window or sending a message outside of a 24 hour window will count towards rate-limiting. -This behavior only applies when using the POST direct_messages/events/new endpoint. - -SpEL expressions are used to compute the request parameters from the input message. - -### 2.1 Beans for injection - -You can import `TwitterMessageConsumerConfiguration` in the application and then inject the following bean. - -- `Consumer> sendDirectMessageConsumer` - -Note: the Message content is expected to be in text format. Consider using the `byteArrayTextToString` utility `Function`. - -You can use `twitterStatusUpdateConsumer` as a qualifier when injecting. - -### 2.2 Configuration Options - -All configuration properties are prefixed with `twitter.message.update`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/twitter/message/TwitterMessageConsumerProperties.java[TwitterMessageConsumerProperties]. - -The twitter function makes uses of link:../spel-function/README.adoc[SpEL function]. - -### 2.3 Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/twitter-message-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a Twitter Message sink. - -## 3. Twitter Friendship Consumer. - -Allows creating `follow`, `unfollow` and `update` relationships with specified `userId` or `screenName`. -The `twitter.friendships.sink.type` property allows to select the desired friendship operation. - -* https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/post-friendships-create[Friendships Create API] - Allows the authenticating user to follow (friend) the user specified in the ID parameter. -Actions taken in this method are asynchronous. -Changes will be eventually consistent. -* https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/post-friendships-update[Friendships Update API] - Enable or disable Retweets and device notifications from the specified user. -* https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/post-friendships-destroy[Friendships Destroy API] - Allows the authenticating user to unfollow the user specified in the ID parameter. - -SpEL expressions are used to compute the request parameters from the input message. -Every operation type has its own parameters. - -### 3.1 Beans for injection - -You can import `TwitterFriendshipsConsumerConfiguration` in the application and then inject the following bean. - -- `Consumer> friendshipConsumer` - -Note: the Message content is expected to be in text format. Consider using the `byteArrayTextToString` utility `Function`. - -You can use `friendshipConsumer` as a qualifier when injecting. - -### 3.2 Configuration Options - -All configuration properties are prefixed with `twitter.friendships.update`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/twitter/friendship/TwitterFriendshipsConsumerProperties.java[TwitterFriendshipsConsumerProperties]. - -The twitter function makes uses of link:../spel-function/README.adoc[SpEL function]. - diff --git a/functions/consumer/twitter-consumer/pom.xml b/functions/consumer/twitter-consumer/pom.xml deleted file mode 100644 index d379928da..000000000 --- a/functions/consumer/twitter-consumer/pom.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - twitter-consumer - twitter-consumer - twitter consumer - - - - org.springframework.cloud.fn - twitter-common - ${project.version} - - - org.springframework.cloud.fn - payload-converter-function - ${project.version} - - - - diff --git a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/friendship/TwitterFriendshipsConsumerConfiguration.java b/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/friendship/TwitterFriendshipsConsumerConfiguration.java deleted file mode 100644 index ae55a818e..000000000 --- a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/friendship/TwitterFriendshipsConsumerConfiguration.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.twitter.friendship; - -import java.util.function.Consumer; - -import twitter4j.Twitter; -import twitter4j.TwitterException; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; - -/** - * - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties(TwitterFriendshipsConsumerProperties.class) -@Import(TwitterConnectionConfiguration.class) -public class TwitterFriendshipsConsumerConfiguration { - - @Bean - @SuppressWarnings("Duplicates") - public Consumer> friendshipConsumer(TwitterFriendshipsConsumerProperties properties, Twitter twitter) { - - return message -> { - try { - TwitterFriendshipsConsumerProperties.OperationType type = - properties.getType().getValue(message, TwitterFriendshipsConsumerProperties.OperationType.class); - //TwitterFriendshipsSinkProperties.OperationType type = TwitterFriendshipsSinkProperties.OperationType.create; - if (properties.getUserId() != null) { - Long userId = properties.getUserId().getValue(message, long.class); - switch (type) { - case create: - boolean follow = properties.getCreate().getFollow().getValue(message, boolean.class); - twitter.createFriendship(userId, follow); - return; - - case update: - boolean enableDeviceNotification = properties.getUpdate().getDevice().getValue(message, boolean.class); - boolean retweets = properties.getUpdate().getRetweets().getValue(message, boolean.class); - twitter.updateFriendship(userId, enableDeviceNotification, retweets); - return; - - case destroy: - twitter.destroyFriendship(userId); - return; - } - } - else if (properties.getScreenName() != null) { - String screenName = properties.getScreenName().getValue(message, String.class); - switch (type) { - case create: - boolean follow = properties.getCreate().getFollow().getValue(message, boolean.class); - twitter.createFriendship(screenName, follow); - return; - - case update: - boolean enableDeviceNotification = properties.getUpdate().getDevice().getValue(message, boolean.class); - boolean retweets = properties.getUpdate().getRetweets().getValue(message, boolean.class); - twitter.updateFriendship(screenName, enableDeviceNotification, retweets); - return; - - case destroy: - twitter.destroyFriendship(screenName); - return; - } - } - else { - throw new IllegalStateException("Either ScreenName or UserID must be set"); - } - } - catch (TwitterException te) { - throw new IllegalStateException("Twitter API error!", te); - } - }; - } -} diff --git a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/friendship/TwitterFriendshipsConsumerProperties.java b/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/friendship/TwitterFriendshipsConsumerProperties.java deleted file mode 100644 index 7cc4b89d3..000000000 --- a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/friendship/TwitterFriendshipsConsumerProperties.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.twitter.friendship; - -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.stereotype.Component; -import org.springframework.validation.annotation.Validated; - - -/** - * @author Christian Tzolov - */ -@Component -@ConfigurationProperties("twitter.friendships.update") -@Validated -public class TwitterFriendshipsConsumerProperties { - - public enum OperationType { - /** Friendship operation types. */ - create, update, destroy - } - - /** - * The screen name of the user to follow (String). - */ - private Expression screenName; - - /** - * The ID of the user to follow (Integer). - */ - private Expression userId; - - /** - * Type of Friendships request. - */ - private Expression type = new SpelExpressionParser().parseExpression("'create'"); - - /** - * Additional properties for the Friendships create requests. - */ - private Create create = new Create(); - - /** - * Additional properties for the Friendships update requests. - */ - private Update update = new Update(); - - public Expression getScreenName() { - return screenName; - } - - public void setScreenName(Expression screenName) { - this.screenName = screenName; - } - - public Expression getUserId() { - return userId; - } - - public void setUserId(Expression userId) { - this.userId = userId; - } - - public Expression getType() { - return type; - } - - public void setType(Expression type) { - this.type = type; - } - - public Create getCreate() { - return create; - } - - public Update getUpdate() { - return update; - } - - @AssertTrue(message = "Either userId or screenName must be provided") - public boolean isUserProvided() { - return this.userId != null || this.screenName != null; - } - - public static class Create { - /** - * The ID of the user to follow (boolean). - */ - @NotNull - private Expression follow = new SpelExpressionParser().parseExpression("'true'"); - - public Expression getFollow() { - return follow; - } - - public void setFollow(Expression follow) { - this.follow = follow; - } - } - - public static class Update { - /** - * Enable/disable device notifications from the target user. - */ - @NotNull - private Expression device = new SpelExpressionParser().parseExpression("'true'"); - - /** - * Enable/disable Retweets from the target user. - */ - @NotNull - private Expression retweets = new SpelExpressionParser().parseExpression("'true'"); - - public Expression getDevice() { - return device; - } - - public void setDevice(Expression device) { - this.device = device; - } - - public Expression getRetweets() { - return retweets; - } - - public void setRetweets(Expression retweets) { - this.retweets = retweets; - } - } -} diff --git a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/message/TwitterMessageConsumerConfiguration.java b/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/message/TwitterMessageConsumerConfiguration.java deleted file mode 100644 index 377d8319a..000000000 --- a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/message/TwitterMessageConsumerConfiguration.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.twitter.message; - -import java.util.function.Consumer; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import twitter4j.Twitter; -import twitter4j.TwitterException; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; - -/** - * - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties(TwitterMessageConsumerProperties.class) -@Import(TwitterConnectionConfiguration.class) -public class TwitterMessageConsumerConfiguration { - - private static final Log logger = LogFactory.getLog(TwitterMessageConsumerConfiguration.class); - - @Bean - public Consumer> sendDirectMessageConsumer(TwitterMessageConsumerProperties messageProperties, Twitter twitter) { - return message -> { - try { - String messageText = messageProperties.getText().getValue(message, String.class); - - if (messageProperties.getUserId() != null) { - Long userId = messageProperties.getUserId().getValue(message, long.class); - if (messageProperties.getMediaId() != null) { - Long mediaId = messageProperties.getMediaId().getValue(message, long.class); - twitter.sendDirectMessage(userId, messageText, mediaId); - } - twitter.sendDirectMessage(userId, messageText); - } - else if (messageProperties.getScreenName() != null) { - String screenName = messageProperties.getScreenName().getValue(message, String.class); - twitter.sendDirectMessage(screenName, messageText); - } - else { - throw new RuntimeException("Either the UserId or screenName must be set"); - } - } - catch (TwitterException e) { - logger.error("Failed to process message:" + message, e); - } - }; - } -} diff --git a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/message/TwitterMessageConsumerProperties.java b/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/message/TwitterMessageConsumerProperties.java deleted file mode 100644 index e37be02b4..000000000 --- a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/message/TwitterMessageConsumerProperties.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.twitter.message; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.validation.annotation.Validated; - - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("twitter.message.update") -@Validated -public class TwitterMessageConsumerProperties { - - private static final Expression DEFAULT_EXPRESSION = new SpelExpressionParser().parseExpression("payload"); - - /** - * The direct message text. URL encode as necessary. Max length of 10,000 characters. - */ - private Expression text = DEFAULT_EXPRESSION; - - /** - * The screen name of the user to whom send the direct message. - */ - private Expression screenName; - - /** - * The user id of the user to whom send the direct message. - */ - private Expression userId; - - /** - * A media id to associate with the message. A Direct Message may only reference a single media id. - */ - private Expression mediaId; - - public Expression getUserId() { - return userId; - } - - public void setUserId(Expression userId) { - this.userId = userId; - } - - public void setText(Expression text) { - this.text = text; - } - - public Expression getText() { - return text; - } - - public Expression getScreenName() { - return screenName; - } - - public void setScreenName(Expression screenName) { - this.screenName = screenName; - } - - public Expression getMediaId() { - return mediaId; - } - - public void setMediaId(Expression mediaId) { - this.mediaId = mediaId; - } -} diff --git a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/status/update/TwitterUpdateConsumerConfiguration.java b/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/status/update/TwitterUpdateConsumerConfiguration.java deleted file mode 100644 index af49801df..000000000 --- a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/status/update/TwitterUpdateConsumerConfiguration.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.twitter.status.update; - -import java.util.function.Consumer; -import java.util.function.Function; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import twitter4j.GeoLocation; -import twitter4j.Status; -import twitter4j.StatusUpdate; -import twitter4j.Twitter; -import twitter4j.TwitterException; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; - -/** - * - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties(TwitterUpdateConsumerProperties.class) -@Import(TwitterConnectionConfiguration.class) -public class TwitterUpdateConsumerConfiguration { - - private static final Log logger = LogFactory.getLog(TwitterUpdateConsumerConfiguration.class); - - @Bean - public Consumer updateStatus(Twitter twitter) { - return statusUpdate -> { - try { - Status status = twitter.updateStatus(statusUpdate); - - if (logger.isDebugEnabled()) { - logger.debug(status); - } - } - catch (TwitterException e) { - logger.error("Failed apply update status: " + statusUpdate, e); - } - }; - } - - @Bean - public Function, StatusUpdate> messageToStatusUpdateFunction( - TwitterUpdateConsumerProperties updateProperties) { - - return message -> { - - String updateText = updateProperties.getText().getValue(message, String.class); - - StatusUpdate statusUpdate = new StatusUpdate(updateText); - - if (updateProperties.getAttachmentUrl() != null) { - statusUpdate.setAttachmentUrl(updateProperties.getAttachmentUrl().getValue(message, String.class)); - } - - if (updateProperties.getPlaceId() != null) { - statusUpdate.setPlaceId(updateProperties.getPlaceId().getValue(message, String.class)); - } - - if (updateProperties.getInReplyToStatusId() != null) { - statusUpdate.setInReplyToStatusId(updateProperties.getInReplyToStatusId().getValue(message, int.class)); - statusUpdate.setAutoPopulateReplyMetadata(true); - } - - if (updateProperties.getDisplayCoordinates() != null) { - statusUpdate.setDisplayCoordinates( - updateProperties.getDisplayCoordinates().getValue(message, boolean.class)); - } - - if (updateProperties.getMediaIds() != null) { - long[] mediaIds = updateProperties.getMediaIds().getValue(message, long[].class); - statusUpdate.setMediaIds(mediaIds); - } - - if (updateProperties.getLocation().getLat() != null) { - double lat = updateProperties.getLocation().getLat().getValue(message, Double.class); - double lon = updateProperties.getLocation().getLon().getValue(message, Double.class); - statusUpdate.setLocation(new GeoLocation(lat, lon)); - } - - return statusUpdate; - }; - } - - @Bean - public Consumer> twitterStatusUpdateConsumer(Function, StatusUpdate> statusUpdateQuery, - Consumer updateStatus) { - return message -> updateStatus.accept(statusUpdateQuery.apply(message)); - } -} diff --git a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/status/update/TwitterUpdateConsumerProperties.java b/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/status/update/TwitterUpdateConsumerProperties.java deleted file mode 100644 index acb2dde61..000000000 --- a/functions/consumer/twitter-consumer/src/main/java/org/springframework/cloud/fn/consumer/twitter/status/update/TwitterUpdateConsumerProperties.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.twitter.status.update; - -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.validation.annotation.Validated; - - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("twitter.update") -@Validated -public class TwitterUpdateConsumerProperties { - - private static final Expression DEFAULT_EXPRESSION = new SpelExpressionParser().parseExpression("payload"); - - /** - * (SpEL expression) The text of the text update. URL encode as necessary. t.co link wrapping will - * affect character counts. Defaults to message's payload - */ - @NotNull - private Expression text = DEFAULT_EXPRESSION; - - /** - * (SpEL expression) In order for a URL to not be counted in the text body of an extended Tweet, provide a URL as a Tweet attachment. - * This URL must be a Tweet permalink, or Direct Message deep link. Arbitrary, non-Twitter URLs must remain in - * the text text. URLs passed to the attachment_url parameter not matching either a Tweet permalink or Direct - * Message deep link will fail at Tweet creation and cause an exception. - */ - private Expression attachmentUrl; - - /** - * (SpEL expression) A place in the world. - */ - private Expression placeId; - - /** - * (SpEL expression) The ID of an existing text that the update is in reply to. Note: This parameter will be ignored unless the - * author of the Tweet this parameter references is mentioned within the text text. Therefore, you must - * include @username, where username is the author of the referenced Tweet, within the update. - * - * When inReplyToStatusId is set the auto_populate_reply_metadata is automatically set as well. Later ensures - * that leading @mentions will be looked up from the original Tweet, and added to the new Tweet from there. - * This wil append @mentions into the metadata of an extended Tweet as a reply chain grows, until the limit - * on @mentions is reached. In cases where the original Tweet has been deleted, - * the reply will fail. - */ - private Expression inReplyToStatusId; - - /** - * (SpEL expression) Whether or not to put a pin on the exact coordinates a Tweet has been sent from. - */ - private Expression displayCoordinates; - - /** - * (SpEL expression) A comma-delimited list of media_ids to associate with the Tweet. You may include up to 4 photos or 1 animated - * GIF or 1 video in a Tweet. See Uploading Media for further details on uploading media. - */ - private Expression mediaIds; - - /** - * (SpEL expression) The location this Tweet refers to. Ignored if geo_enabled for the user is false! - */ - private final Location location = new Location(); - - public Expression getText() { - return text; - } - - public void setText(Expression text) { - this.text = text; - } - - public Expression getAttachmentUrl() { - return attachmentUrl; - } - - public void setAttachmentUrl(Expression attachmentUrl) { - this.attachmentUrl = attachmentUrl; - } - - public Expression getPlaceId() { - return placeId; - } - - public void setPlaceId(Expression placeId) { - this.placeId = placeId; - } - - public Expression getInReplyToStatusId() { - return inReplyToStatusId; - } - - public void setInReplyToStatusId(Expression inReplyToStatusId) { - this.inReplyToStatusId = inReplyToStatusId; - } - - public Expression getDisplayCoordinates() { - return displayCoordinates; - } - - public void setDisplayCoordinates(Expression displayCoordinates) { - this.displayCoordinates = displayCoordinates; - } - - public Expression getMediaIds() { - return mediaIds; - } - - public void setMediaIds(Expression mediaIds) { - this.mediaIds = mediaIds; - } - - public Location getLocation() { - return location; - } - - @AssertTrue(message = "Lat and Long must be set together or both not being set") - public boolean validateLatLon() { - return (this.getLocation().getLat() != null && this.getLocation().getLon() != null) - || (this.getLocation().getLat() == null && this.getLocation().getLon() == null); - } - - public static class Location { - /** - * The latitude of the location this Tweet refers to. This parameter will be ignored unless it is inside the range - * -90.0 to +90.0 (North is positive) inclusive. It will also be ignored if there is no corresponding long parameter. - */ - private Expression lat; - - /** - * The longitude of the location this Tweet refers to. The valid ranges for longitude are -180.0 to +180.0 (East - * is positive) inclusive. This parameter will be ignored if outside that range, if it is not a number, - * if geo_enabled is disabled, or if there no corresponding lat parameter. - */ - private Expression lon; - - public Expression getLat() { - return lat; - } - - public void setLat(Expression lat) { - this.lat = lat; - } - - public Expression getLon() { - return lon; - } - - public void setLon(Expression lon) { - this.lon = lon; - } - } -} diff --git a/functions/consumer/twitter-consumer/src/test/java/org/springframework/cloud/fn/consumer/twitter/status/update/TwitterUpdateSinkFunctionConfigurationTests.java b/functions/consumer/twitter-consumer/src/test/java/org/springframework/cloud/fn/consumer/twitter/status/update/TwitterUpdateSinkFunctionConfigurationTests.java deleted file mode 100644 index 25d4e17a1..000000000 --- a/functions/consumer/twitter-consumer/src/test/java/org/springframework/cloud/fn/consumer/twitter/status/update/TwitterUpdateSinkFunctionConfigurationTests.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.twitter.status.update; - -import java.util.function.Consumer; -import java.util.function.Function; - -import org.junit.jupiter.api.Test; -import twitter4j.StatusUpdate; -import twitter4j.Twitter; -import twitter4j.TwitterException; - -import org.springframework.expression.Expression; -import org.springframework.expression.ExpressionParser; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -/** - * @author Christian Tzolov - */ -public class TwitterUpdateSinkFunctionConfigurationTests { - - @Test - public void testStatusUpdateConsumer() throws TwitterException { - Twitter twitter = mock(Twitter.class); - - Consumer statusUpdateConsumer = - new TwitterUpdateConsumerConfiguration().updateStatus(twitter); - - StatusUpdate statusUpdateQuery = new StatusUpdate("Hello World"); - statusUpdateConsumer.accept(statusUpdateQuery); - verify(twitter).updateStatus(eq(statusUpdateQuery)); - } - - @Test - public void testMessageToStatusUpdateFunction() { - TwitterUpdateConsumerProperties properties = new TwitterUpdateConsumerProperties(); - - properties.setAttachmentUrl(expression("'attachmentUrl'")); - properties.setPlaceId(expression("'myPlaceId'")); - properties.setInReplyToStatusId(expression("'666666'")); - properties.setDisplayCoordinates(expression("'true'")); - properties.setMediaIds(expression("'471592142565957632, 471592142565957633'")); - properties.getLocation().setLat(expression("'37.78217'")); - properties.getLocation().setLon(expression("'-122.40062'")); - - Function, StatusUpdate> messageToStatusUpdateFunction = - new TwitterUpdateConsumerConfiguration().messageToStatusUpdateFunction(properties); - - StatusUpdate result = messageToStatusUpdateFunction.apply(new GenericMessage<>("Hello World")); - - assertThat(result).isNotNull(); - assertThat(result.getStatus()).isEqualTo("Hello World"); - assertThat(result.getAttachmentUrl()).isEqualTo("attachmentUrl"); - assertThat(result.getPlaceId()).isEqualTo("myPlaceId"); - assertThat(result.getInReplyToStatusId()).isEqualTo(666666L); - assertThat(result.isDisplayCoordinates()).isTrue(); - assertThat(result.getLocation().getLatitude()).isEqualTo(37.78217); - assertThat(result.getLocation().getLongitude()).isEqualTo(-122.40062); - } - - private Expression expression(String expressionString) { - ExpressionParser parser = new SpelExpressionParser(); - return parser.parseExpression(expressionString); - } -} diff --git a/functions/consumer/wavefront-consumer/README.adoc b/functions/consumer/wavefront-consumer/README.adoc deleted file mode 100644 index 3de258ce8..000000000 --- a/functions/consumer/wavefront-consumer/README.adoc +++ /dev/null @@ -1,29 +0,0 @@ -# Wavefront Consumer - -This module provides a Wavefront Consumer that can be reused and composed in other applications. - -## Beans for injection - -You can import the `WavefrontConsumerConfiguration` in the application and then inject the following bean. - -`wavefrontConsumer` - -You can use `wavefrontConsumer` as a qualifier when injecting. - -Type for injection: `Consumer>` - -You can ignore the return value from the function as this is a consumer and simply will send the data to Wavefront. - -## Configuration Options - -All configuration properties are prefixed with `wavefront`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerProperties.java[WavefrontConsumerProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/wavefront[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/wavefront-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a Wavefront sink. diff --git a/functions/consumer/wavefront-consumer/pom.xml b/functions/consumer/wavefront-consumer/pom.xml deleted file mode 100644 index 22fbe7555..000000000 --- a/functions/consumer/wavefront-consumer/pom.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - wavefront-consumer - wavefront-consumer - Wavefront Consumer - - - 2.4.0 - - - - - org.springframework.boot - spring-boot-starter-web - - - - diff --git a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerConfiguration.java b/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerConfiguration.java deleted file mode 100644 index bcd35214f..000000000 --- a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerConfiguration.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront; - -import java.util.function.Consumer; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.boot.autoconfigure.web.client.RestTemplateAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.cloud.fn.consumer.wavefront.service.DirectConnectionWavefrontService; -import org.springframework.cloud.fn.consumer.wavefront.service.ProxyConnectionWavefrontService; -import org.springframework.cloud.fn.consumer.wavefront.service.WavefrontService; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; -import org.springframework.util.StringUtils; - -/** - * @author Timo Salm - */ -@Configuration -@EnableConfigurationProperties(WavefrontConsumerProperties.class) -@Import(RestTemplateAutoConfiguration.class) -public class WavefrontConsumerConfiguration { - - private static final Log logger = LogFactory.getLog(WavefrontConsumerConfiguration.class); - - @Bean - public Consumer> wavefrontConsumer(final WavefrontConsumerProperties properties, - final WavefrontService service) { - - return message -> { - final WavefrontFormat wavefrontFormat = new WavefrontFormat(properties, message); - final String formattedString = wavefrontFormat.getFormattedString(); - service.send(formattedString); - if (logger.isDebugEnabled()) { - logger.debug(formattedString); - } - }; - } - - @Bean - public WavefrontService wavefrontService(final WavefrontConsumerProperties properties, - final RestTemplateBuilder restTemplateBuilder) { - - if (!StringUtils.isEmpty(properties.getProxyUri())) { - return new ProxyConnectionWavefrontService(restTemplateBuilder, properties.getProxyUri()); - } - else { - return new DirectConnectionWavefrontService(restTemplateBuilder, properties.getUri(), - properties.getApiToken()); - } - } -} diff --git a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerProperties.java b/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerProperties.java deleted file mode 100644 index f5b28ce91..000000000 --- a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerProperties.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront; - -import java.util.Map; - -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; -import jakarta.validation.constraints.Pattern; -import jakarta.validation.constraints.Size; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.util.StringUtils; -import org.springframework.validation.annotation.Validated; - -/** - * @author Timo Salm - */ -@ConfigurationProperties("wavefront") -@Validated -public class WavefrontConsumerProperties { - - /** - * The name of the metric.Defaults to the application name. - */ - @Value("${spring.application.name:wavefront.consumer}") - private String metricName; - - /** - * Unique application, host, container, or instance that emits metrics. - */ - private String source; - - /** - * A SpEL expression that evaluates to a metric value. - */ - private Expression metricExpression; - - /** - * A SpEL expression that evaluates to a timestamp of the metric (optional). - */ - private Expression timestampExpression; - - /** - * Collection of custom metadata associated with the metric.Point tags cannot be empty. - * Valid characters for keys: alphanumeric, hyphen ('-'), underscore ('_'), dot ('.'). - * For values any character is allowed, including spaces. To include a double quote, escape it with a backslash, - * A backslash cannot be the last character in the tag value. - * Maximum allowed length for a combination of a point tag key and value is 254 characters - * (255 including the '=' separating key and value). - * If the value is longer, the point is rejected and logged - */ - private Map tagExpression; - - /** - * The URL of the Wavefront environment. - */ - private String uri; - - /** - * Wavefront API access token. - */ - private String apiToken; - - /** - * The URL of the Wavefront proxy. - */ - private String proxyUri; - - public WavefrontConsumerProperties() { - } - - WavefrontConsumerProperties(final String metricName, final String source, final Expression metricExpression, - final Expression timestampExpression, final Map pointTagExpressions, - final String wavefrontServerUri, final String wavefrontApiToken, final String wavefrontProxyUrl) { - - setMetricName(metricName); - setSource(source); - setMetricExpression(metricExpression); - setTimestampExpression(timestampExpression); - setTagExpression(pointTagExpressions); - setUri(wavefrontServerUri); - setApiToken(wavefrontApiToken); - setProxyUri(wavefrontProxyUrl); - } - - @NotEmpty - @Pattern(regexp = "^[a-zA-Z0-9./_,-]+") - public String getMetricName() { - return metricName; - } - - public void setMetricName(String metricName) { - this.metricName = metricName; - } - - @NotEmpty - @Size(max = 128) - @Pattern(regexp = "^[a-zA-Z0-9._-]+") - public String getSource() { - return source; - } - - public void setSource(String source) { - this.source = source; - } - - @NotNull - public Expression getMetricExpression() { - return metricExpression; - } - - public void setMetricExpression(Expression metricExpression) { - this.metricExpression = metricExpression; - } - - public Expression getTimestampExpression() { - return timestampExpression; - } - - public void setTimestampExpression(Expression timestampExpression) { - this.timestampExpression = timestampExpression; - } - - public Map getTagExpression() { - return tagExpression; - } - - public void setTagExpression(Map tagExpression) { - this.tagExpression = tagExpression; - } - - public String getUri() { - return uri; - } - - public void setUri(String uri) { - this.uri = uri; - } - - public String getApiToken() { - return apiToken; - } - - public void setApiToken(String apiToken) { - this.apiToken = apiToken; - } - - public String getProxyUri() { - return proxyUri; - } - - public void setProxyUri(String proxyUri) { - this.proxyUri = proxyUri; - } - - @AssertTrue(message = "Exactly one of 'proxy-uri' or the pair of ('uri' and 'api-token') must be set!") - public boolean isMutuallyExclusiveProxyAndDirectAccessWavefrontConfiguration() { - return StringUtils.isEmpty(getProxyUri()) ^ (StringUtils.isEmpty(getUri()) || StringUtils.isEmpty(getApiToken())); - } -} diff --git a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontFormat.java b/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontFormat.java deleted file mode 100644 index 51944c0a4..000000000 --- a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontFormat.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront; - -import java.util.AbstractMap; -import java.util.Map; -import java.util.Objects; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import jakarta.validation.ValidationException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.expression.EvaluationException; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.SpelEvaluationException; -import org.springframework.messaging.Message; - -/** - * @author Timo Salm - */ -public class WavefrontFormat { - - private static final Log logger = LogFactory.getLog(WavefrontFormat.class); - - private final WavefrontConsumerProperties properties; - private final Message message; - - public WavefrontFormat(final WavefrontConsumerProperties properties, Message message) { - this.properties = properties; - this.message = message; - } - - public String getFormattedString() { - final Number metricValue = extractMetricValueFromPayload(); - - final Map pointTagsMap = extractPointTagsMapFromPayload( - properties.getTagExpression(), message); - validatePointTagsKeyValuePairs(pointTagsMap); - final String formattedPointTagsPart = getFormattedPointTags(pointTagsMap); - - if (properties.getTimestampExpression() == null) { - return String.format("\"%s\" %s source=%s %s", properties.getMetricName(), metricValue, - properties.getSource(), formattedPointTagsPart).trim(); - } - - final Long timestamp = extractTimestampFromPayload(); - return String.format("\"%s\" %s %d source=%s %s", properties.getMetricName(), metricValue, timestamp, - properties.getSource(), formattedPointTagsPart).trim(); - } - - private Long extractTimestampFromPayload() { - try { - return properties.getTimestampExpression().getValue(message, Long.class); - } - catch (SpelEvaluationException e) { - throw new ValidationException("The timestamp value has to be a number that reflects the epoch seconds of the " + - "metric (e.g. 1382754475).", e); - } - } - - private Number extractMetricValueFromPayload() { - try { - return properties.getMetricExpression().getValue(message, Number.class); - } - catch (SpelEvaluationException e) { - throw new ValidationException("The metric value has to be a double-precision floating point number or a " + - "long integer. It can be positive, negative, or 0.", e); - } - } - - private String getFormattedPointTags(Map pointTagsMap) { - return pointTagsMap.entrySet().stream() - .map(it -> String.format("%s=\"%s\"", it.getKey(), it.getValue())) - .collect(Collectors.joining(" ")); - } - - private Map extractPointTagsMapFromPayload(Map pointTagsExpressionsPointValue, Message message) { - return pointTagsExpressionsPointValue.entrySet().stream() - .map(it -> { - try { - final Object pointValue = it.getValue().getValue(message); - return new AbstractMap.SimpleEntry<>(it.getKey(), pointValue); - } - catch (EvaluationException e) { - logger.warn("Unable to extract point tag for key " + it.getKey() + " from payload", e); - return null; - } - }) - .filter(Objects::nonNull) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private void validatePointTagsKeyValuePairs(Map pointTagsMap) { - pointTagsMap.forEach((key, value) -> { - if (!Pattern.matches("^[a-zA-Z0-9._-]+", key)) { - throw new ValidationException("Point tag key \"" + key + "\" contains invalid characters: Valid " + - "characters are alphanumeric, hyphen (\"-\"), underscore (\"_\"), dot (\".\")"); - } - - final int keyValueCombinationLength = key.length() + value.toString().length(); - if (keyValueCombinationLength > 254) { - logger.warn("Maximum allowed length for a combination of a point tag key and value " + - "is 254 characters. The length of combination for key " + key + " is " + - keyValueCombinationLength + "."); - } - }); - } -} diff --git a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/service/DirectConnectionWavefrontService.java b/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/service/DirectConnectionWavefrontService.java deleted file mode 100644 index d2860d3ad..000000000 --- a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/service/DirectConnectionWavefrontService.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront.service; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.web.client.RestTemplate; - -/** - * @author Timo Salm - */ -public class DirectConnectionWavefrontService implements WavefrontService { - - private static final Log logger = LogFactory.getLog(DirectConnectionWavefrontService.class); - - private final RestTemplate restTemplate; - private final String wavefrontDomain; - private final String wavefrontToken; - - public DirectConnectionWavefrontService(final RestTemplateBuilder restTemplateBuilder, - final String wavefrontServerUri, final String wavefrontApiToken) { - this.restTemplate = restTemplateBuilder.build(); - this.wavefrontDomain = wavefrontServerUri; - this.wavefrontToken = wavefrontApiToken; - } - - @Override - public void send(String metricInWavefrontFormat) { - if (logger.isDebugEnabled()) { - logger.debug("Send metric directly to Wavefront"); - } - final HttpHeaders headers = new HttpHeaders(); - headers.setBearerAuth(wavefrontToken); - final HttpEntity httpEntity = new HttpEntity<>(metricInWavefrontFormat, headers); - restTemplate.exchange(wavefrontDomain + "/report", HttpMethod.POST, httpEntity, Void.class); - } -} diff --git a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/service/ProxyConnectionWavefrontService.java b/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/service/ProxyConnectionWavefrontService.java deleted file mode 100644 index 1b61aebb9..000000000 --- a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/service/ProxyConnectionWavefrontService.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront.service; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.web.client.RestTemplate; - -/** - * @author Timo Salm - */ -public class ProxyConnectionWavefrontService implements WavefrontService { - - private static final Log logger = LogFactory.getLog(ProxyConnectionWavefrontService.class); - - private final RestTemplate restTemplate; - private final String wavefrontProxyUrl; - - public ProxyConnectionWavefrontService(final RestTemplateBuilder restTemplateBuilder, - final String wavefrontProxyUri) { - this.restTemplate = restTemplateBuilder.build(); - this.wavefrontProxyUrl = wavefrontProxyUri; - } - - @Override - public void send(String metricInWavefrontFormat) { - if (logger.isDebugEnabled()) { - logger.debug("Send metric to Wavefront proxy"); - } - restTemplate.postForEntity(wavefrontProxyUrl, metricInWavefrontFormat, Void.class); - } -} diff --git a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/service/WavefrontService.java b/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/service/WavefrontService.java deleted file mode 100644 index 2d75f1d94..000000000 --- a/functions/consumer/wavefront-consumer/src/main/java/org/springframework/cloud/fn/consumer/wavefront/service/WavefrontService.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront.service; - -/** - * @author Timo Salm - */ -public interface WavefrontService { - void send(String metricInWavefrontFormat); -} diff --git a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerConfigurationTest.java b/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerConfigurationTest.java deleted file mode 100644 index 10e32b8a9..000000000 --- a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerConfigurationTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront; - -import java.util.Date; -import java.util.Locale; -import java.util.function.Consumer; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.cloud.fn.consumer.wavefront.service.WavefrontService; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; - -/** - * @author Timo Salm - */ -@SpringBootTest(properties = { - "wavefront.metric-name=vehicle-location", - "wavefront.source=vehicle-api", - "wavefront.metric-expression=#jsonPath(payload,'$.mileage')", - "wavefront.timestamp-expression=#jsonPath(payload,'$.receivedAt')", - "wavefront.tag-expression.vin=#jsonPath(payload,'$.vin')", - "wavefront.tag-expression.latitude=#jsonPath(payload,'$.location.latitude')", - "wavefront.proxy-uri=testUrl" -}) -public class WavefrontConsumerConfigurationTest { - - @Autowired - private Consumer> wavefrontConsumer; - - @MockBean - private WavefrontService wavefrontServiceMock; - - @BeforeEach - public void init() { - Locale.setDefault(Locale.US); - } - - @Test - void testWavefrontConsumer() { - final long timestamp = new Date().getTime(); - final String dataJsonString = "{ \"mileage\": 1.5, \"receivedAt\": " + timestamp + ", \"vin\": \"test-vin\", " + - "\"location\": {\"latitude\": 4.53, \"longitude\": 2.89 }}"; - - wavefrontConsumer.accept(new GenericMessage(dataJsonString.getBytes())); - - final String formattedString = "\"vehicle-location\" 1.5 " + timestamp + " source=vehicle-api " + - "latitude=\"4.53\" vin=\"test-vin\""; - Mockito.verify(wavefrontServiceMock, Mockito.times(1)).send(formattedString); - } - - @SpringBootApplication - static class WavefrontConsumerTestApplication { - } -} diff --git a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerPropertiesTest.java b/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerPropertiesTest.java deleted file mode 100644 index ac7e52b4d..000000000 --- a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontConsumerPropertiesTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront; - -import java.util.Arrays; -import java.util.List; - -import jakarta.validation.Validation; -import jakarta.validation.Validator; -import jakarta.validation.ValidatorFactory; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import org.springframework.expression.Expression; -import org.springframework.expression.spel.standard.SpelExpressionParser; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Timo Salm - */ -public class WavefrontConsumerPropertiesTest { - - private final Expression testExpression = new SpelExpressionParser().parseExpression("#jsonPath(payload,'$')"); - private Validator validator; - - @BeforeEach - public void setUp() { - ValidatorFactory factory = Validation.buildDefaultValidatorFactory(); - validator = factory.getValidator(); - } - - @Test - void testRequiredProperties() { - final WavefrontConsumerProperties properties = new WavefrontConsumerProperties("v", "v", testExpression, null, null, null, null, "proxy"); - final List emptyValues = Arrays.asList(null, ""); - - emptyValues.forEach(emptyValue -> { - assertThat(validator.validate(properties).isEmpty()).isTrue(); - properties.setMetricName(emptyValue); - assertThat(validator.validate(properties).isEmpty()).isFalse(); - properties.setMetricName("v"); - - assertThat(validator.validate(properties).isEmpty()).isTrue(); - properties.setSource(emptyValue); - assertThat(validator.validate(properties).isEmpty()).isFalse(); - properties.setSource("v"); - }); - assertThat(validator.validate(properties).isEmpty()).isTrue(); - properties.setMetricExpression(null); - assertThat(validator.validate(properties).isEmpty()).isFalse(); - properties.setMetricExpression(testExpression); - } - - @Test - void testValidMetricNameValues() { - final WavefrontConsumerProperties properties = new WavefrontConsumerProperties("v", "v", testExpression, null, null, null, null, "proxy"); - final List validMetricNameValues = Arrays.asList("b", "B", "2", ".", "/", "_", ",", "-", "c.8W-2h_dE_,J-h/"); - assertThat(validator.validate(properties).isEmpty()).isTrue(); - - validMetricNameValues.forEach(validMetricNameValue -> { - properties.setMetricName(validMetricNameValue); - assertThat(validator.validate(properties).isEmpty()).isTrue(); - }); - - final List invalidMetricNameValues = Arrays.asList(" ", ":", "a B", "#"); - invalidMetricNameValues.forEach(invalidMetricNameValue -> { - properties.setMetricName(invalidMetricNameValue); - assertThat(validator.validate(properties).isEmpty()).isFalse(); - }); - } - - @Test - void testValidSourceValues() { - final WavefrontConsumerProperties properties = new WavefrontConsumerProperties("v", "v", testExpression, null, null, null, null, "proxy"); - final List validSourceValues = Arrays.asList("b", "B", "2", ".", "_", "-", "c.8W-2h_dE_J-h", - createStringOfLength(128)); - assertThat(validator.validate(properties).isEmpty()).isTrue(); - - validSourceValues.forEach(validSourceValue -> { - properties.setSource(validSourceValue); - assertThat(validator.validate(properties).isEmpty()).isTrue(); - }); - - final List invalidSourceValues = Arrays.asList(" ", ":", "a B", "#", "/", ",", createStringOfLength(129)); - invalidSourceValues.forEach(invalidSourceValue -> { - properties.setSource(invalidSourceValue); - assertThat(validator.validate(properties).isEmpty()).isFalse(); - }); - } - - private String createStringOfLength(int length) { - char[] chars = new char[length]; - Arrays.fill(chars, 'a'); - return new String(chars); - } -} diff --git a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontFormatTest.java b/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontFormatTest.java deleted file mode 100644 index 8983f267b..000000000 --- a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/WavefrontFormatTest.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront; - -import java.util.AbstractMap; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.read.ListAppender; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.slf4j.LoggerFactory; - -import org.springframework.beans.BeanUtils; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.standard.SpelExpression; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.expression.spel.support.StandardEvaluationContext; -import org.springframework.integration.json.JsonPathUtils; -import org.springframework.messaging.support.GenericMessage; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Timo Salm - */ -public class WavefrontFormatTest { - - private final SpelExpressionParser parser = new SpelExpressionParser(); - - @BeforeEach - public void init() { - Locale.setDefault(Locale.US); - } - - @Test - void testGetFormattedString() { - final long timestamp = new Date().getTime(); - final String dataJsonString = "{ \"value\": 1.5, \"timestamp\": " + timestamp + ", " - + "\"testProp1\": \"testvalue1\", \"testProp2\": \"testvalue2\" }"; - - final Map pointTagsExpressionsPointValueMap = Stream.of( - new AbstractMap.SimpleEntry<>("testpoint1", expression("$.testProp1")), - new AbstractMap.SimpleEntry<>("testpoint2", expression("$.testProp2")) - ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final WavefrontConsumerProperties properties = new WavefrontConsumerProperties("testMetricName", "testSource", - expression("$.value"), expression("$.timestamp"), pointTagsExpressionsPointValueMap, null, null, null); - - final String result = new WavefrontFormat(properties, new GenericMessage<>(dataJsonString)).getFormattedString(); - assertThat(result).isEqualTo("\"testMetricName\" 1.5 " + timestamp + " source=testSource testpoint2=\"testvalue2\"" - + " testpoint1=\"testvalue1\"", result); - } - - @Test - void testGetFormattedStringWithoutTimeStamp() { - final String dataJsonString = "{ \"value\": 1.5, \"testProp1\": \"testvalue1\", \"testProp2\": \"testvalue2\" }"; - - final Map pointTagsExpressionsPointValueMap = - Stream.of(new AbstractMap.SimpleEntry<>("testpoint1", expression("$.testProp1"))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final WavefrontConsumerProperties properties = new WavefrontConsumerProperties("testMetricName", "testSource", - expression("$.value"), null, pointTagsExpressionsPointValueMap, null, null, null); - - final String result = new WavefrontFormat(properties, new GenericMessage<>(dataJsonString)).getFormattedString(); - assertThat(result).isEqualTo("\"testMetricName\" 1.5 source=testSource testpoint1=\"testvalue1\""); - } - - @Test - void testGetFormattedStringWithoutPointTags() { - final long timestamp = new Date().getTime(); - final String dataJsonString = "{ \"value\": 1.5, \"timestamp\": " + timestamp + "}"; - - final WavefrontConsumerProperties properties = new WavefrontConsumerProperties("testMetricName", "testSource", - expression("$.value"), expression("$.timestamp"), Collections.emptyMap(), null, null, null); - - final String result = new WavefrontFormat(properties, new GenericMessage<>(dataJsonString)).getFormattedString(); - assertThat(result).isEqualTo("\"testMetricName\" 1.5 " + timestamp + " source=testSource"); - } - - @Test - void testInvalidMetricValue() { - final String dataJsonString = "{ \"value\": a}"; - final WavefrontConsumerProperties properties = new WavefrontConsumerProperties("testMetricName", "testSource", - expression("$.value"), null, Collections.emptyMap(), null, null, null); - final Exception exception = Assertions.assertThrows(RuntimeException.class, () -> - new WavefrontFormat(properties, new GenericMessage<>(dataJsonString)).getFormattedString()); - assertThat(exception.getLocalizedMessage().startsWith("The metric value has to be a double-precision floating")) - .isTrue(); - } - - @Test - void testInvalidTimestampValue() { - final String dataJsonString = "{ \"value\": 1.5, \"timestamp\": 2020-06-02T13:53:18+0000}"; - final WavefrontConsumerProperties properties = new WavefrontConsumerProperties("testMetricName", "testSource", - expression("$.value"), expression("$.timestamp"), Collections.emptyMap(), null, null, null); - final Exception exception = Assertions.assertThrows(RuntimeException.class, - () -> new WavefrontFormat(properties, new GenericMessage<>(dataJsonString)).getFormattedString()); - assertThat(exception.getLocalizedMessage().startsWith("The timestamp value has to be a number")).isTrue(); - } - - @Test - void testInvalidPointTagsLengthWarning() { - final Logger logger = (Logger) LoggerFactory.getLogger(WavefrontFormat.class); - final ListAppender listAppender = new ListAppender<>(); - listAppender.start(); - logger.addAppender(listAppender); - - final String testPointTagKey = createStringOfLength(127); - - final Map pointTagsExpressionsPointValueMap = - Stream.of(new AbstractMap.SimpleEntry<>(testPointTagKey, expression("$.testPoint1"))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final WavefrontConsumerProperties properties = new WavefrontConsumerProperties("testMetricName", "testSource", - expression("$.value"), null, pointTagsExpressionsPointValueMap, null, null, null); - - final String dataJsonString = "{ \"value\": 1.5, \"testPoint1\": \"" + - createStringOfLength(254 - testPointTagKey.length()) + "\" }"; - new WavefrontFormat(properties, new GenericMessage<>(dataJsonString)).getFormattedString(); - assertThat(listAppender.list.stream() - .filter(event -> event.getMessage().startsWith("Maximum allowed length for a combination")) - .count()).isEqualTo(0); - - final String dataJsonStringWithTooLongValue = "{ \"value\": 1.5, \"testPoint1\": \"" + - createStringOfLength(255 - testPointTagKey.length()) + "\" }"; - new WavefrontFormat(properties, new GenericMessage<>(dataJsonStringWithTooLongValue)).getFormattedString(); - - assertThat(listAppender.list.stream() - .filter(event -> event.getMessage().startsWith("Maximum allowed length for a combination") - && event.getLevel().equals(Level.WARN)) - .count()).isEqualTo(1); - } - - @Test - void testInvalidPointTagKeys() { - final String dataJsonString = "{ \"value\": 1.5, \"testPoint1\": \"testvalue1\" }"; - - final List validPointTagKeys = Arrays.asList("b", "B", "2", ".", "_", "-", "c.8W-2h_dE_J-h"); - for (String validPointTagKey : validPointTagKeys) { - final Map pointTagsExpressionsPointValueMap = - Stream.of(new AbstractMap.SimpleEntry<>(validPointTagKey, expression("$.testPoint1"))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final WavefrontConsumerProperties properties = new WavefrontConsumerProperties("testMetricName", "testSource", - expression("$.value"), null, pointTagsExpressionsPointValueMap, null, null, null); - new WavefrontFormat(properties, new GenericMessage<>(dataJsonString)).getFormattedString(); - } - - final List invalidPointTagKeys = Arrays.asList(" ", ":", "a B", "#", "/", ","); - - invalidPointTagKeys.forEach(invalidPointTagKey -> { - final Map pointTagsExpressionsPointValueMap = - Stream.of(new AbstractMap.SimpleEntry<>(invalidPointTagKey, expression("$.testPoint1"))) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - final WavefrontConsumerProperties properties = new WavefrontConsumerProperties("testMetricName", "testSource", - expression("$.value"), null, pointTagsExpressionsPointValueMap, null, null, null); - - final Exception exception = Assertions.assertThrows(RuntimeException.class, - () -> new WavefrontFormat(properties, new GenericMessage<>(dataJsonString)).getFormattedString()); - assertThat(exception.getLocalizedMessage() - .startsWith("Point tag key \"" + invalidPointTagKey + "\" contains invalid characters")).isTrue(); - }); - } - - private String createStringOfLength(int length) { - char[] chars = new char[length]; - Arrays.fill(chars, 'a'); - return new String(chars); - } - - private Expression expression(final String jsonPathExpression) { - final Expression expression = parser.parseExpression("#jsonPath(payload,'" + jsonPathExpression + "')"); - final StandardEvaluationContext evaluationContext = (StandardEvaluationContext) ((SpelExpression) expression).getEvaluationContext(); - evaluationContext.registerFunction("jsonPath", Objects.requireNonNull(BeanUtils.resolveSignature("evaluate", JsonPathUtils.class))); - return expression; - } -} diff --git a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/service/DirectConnectionWavefrontServiceTest.java b/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/service/DirectConnectionWavefrontServiceTest.java deleted file mode 100644 index 968ccd58d..000000000 --- a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/service/DirectConnectionWavefrontServiceTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront.service; - -import java.util.Objects; - -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; - -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpMethod; -import org.springframework.web.client.RestTemplate; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - - -/** - * @author Timo Salm - */ -public class DirectConnectionWavefrontServiceTest { - - @Test - void testSendMetricInWavefrontFormat() { - final RestTemplateBuilder restTemplateBuilderMock = mock(RestTemplateBuilder.class); - final RestTemplate restTemplateMock = mock(RestTemplate.class); - when(restTemplateBuilderMock.build()).thenReturn(restTemplateMock); - - final String metricInWavefrontFormat = "testMetric"; - final String wavefrontServerUri = "testWavefrontDomain"; - final String wavefrontApiToken = "testWavefrontToken"; - - final WavefrontService service = new DirectConnectionWavefrontService(restTemplateBuilderMock, - wavefrontServerUri, wavefrontApiToken); - service.send(metricInWavefrontFormat); - - final ArgumentCaptor argument = ArgumentCaptor.forClass(HttpEntity.class); - verify(restTemplateMock, Mockito.times(1)) - .exchange(eq(wavefrontServerUri + "/report"), eq(HttpMethod.POST), argument.capture(), - eq(Void.class)); - assertThat(Objects.requireNonNull(argument.getValue().getHeaders().get("Authorization")).get(0)) - .isEqualTo("Bearer " + wavefrontApiToken); - assertThat(Objects.requireNonNull(argument.getValue().getBody())).isEqualTo(metricInWavefrontFormat); - } -} diff --git a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/service/ProxyConnectionWavefrontServiceTest.java b/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/service/ProxyConnectionWavefrontServiceTest.java deleted file mode 100644 index 3f8f60bd0..000000000 --- a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/service/ProxyConnectionWavefrontServiceTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront.service; - -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.web.client.RestTemplate; - -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -/** - * @author Timo Salm - */ -public class ProxyConnectionWavefrontServiceTest { - - @Test - void testSendMetricInWavefrontFormat() { - final RestTemplateBuilder restTemplateBuilderMock = mock(RestTemplateBuilder.class); - final RestTemplate restTemplateMock = mock(RestTemplate.class); - when(restTemplateBuilderMock.build()).thenReturn(restTemplateMock); - - final String metricInWavefrontFormat = "testMetric"; - final String wavefrontProxyUrl = "testWavefrontProxyUrl"; - - final WavefrontService service = new ProxyConnectionWavefrontService(restTemplateBuilderMock, wavefrontProxyUrl); - service.send(metricInWavefrontFormat); - - verify(restTemplateMock, Mockito.times(1)) - .postForEntity(eq(wavefrontProxyUrl), eq(metricInWavefrontFormat), eq(Void.class)); - } -} diff --git a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/service/WavefrontServiceConditionTest.java b/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/service/WavefrontServiceConditionTest.java deleted file mode 100644 index 7cbb224e8..000000000 --- a/functions/consumer/wavefront-consumer/src/test/java/org/springframework/cloud/fn/consumer/wavefront/service/WavefrontServiceConditionTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.wavefront.service; - -import java.util.Objects; -import java.util.UUID; - -import org.junit.jupiter.api.Test; - -import org.springframework.boot.context.annotation.UserConfigurations; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.cloud.fn.common.config.SpelExpressionConverterConfiguration; -import org.springframework.cloud.fn.consumer.wavefront.WavefrontConsumerConfiguration; -import org.springframework.core.NestedExceptionUtils; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Timo Salm - */ -public class WavefrontServiceConditionTest { - - private final ApplicationContextRunner runner = new ApplicationContextRunner() - .withConfiguration(UserConfigurations.of(WavefrontConsumerConfiguration.class, - SpelExpressionConverterConfiguration.class)); - - @Test - public void proxyConnectionShouldBeUsedIfWavefrontProxyAddressSet() { - runner.withPropertyValues( - "wavefront.metric-name=vehicle-location", - "wavefront.source=vehicle-api", - "wavefront.metric-expression=#jsonPath(payload,'$.mileage')", - "wavefront.proxy-uri=http://wavefront-proxy.internal:2878", - "wavefront.uri=", - "wavefront.api-token=" - ).run(context -> { - assertThat(context).hasSingleBean(ProxyConnectionWavefrontService.class); - assertThat(context).doesNotHaveBean(DirectConnectionWavefrontService.class); - }); - } - - @Test - public void proxyConnectionShouldBeUsedIfWavefrontProxyAddressAndDomainAndTokenSet() { - runner.withPropertyValues( - "wavefront.metric-name=vehicle-location", - "wavefront.source=vehicle-api", - "wavefront.metric-expression=#jsonPath(payload,'$.mileage')", - "wavefront.proxy-uri=http://wavefront-proxy.internal:2878", - "wavefront.uri=https://my.wavefront.com", - "wavefront.api-token=" + UUID.randomUUID() - ).run(context -> { - assertThat(context).hasFailed(); - final Throwable rootCause = NestedExceptionUtils.getRootCause(context.getStartupFailure()); - assertThat(Objects.requireNonNull(rootCause).getLocalizedMessage()) - .contains("Exactly one of 'proxy-uri' or the pair of ('uri' and 'api-token') must be set!"); - }); - } - - @Test - public void directConnectionShouldBeUsedIfWavefrontDomainAndTokenSet() { - runner.withPropertyValues( - "wavefront.metric-name=vehicle-location", - "wavefront.source=vehicle-api", - "wavefront.metric-expression=#jsonPath(payload,'$.mileage')", - "wavefront.proxy-uri=", - "wavefront.uri=https://my.wavefront.com", - "wavefront.api-token=" + UUID.randomUUID() - ).run(context -> { - assertThat(context).hasSingleBean(DirectConnectionWavefrontService.class); - assertThat(context).doesNotHaveBean(ProxyConnectionWavefrontService.class); - }); - } - - @Test - public void applicationStartupShouldFailWithMeaningfulErrorMessageIfWavefrontProxyAddressOrDomainAndTokenNotSet() { - runner.withPropertyValues( - "wavefront.metric-name=vehicle-location", - "wavefront.source=vehicle-api", - "wavefront.metric-expression=#jsonPath(payload,'$.mileage')", - "wavefront.proxy-uri=", - "wavefront.uri=", - "wavefront.api-token=" - ).run(context -> { - assertThat(context).hasFailed(); - final Throwable rootCause = NestedExceptionUtils.getRootCause(context.getStartupFailure()); - assertThat(Objects.requireNonNull(rootCause).getLocalizedMessage()) - .contains("Exactly one of 'proxy-uri' or the pair of ('uri' and 'api-token') must be set!"); - }); - } -} diff --git a/functions/consumer/websocket-consumer/README.adoc b/functions/consumer/websocket-consumer/README.adoc deleted file mode 100644 index 16669f237..000000000 --- a/functions/consumer/websocket-consumer/README.adoc +++ /dev/null @@ -1,25 +0,0 @@ -# Websocket Consumer - -A consumer that allows you to send messages using websocket. - -## Beans for injection - -You can import `WebsocketConsumerConfiguration` in the application and then inject the following bean. - -`Consumer> websocketConsumer` - -You can use `websocketConsumer` as a qualifier when injecting. - -## Configuration Options - -All configuration properties are prefixed with `websocket.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerProperties.java[WebsocketConsumerProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/websocket[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/websocket-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a TCP sink. \ No newline at end of file diff --git a/functions/consumer/websocket-consumer/pom.xml b/functions/consumer/websocket-consumer/pom.xml deleted file mode 100644 index c301e9824..000000000 --- a/functions/consumer/websocket-consumer/pom.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - websocket-consumer - websocket-consumer - websocket consumer - - - - org.springframework.boot - spring-boot-starter-actuator - - - org.springframework - spring-websocket - - - io.netty - netty-all - - - org.springframework.boot - spring-boot-starter-web - test - - - org.springframework.cloud.fn - function-test-support - ${project.version} - test - - - - diff --git a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerConfiguration.java b/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerConfiguration.java deleted file mode 100644 index 37908f5eb..000000000 --- a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerConfiguration.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2014-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.websocket; - -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.function.Consumer; - -import io.netty.channel.Channel; -import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; -import jakarta.annotation.PostConstruct; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.consumer.websocket.actuator.WebsocketConsumerTraceEndpoint; -import org.springframework.cloud.fn.consumer.websocket.trace.InMemoryTraceRepository; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.messaging.Message; -import org.springframework.messaging.simp.SimpMessageHeaderAccessor; -import org.springframework.messaging.simp.SimpMessageType; - -/** - * @author Oliver Moser - * @author Gary Russell - * @author Artem Bilan - * @author Chris Bono - */ -@Configuration -@EnableConfigurationProperties(WebsocketConsumerProperties.class) -public class WebsocketConsumerConfiguration { - - private static final Log logger = LogFactory.getLog(WebsocketConsumerConfiguration.class); - - @Value("${endpoints.websocketconsumertrace.enabled:false}") - private boolean traceEndpointEnabled; - - @Autowired - private WebsocketConsumerServer websocketConsumerServer; - - @PostConstruct - public void init() throws InterruptedException { - websocketConsumerServer.run(); - } - - @Bean - @ConditionalOnProperty(value = "endpoints.websocketsinktrace.enabled", havingValue = "true") - public WebsocketConsumerTraceEndpoint websocketTraceEndpoint(InMemoryTraceRepository websocketTraceRepository) { - return new WebsocketConsumerTraceEndpoint(websocketTraceRepository); - } - - @Bean - public Consumer> websocketConsumer(InMemoryTraceRepository websocketTraceRepository) { - return message -> { - if (logger.isTraceEnabled()) { - logger.trace("Handling message: " + message); - } - SimpMessageHeaderAccessor headers = SimpMessageHeaderAccessor.wrap(message); - headers.setMessageTypeIfNotSet(SimpMessageType.MESSAGE); - String messagePayload = message.getPayload().toString(); - for (Channel channel : WebsocketConsumerServer.channels) { - if (logger.isTraceEnabled()) { - logger.trace(String.format("Writing message %s to channel %s", messagePayload, channel.localAddress())); - } - - channel.write(new TextWebSocketFrame(messagePayload)); - channel.flush(); - } - - if (this.traceEndpointEnabled) { - addMessageToTraceRepository(websocketTraceRepository, message); - } - }; - } - - private void addMessageToTraceRepository(InMemoryTraceRepository websocketTraceRepository, Message message) { - Map trace = new LinkedHashMap<>(); - trace.put("type", "text"); - trace.put("direction", "out"); - trace.put("id", message.getHeaders().getId()); - trace.put("payload", message.getPayload().toString()); - websocketTraceRepository.add(trace); - } - - @Configuration - static class WebsocketConsumerServerConfiguration { - @Bean - public InMemoryTraceRepository websocketTraceRepository() { - return new InMemoryTraceRepository(); - } - - @Bean - public WebsocketConsumerServer server(WebsocketConsumerProperties properties, WebsocketConsumerServerInitializer initializer) { - return new WebsocketConsumerServer(properties, initializer); - } - - @Bean - public WebsocketConsumerServerInitializer initializer(InMemoryTraceRepository websocketTraceRepository) { - return new WebsocketConsumerServerInitializer(websocketTraceRepository); - } - } - -} diff --git a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerProperties.java b/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerProperties.java deleted file mode 100644 index 4c881c367..000000000 --- a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerProperties.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2014-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.websocket; - -import io.netty.handler.logging.LogLevel; - -import org.springframework.boot.context.properties.ConfigurationProperties; - -/** - * @author Oliver Moser - * @author Gary Russell - */ -@ConfigurationProperties("websocket.consumer") -public class WebsocketConsumerProperties { - - /** - * Default log level. - */ - public static final String DEFAULT_LOGLEVEL = LogLevel.WARN.toString(); - - /** - * Default path. - */ - public static final String DEFAULT_PATH = "/websocket"; - - /** - * Default number of threads. - */ - public static final int DEFAULT_THREADS = 1; - - /** - * Default port. - */ - public static final int DEFAULT_PORT = 9292; - - /** - * whether or not to create a {@link io.netty.handler.ssl.SslContext}. - */ - boolean ssl; - - /** - * the port on which the Netty server listens. Default is 9292 - */ - int port = DEFAULT_PORT; - - /** - * the number of threads for the Netty {@link io.netty.channel.EventLoopGroup}. Default is 1 - */ - int threads = DEFAULT_THREADS; - - /** - * the logLevel for netty channels. Default is WARN - */ - String logLevel = DEFAULT_LOGLEVEL; - - /** - * the path on which a WebsocketSink consumer needs to connect. Default is /websocket - */ - String path = DEFAULT_PATH; - - public boolean isSsl() { - return ssl; - } - - public void setSsl(boolean ssl) { - this.ssl = ssl; - } - - public int getPort() { - return port; - } - - public void setPort(int port) { - this.port = port; - } - - public int getThreads() { - return threads; - } - - public void setThreads(int threads) { - this.threads = threads; - } - - public String getLogLevel() { - return logLevel; - } - - public void setLogLevel(String logLevel) { - this.logLevel = logLevel; - } - - public String getPath() { - return path; - } - - public void setPath(String path) { - this.path = path; - } -} diff --git a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerServer.java b/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerServer.java deleted file mode 100644 index 1ac4844d9..000000000 --- a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerServer.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2014-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.websocket; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.Channel; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.logging.LogLevel; -import io.netty.handler.logging.LoggingHandler; -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * Bootstraps a Netty server using the {@link WebsocketConsumerServerInitializer}. Also adds - * a {@link LoggingHandler} and uses the logLevel - * from {@link WebsocketConsumerProperties#logLevel}. - * - * @author Oliver Moser - * @author Gary Russell - * @author Chris Bono - */ -public class WebsocketConsumerServer { - - private static final Log logger = LogFactory.getLog(WebsocketConsumerServer.class); - - static final List channels = Collections.synchronizedList(new ArrayList()); - - private WebsocketConsumerProperties properties; - - private WebsocketConsumerServerInitializer initializer; - - private EventLoopGroup bossGroup; - - private EventLoopGroup workerGroup; - - private int port; - - public WebsocketConsumerServer(WebsocketConsumerProperties properties, WebsocketConsumerServerInitializer initializer) { - this.properties = properties; - this.initializer = initializer; - } - - public int getPort() { - return this.port; - } - - @PostConstruct - public void init() { - bossGroup = new NioEventLoopGroup(properties.getThreads()); - workerGroup = new NioEventLoopGroup(); - } - - @PreDestroy - public void shutdown() { - bossGroup.shutdownGracefully(); - workerGroup.shutdownGracefully(); - } - - public void run() throws InterruptedException { - NioServerSocketChannel channel = (NioServerSocketChannel) new ServerBootstrap().group(bossGroup, workerGroup) - .channel(NioServerSocketChannel.class) - .handler(new LoggingHandler(nettyLogLevel())) - .childHandler(initializer) - .bind(properties.getPort()) - .sync() - .channel(); - this.port = channel.localAddress().getPort(); - dumpProperties(); - } - - private void dumpProperties() { - logger.info("███████████████████████████████████████████████████████████"); - logger.info(" >> websocket-sink config << "); - logger.info(""); - logger.info(String.format("port: %s", this.port)); - logger.info(String.format("ssl: %s", this.properties.isSsl())); - logger.info(String.format("path: %s", this.properties.getPath())); - logger.info(String.format("logLevel: %s", this.properties.getLogLevel())); - logger.info(String.format("threads: %s", this.properties.getThreads())); - logger.info(""); - logger.info("████████████████████████████████████████████████████████████"); - } - - // - // HELPERS - // - private LogLevel nettyLogLevel() { - return LogLevel.valueOf(properties.getLogLevel().toUpperCase()); - } - -} diff --git a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerServerHandler.java b/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerServerHandler.java deleted file mode 100644 index a1856df6c..000000000 --- a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerServerHandler.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright 2014-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.websocket; - -import java.util.LinkedHashMap; -import java.util.Map; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.handler.codec.http.DefaultFullHttpResponse; -import io.netty.handler.codec.http.FullHttpRequest; -import io.netty.handler.codec.http.FullHttpResponse; -import io.netty.handler.codec.http.HttpUtil; -import io.netty.handler.codec.http.websocketx.CloseWebSocketFrame; -import io.netty.handler.codec.http.websocketx.PingWebSocketFrame; -import io.netty.handler.codec.http.websocketx.PongWebSocketFrame; -import io.netty.handler.codec.http.websocketx.TextWebSocketFrame; -import io.netty.handler.codec.http.websocketx.WebSocketFrame; -import io.netty.handler.codec.http.websocketx.WebSocketServerHandshaker; -import io.netty.handler.codec.http.websocketx.WebSocketServerHandshakerFactory; -import io.netty.util.CharsetUtil; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.cloud.fn.consumer.websocket.trace.InMemoryTraceRepository; - -import static io.netty.handler.codec.http.HttpHeaderNames.HOST; -import static io.netty.handler.codec.http.HttpMethod.GET; -import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST; -import static io.netty.handler.codec.http.HttpResponseStatus.FORBIDDEN; -import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; - -/** - * Handles handshakes and messages. Based on the Netty websocket examples. - * - * @author Netty Project - * @author Oliver Moser - * @author Gary Russell - * @author Artem Bilan - */ -public class WebsocketConsumerServerHandler extends SimpleChannelInboundHandler { - - private static final Log logger = LogFactory.getLog(WebsocketConsumerServerHandler.class); - - private final boolean traceEnabled; - - private final InMemoryTraceRepository websocketTraceRepository; - - private final WebsocketConsumerProperties properties; - - private WebSocketServerHandshaker handshaker; - - public WebsocketConsumerServerHandler(InMemoryTraceRepository websocketTraceRepository, - WebsocketConsumerProperties properties, - boolean traceEnabled) { - - this.websocketTraceRepository = websocketTraceRepository; - this.properties = properties; - this.traceEnabled = traceEnabled; - } - - @Override - public void channelRead0(ChannelHandlerContext ctx, Object msg) { - if (msg instanceof FullHttpRequest) { - handleHttpRequest(ctx, (FullHttpRequest) msg); - } - else if (msg instanceof WebSocketFrame) { - handleWebSocketFrame(ctx, (WebSocketFrame) msg); - } - } - - @Override - public void channelReadComplete(ChannelHandlerContext ctx) { - ctx.flush(); - } - - private void handleHttpRequest(ChannelHandlerContext ctx, FullHttpRequest req) { - // Handle a bad request. - if (!req.decoderResult().isSuccess()) { - logger.warn(String.format("Bad request: %s", req.uri())); - sendHttpResponse(ctx, req, new DefaultFullHttpResponse(HTTP_1_1, BAD_REQUEST)); - return; - } - - // Allow only GET methods. - if (req.method() != GET) { - logger.warn(String.format("Unsupported HTTP method: %s", req.method())); - sendHttpResponse(ctx, req, new DefaultFullHttpResponse(HTTP_1_1, FORBIDDEN)); - return; - } - - // enable subclasses to do additional processing - if (!additionalHttpRequestHandler(ctx, req)) { - return; - } - - // Handshake - WebSocketServerHandshakerFactory wsFactory - = new WebSocketServerHandshakerFactory(getWebSocketLocation(req), null, true); - - this.handshaker = wsFactory.newHandshaker(req); - if (this.handshaker == null) { - WebSocketServerHandshakerFactory.sendUnsupportedVersionResponse(ctx.channel()); - } - else { - this.handshaker.handshake(ctx.channel(), req); - WebsocketConsumerServer.channels.add(ctx.channel()); - } - } - - private void handleWebSocketFrame(ChannelHandlerContext ctx, WebSocketFrame frame) { - // Check for closing frame - if (frame instanceof CloseWebSocketFrame) { - addTraceForFrame(frame, "close"); - this.handshaker.close(ctx.channel(), (CloseWebSocketFrame) frame.retain()); - return; - } - if (frame instanceof PingWebSocketFrame) { - addTraceForFrame(frame, "ping"); - ctx.channel().write(new PongWebSocketFrame(frame.content().retain())); - return; - } - if (!(frame instanceof TextWebSocketFrame)) { - throw new UnsupportedOperationException(String.format("%s frame types not supported", frame.getClass() - .getName())); - } - - // todo [om] think about BinaryWebsocketFrame - - handleTextWebSocketFrameInternal((TextWebSocketFrame) frame, ctx); - } - - private boolean additionalHttpRequestHandler(ChannelHandlerContext ctx, FullHttpRequest req) { - // implement other HTTP request logic - return true; // continue processing - } - - // simple echo implementation - private void handleTextWebSocketFrameInternal(TextWebSocketFrame frame, ChannelHandlerContext ctx) { - if (logger.isTraceEnabled()) { - logger.trace(String.format("%s received %s", ctx.channel(), frame.text())); - } - - addTraceForFrame(frame, "text"); - ctx.channel().write(new TextWebSocketFrame("Echo: " + frame.text())); - } - - // add trace information for received frame - private void addTraceForFrame(WebSocketFrame frame, String type) { - Map trace = new LinkedHashMap<>(); - trace.put("type", type); - trace.put("direction", "in"); - if (frame instanceof TextWebSocketFrame) { - trace.put("payload", ((TextWebSocketFrame) frame).text()); - } - - if (this.traceEnabled) { - this.websocketTraceRepository.add(trace); - } - } - - private void sendHttpResponse(ChannelHandlerContext ctx, FullHttpRequest req, FullHttpResponse res) { - // Generate an error page if response getStatus code is not OK (200). - if (res.status().code() != 200) { - ByteBuf buf = Unpooled.copiedBuffer(res.status().toString(), CharsetUtil.UTF_8); - res.content().writeBytes(buf); - buf.release(); - HttpUtil.setContentLength(res, res.content().readableBytes()); - } - - // Send the response and close the connection if necessary. - ChannelFuture f = ctx.channel().writeAndFlush(res); - if (!HttpUtil.isKeepAlive(req) || res.status().code() != 200) { - f.addListener(ChannelFutureListener.CLOSE); - } - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - logger.error("Websocket error", cause); - cause.printStackTrace(); - ctx.close(); - } - - private String getWebSocketLocation(FullHttpRequest req) { - String location = req.headers().get(HOST) + this.properties.getPath(); - if (this.properties.isSsl()) { - return "wss://" + location; - } - else { - return "ws://" + location; - } - } -} diff --git a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerServerInitializer.java b/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerServerInitializer.java deleted file mode 100644 index ffdefc021..000000000 --- a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerServerInitializer.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2014-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.websocket; - -import java.security.cert.CertificateException; - -import javax.net.ssl.SSLException; - -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelPipeline; -import io.netty.channel.socket.SocketChannel; -import io.netty.handler.codec.http.HttpObjectAggregator; -import io.netty.handler.codec.http.HttpServerCodec; -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslContextBuilder; -import io.netty.handler.ssl.util.SelfSignedCertificate; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.cloud.fn.consumer.websocket.trace.InMemoryTraceRepository; - -/** - * Does some basic initialization and setup. - * - *
    - *
  • Configure the {@link SslContext} based on {@link WebsocketConsumerProperties#ssl}
    • - *
  • add the {@link WebsocketConsumerServerHandler} to the underlying {@link ChannelPipeline}
    • - *
- * - * @author Oliver Moser - * @author Gary Russell - * @author Artem Bilan - */ -public class WebsocketConsumerServerInitializer extends ChannelInitializer { - - /** - * Max content length. - */ - public static final int MAX_CONTENT_LENGTH = 65536; - - private final InMemoryTraceRepository traceRepository; - - @Autowired - private WebsocketConsumerProperties properties; - - @Value("${endpoints.websocketsinktrace.enabled:false}") - private boolean traceEnabled; - - public WebsocketConsumerServerInitializer(InMemoryTraceRepository traceRepository) { - this.traceRepository = traceRepository; - } - - @Override - public void initChannel(SocketChannel ch) throws Exception { - ChannelPipeline pipeline = ch.pipeline(); - - final SslContext sslCtx = configureSslContext(); - if (sslCtx != null) { - pipeline.addLast(sslCtx.newHandler(ch.alloc())); - } - - pipeline.addLast(new HttpServerCodec()); - pipeline.addLast(new HttpObjectAggregator(MAX_CONTENT_LENGTH)); - pipeline.addLast(new WebsocketConsumerServerHandler(this.traceRepository, this.properties, this.traceEnabled)); - } - - private SslContext configureSslContext() throws CertificateException, SSLException { - if (this.properties.isSsl()) { - SelfSignedCertificate ssc = new SelfSignedCertificate(); - return SslContextBuilder.forServer(ssc.certificate(), ssc.privateKey()).build(); - } - else { - return null; - } - } -} diff --git a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/actuator/WebsocketConsumerTraceEndpoint.java b/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/actuator/WebsocketConsumerTraceEndpoint.java deleted file mode 100644 index 1609956ad..000000000 --- a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/actuator/WebsocketConsumerTraceEndpoint.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2014-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.websocket.actuator; - -import java.util.List; - -import jakarta.annotation.PostConstruct; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.boot.actuate.endpoint.annotation.Endpoint; -import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.cloud.fn.consumer.websocket.trace.InMemoryTraceRepository; -import org.springframework.cloud.fn.consumer.websocket.trace.Trace; - -/** - * Simple Spring Boot Actuator {@link Endpoint} implementation that - * provides access to Websocket messages last sent/received. - * - * @author Oliver Moser - * @author Artem Bilan - */ -@ConfigurationProperties(prefix = "endpoints.websocketconsumertrace") -@Endpoint(id = "websocketconsumertrace") -public class WebsocketConsumerTraceEndpoint { - - private static final Log logger = LogFactory.getLog(WebsocketConsumerTraceEndpoint.class); - - private boolean enabled; - - private final InMemoryTraceRepository repository; - - public WebsocketConsumerTraceEndpoint(InMemoryTraceRepository repository) { - this.repository = repository; - logger.info(String.format("/websocketsinktrace enabled: %b", this.enabled)); - } - - @PostConstruct - public void init() { - - } - - @ReadOperation - public List traces() { - return this.repository.findAll(); - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - -} diff --git a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/trace/InMemoryTraceRepository.java b/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/trace/InMemoryTraceRepository.java deleted file mode 100644 index bdd47f119..000000000 --- a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/trace/InMemoryTraceRepository.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2018-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.websocket.trace; - -import java.util.Collections; -import java.util.Date; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -/** - * A repository for {@link Trace}s. - * - * It is a copy of {@code InMemoryTraceRepository} from Spring Boot 1.5.x. - * Since Spring Boot 2.0 traces are only available for HTTP. - * - * @author Dave Syer - * @author Olivier Bourgain - * @author Artem Bilan - * - * @since 2.0 - */ -public class InMemoryTraceRepository { - - private int capacity = 100; - - private boolean reverse = true; - - private final List traces = new LinkedList<>(); - - /** - * Flag to say that the repository lists traces in reverse order. - * @param reverse flag value (default true) - */ - public void setReverse(boolean reverse) { - synchronized (this.traces) { - this.reverse = reverse; - } - } - - /** - * Set the capacity of the in-memory repository. - * @param capacity the capacity - */ - public void setCapacity(int capacity) { - synchronized (this.traces) { - this.capacity = capacity; - } - } - - public List findAll() { - synchronized (this.traces) { - return Collections.unmodifiableList(this.traces); - } - } - - public void add(Map map) { - Trace trace = new Trace(new Date(), map); - synchronized (this.traces) { - while (this.traces.size() >= this.capacity) { - this.traces.remove(this.reverse ? this.capacity - 1 : 0); - } - if (this.reverse) { - this.traces.add(0, trace); - } - else { - this.traces.add(trace); - } - } - } -} diff --git a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/trace/Trace.java b/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/trace/Trace.java deleted file mode 100644 index 636fbe3f1..000000000 --- a/functions/consumer/websocket-consumer/src/main/java/org/springframework/cloud/fn/consumer/websocket/trace/Trace.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2018-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.websocket.trace; - -import java.util.Date; -import java.util.Map; - -import org.springframework.util.Assert; - -/** - * A value object representing a trace event: at a particular time with a simple (map) - * information. Can be used for analyzing contextual information such as HTTP headers. - * - *

It is a copy of {@code InMemoryTraceRepository} from Spring Boot 1.5.x. - * Since Spring Boot 2.0 traces are only available for HTTP. - * - * @author Dave Syer - * @author Artem Bilan - * - * @since 2.0 - */ -public class Trace { - - private final Date timestamp; - - private final Map info; - - public Trace(Date timestamp, Map info) { - Assert.notNull(timestamp, "Timestamp must not be null"); - Assert.notNull(info, "Info must not be null"); - this.timestamp = timestamp; - this.info = info; - } - - public Date getTimestamp() { - return this.timestamp; - } - - public Map getInfo() { - return this.info; - } -} diff --git a/functions/consumer/websocket-consumer/src/test/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerTests.java b/functions/consumer/websocket-consumer/src/test/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerTests.java deleted file mode 100644 index 07bcf7c0b..000000000 --- a/functions/consumer/websocket-consumer/src/test/java/org/springframework/cloud/fn/consumer/websocket/WebsocketConsumerTests.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2014-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.websocket; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.function.Consumer; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.test.support.websocket.WebsocketConsumerClientHandler; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.web.socket.WebSocketSession; -import org.springframework.web.socket.client.standard.StandardWebSocketClient; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Oliver Moser - * @author Gary Russell - * @author Artem Bilan - * @author Corneil du Plessis - */ -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = { - "websocket.consumer.port=0", - "websocket.consumer.path=/some_websocket_path", - "websocket.consumer.logLevel=DEBUG", - "websocket.consumer.threads=2" - }) -@DirtiesContext -public class WebsocketConsumerTests { - - public static final int TIMEOUT = 10000; - - public static final int MESSAGE_COUNT = 100; - - public static final int CLIENT_COUNT = 10; - - @Autowired - private WebsocketConsumerProperties properties; - - @Autowired - private WebsocketConsumerServer consumerServer; - - @Autowired - Consumer> websocketConsumer; - - @Test - public void checkCmdlineArgs() { - assertThat(properties.getPath()).isEqualTo("/some_websocket_path"); - assertThat(properties.getPort()).isEqualTo((0)); - assertThat(properties.getLogLevel()).isEqualTo(("DEBUG")); - assertThat(properties.getThreads()).isEqualTo((2)); - } - - @Test - @Timeout(TIMEOUT) - public void testMultipleMessageSingleSubscriber() throws Exception { - WebsocketConsumerClientHandler handler = new WebsocketConsumerClientHandler("handler_0", MESSAGE_COUNT, TIMEOUT); - doHandshake(handler); - - List messagesToSend = submitMultipleMessages(MESSAGE_COUNT); - handler.await(); - - assertThat(handler.getReceivedMessages().size()).isEqualTo(MESSAGE_COUNT); - messagesToSend.forEach(s -> assertThat(handler.getReceivedMessages().contains(s)).isTrue()); - } - - @Test - @Timeout(TIMEOUT) - public void testSingleMessageMultipleSubscribers() throws Exception { - - // create multiple handlers - List handlers = createHandlerList(CLIENT_COUNT, 1); - - // submit a single message - String payload = UUID.randomUUID().toString(); - websocketConsumer.accept(MessageBuilder.withPayload(payload).build()); - - // await completion on each handler - for (WebsocketConsumerClientHandler handler : handlers) { - handler.await(); - assertThat(handler.getReceivedMessages().size()).isEqualTo(1); - assertThat(handler.getReceivedMessages().get(0)).isEqualTo(payload); - } - } - - @Test - @Timeout(TIMEOUT) - public void testMultipleMessagesMultipleSubscribers() throws Exception { - - // create multiple handlers - List handlers = createHandlerList(CLIENT_COUNT, MESSAGE_COUNT); - - // submit mulitple message - List messagesReceived = submitMultipleMessages(MESSAGE_COUNT); - - // wait on each handle - for (WebsocketConsumerClientHandler handler : handlers) { - handler.await(); - assertThat(handler.getReceivedMessages().size()).isEqualTo(messagesReceived.size()); - assertThat(handler.getReceivedMessages()).isEqualTo(messagesReceived); - } - } - - private WebSocketSession doHandshake(WebsocketConsumerClientHandler handler) - throws InterruptedException, ExecutionException { - String wsEndpoint = "ws://localhost:" + this.consumerServer.getPort() + this.properties.getPath(); - return new StandardWebSocketClient().doHandshake(handler, wsEndpoint).get(); - } - - private List submitMultipleMessages(int messageCount) { - List messagesToSend = new ArrayList<>(messageCount); - synchronized (websocketConsumer) { - for (int i = 0; i < messageCount; i++) { - String message = "message_" + i; - messagesToSend.add(message); - websocketConsumer.accept(MessageBuilder.withPayload(message).build()); - } - } - - return messagesToSend; - } - - private List createHandlerList(int handlerCount, int messageCount) throws - InterruptedException, - ExecutionException { - - List handlers = new ArrayList<>(handlerCount); - for (int i = 0; i < handlerCount; i++) { - WebsocketConsumerClientHandler handler = new WebsocketConsumerClientHandler("handler_" + i, messageCount, TIMEOUT); - WebSocketSession session = doHandshake(handler); - assertThat(session.isOpen()); - handlers.add(handler); - } - return handlers; - } - - @SpringBootApplication - public static class WebsocketConsumerTestApplication { - - } -} diff --git a/functions/consumer/xmpp-consumer/README.adoc b/functions/consumer/xmpp-consumer/README.adoc deleted file mode 100644 index 69fb68488..000000000 --- a/functions/consumer/xmpp-consumer/README.adoc +++ /dev/null @@ -1,30 +0,0 @@ -# XMPP Consumer - -A consumer that allows you to send messages through a XMPP server. - -## Beans for injection - -You can import the `XmppConsumerConfiguration` in the application and then inject the following bean. - -`Consumer xmppConsumer` - -You need to inject this as `Consumer xmppConsumer`. - -You can use `xmppConsumer` as a qualifier when injecting. - -**NOTE:** This is a functional endpoint. One will need to subscribe to this endpoint in order to start accepting data -on it. - -## Configuration Options - -All configuration properties are prefixed with `xmpp.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/xmpp/XmppConsumerProperties.java[XmppConsumerProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/xmpp/[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/xmpp-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a XMPP Sink. diff --git a/functions/consumer/xmpp-consumer/pom.xml b/functions/consumer/xmpp-consumer/pom.xml deleted file mode 100644 index 6c860280d..000000000 --- a/functions/consumer/xmpp-consumer/pom.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - xmpp-consumer - xmpp-consumer - XMPP consumer - - - - org.springframework.cloud.fn - xmpp-common - ${project.version} - - - org.springframework.cloud.fn - function-test-support - ${project.version} - test - - - - diff --git a/functions/consumer/xmpp-consumer/src/main/java/org/springframework/cloud/fn/consumer/xmpp/XmppConsumerConfiguration.java b/functions/consumer/xmpp-consumer/src/main/java/org/springframework/cloud/fn/consumer/xmpp/XmppConsumerConfiguration.java deleted file mode 100644 index 9827a5755..000000000 --- a/functions/consumer/xmpp-consumer/src/main/java/org/springframework/cloud/fn/consumer/xmpp/XmppConsumerConfiguration.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2014-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.xmpp; - -import java.util.function.Consumer; - -import org.jivesoftware.smack.XMPPConnection; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.xmpp.XmppConnectionFactoryConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.integration.xmpp.XmppHeaders; -import org.springframework.integration.xmpp.outbound.ChatMessageSendingMessageHandler; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; - -/** - * - * @author Daniel Frey - * @since 4.0.0 - */ -@Configuration -@EnableConfigurationProperties(XmppConsumerProperties.class) -@Import(XmppConnectionFactoryConfiguration.class) -public class XmppConsumerConfiguration { - - @Bean - public ChatMessageSendingMessageHandler chatMessageSendingMessageHandler(XMPPConnection xmppConnection) { - - return new ChatMessageSendingMessageHandler(xmppConnection); - } - - @Bean - public Consumer> xmppConsumer(ChatMessageSendingMessageHandler chatMessageSendingMessageHandler, XmppConsumerProperties properties) { - return message -> { - - var send = MessageBuilder - .fromMessage(message) - .setHeaderIfAbsent(XmppHeaders.TO, properties.getChatTo()) - .build(); - - chatMessageSendingMessageHandler.handleMessage(send); - - }; - } - -} diff --git a/functions/consumer/xmpp-consumer/src/main/java/org/springframework/cloud/fn/consumer/xmpp/XmppConsumerProperties.java b/functions/consumer/xmpp-consumer/src/main/java/org/springframework/cloud/fn/consumer/xmpp/XmppConsumerProperties.java deleted file mode 100644 index 32a44dfb2..000000000 --- a/functions/consumer/xmpp-consumer/src/main/java/org/springframework/cloud/fn/consumer/xmpp/XmppConsumerProperties.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2014-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.xmpp; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * - * @author Daniel Frey - * @since 4.0.0 - */ -@ConfigurationProperties("xmpp.consumer") -@Validated -public class XmppConsumerProperties { - - /** - * XMPP handle to send message to. - */ - private String chatTo; - - public void setChatTo(String chatTo) { - this.chatTo = chatTo; - } - - public String getChatTo() { - return chatTo; - } - -} diff --git a/functions/consumer/xmpp-consumer/src/test/java/org/springframework/cloud/fn/consumer/xmpp/XmppConsumerConfigurationTests.java b/functions/consumer/xmpp-consumer/src/test/java/org/springframework/cloud/fn/consumer/xmpp/XmppConsumerConfigurationTests.java deleted file mode 100644 index 6bb0b6b8d..000000000 --- a/functions/consumer/xmpp-consumer/src/test/java/org/springframework/cloud/fn/consumer/xmpp/XmppConsumerConfigurationTests.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright 2014-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.xmpp; - -import java.io.IOException; -import java.time.Duration; -import java.util.function.Consumer; - -import org.jivesoftware.smack.ConnectionConfiguration; -import org.jivesoftware.smack.SmackException; -import org.jivesoftware.smack.XMPPException; -import org.jivesoftware.smack.filter.StanzaTypeFilter; -import org.jivesoftware.smack.packet.Stanza; -import org.jivesoftware.smack.tcp.XMPPTCPConnection; -import org.jivesoftware.smack.tcp.XMPPTCPConnectionConfiguration; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.jxmpp.stringprep.XmppStringprepException; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.test.support.xmpp.XmppTestContainerSupport; -import org.springframework.context.annotation.Import; -import org.springframework.integration.xmpp.XmppHeaders; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.springframework.cloud.fn.test.support.xmpp.XmppTestContainerSupport.JOHN_USER; -import static org.springframework.cloud.fn.test.support.xmpp.XmppTestContainerSupport.SERVICE_NAME; -import static org.springframework.cloud.fn.test.support.xmpp.XmppTestContainerSupport.USER_PW; - -/** - * @author Daniel Frey - * @author Chris Bono - */ -@SpringBootTest( - properties = { - "xmpp.factory.user=" + JOHN_USER, - "xmpp.factory.password=" + USER_PW, - "xmpp.factory.service-name=" + SERVICE_NAME, - "xmpp.factory.security-mode=disabled" - } -) -@DirtiesContext -public class XmppConsumerConfigurationTests implements XmppTestContainerSupport { - - @DynamicPropertySource - static void registerConfigurationProperties(DynamicPropertyRegistry registry) { - registry.add("xmpp.factory.host", () -> XmppTestContainerSupport.getXmppHost()); - registry.add("xmpp.factory.port", () -> XmppTestContainerSupport.getXmppMappedPort()); - } - - @Autowired - private Consumer> xmppConsumer; - - // A client connection is needed to receive the message from the xmpp server - // to verify it was received successfully - private XMPPTCPConnection clientConnection; - - @BeforeEach - void setup() throws IOException, SmackException, XMPPException, InterruptedException { - - var builder = XMPPTCPConnectionConfiguration.builder(); - builder.setSecurityMode(ConnectionConfiguration.SecurityMode.disabled); - builder.setHost(XmppTestContainerSupport.getXmppHost()); - builder.setPort(XmppTestContainerSupport.getXmppMappedPort()); - builder.setResource(SERVICE_NAME); - builder.setUsernameAndPassword(JANE_USER, USER_PW) - .setXmppDomain(SERVICE_NAME); - this.clientConnection = new XMPPTCPConnection(builder.build()); - this.clientConnection.connect(); - this.clientConnection.login(); - - } - - @AfterEach - void teardown() { - this.clientConnection.instantShutdown(); - } - - @Test - void messageHandlerConfiguration() { - var collector - = this.clientConnection.createStanzaCollector(StanzaTypeFilter.MESSAGE); - - var testMessage = - MessageBuilder.withPayload("test") - .setHeader(XmppHeaders.TO, JANE_USER + "@" + SERVICE_NAME) - .build(); - - await().atMost(Duration.ofSeconds(20)).pollDelay(Duration.ofMillis(100)) - .untilAsserted(() -> { - xmppConsumer.accept(testMessage); - Stanza stanza = collector.nextResult(); - assertStanza(stanza); - }); - } - - @Test - void xmppMessageHandlerConfiguration() throws XmppStringprepException { - var collector - = this.clientConnection.createStanzaCollector(StanzaTypeFilter.MESSAGE); - - var testMessage = - MessageBuilder.withPayload(org.jivesoftware.smack.packet.MessageBuilder.buildMessage().addBody("en_us", "test").to(JANE_USER + "@" + SERVICE_NAME).build()) - .build(); - - await().atMost(Duration.ofSeconds(20)).pollDelay(Duration.ofMillis(100)) - .untilAsserted(() -> { - xmppConsumer.accept(testMessage); - Stanza stanza = collector.nextResult(); - assertStanza(stanza); - }); - } - - private void assertStanza(Stanza stanza) { - assertTo(stanza); - assertFrom(stanza); - } - - private void assertTo(Stanza stanza) { - assertThat(stanza.getTo().asBareJid().asUnescapedString()).isEqualTo(JANE_USER + "@" + SERVICE_NAME); - } - - private void assertFrom(Stanza stanza) { - assertThat(stanza.getFrom().asBareJid().asUnescapedString()).isEqualTo(JOHN_USER + "@" + SERVICE_NAME); - } - - @SpringBootConfiguration - @EnableAutoConfiguration - @Import(XmppConsumerConfiguration.class) - static class XmppConsumerTestApplication { } - -} diff --git a/functions/consumer/zeromq-consumer/README.adoc b/functions/consumer/zeromq-consumer/README.adoc deleted file mode 100644 index 58976d0bb..000000000 --- a/functions/consumer/zeromq-consumer/README.adoc +++ /dev/null @@ -1,30 +0,0 @@ -# ZeroMQ Consumer - -A consumer that allows you to send messages through a ZeroMQ socker. - -## Beans for injection - -You can import the `ZeroMqConsumerConfiguration` in the application and then inject the following bean. - -`Function>, Mono> zeromqConsumer` - -You need to inject this as `Function>, Mono> zeromqConsumer`. - -You can use `zeromqConsumer` as a qualifier when injecting. - -**NOTE:** This is a functional endpoint. One will need to subscribe to this endpoint in order to start accepting data -on it. - -## Configuration Options - -All configuration properties are prefixed with `zeromq.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/zeromq/ZeroMqConsumerProperties.java[ZeroMqConsumerProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/zeromq/[test suite] for the various ways, this consumer is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/sink/zeromq-sink/README.adoc[README] where this consumer is used to create a Spring Cloud Stream application where it makes a ZeroMQ Sink. diff --git a/functions/consumer/zeromq-consumer/pom.xml b/functions/consumer/zeromq-consumer/pom.xml deleted file mode 100644 index 6832a3d0f..000000000 --- a/functions/consumer/zeromq-consumer/pom.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - zeromq-consumer - zeromq-consumer - ZeroMQ consumer - - - - org.springframework.integration - spring-integration-zeromq - - - - diff --git a/functions/consumer/zeromq-consumer/src/main/java/org/springframework/cloud/fn/consumer/zeromq/ZeroMqConsumerConfiguration.java b/functions/consumer/zeromq-consumer/src/main/java/org/springframework/cloud/fn/consumer/zeromq/ZeroMqConsumerConfiguration.java deleted file mode 100644 index 035ba3f71..000000000 --- a/functions/consumer/zeromq-consumer/src/main/java/org/springframework/cloud/fn/consumer/zeromq/ZeroMqConsumerConfiguration.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.zeromq; - -import java.util.function.Consumer; -import java.util.function.Function; - -import org.zeromq.ZContext; -import org.zeromq.ZMQ; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.mapping.OutboundMessageMapper; -import org.springframework.integration.zeromq.outbound.ZeroMqMessageHandler; -import org.springframework.messaging.Message; - -/** - * - * @author Daniel Frey - * @since 3.1.0 - */ -@Configuration -@EnableConfigurationProperties(ZeroMqConsumerProperties.class) -public class ZeroMqConsumerConfiguration { - - @Bean - public ZContext zContext() { - return new ZContext(); - } - - @Bean - public ZeroMqMessageHandler zeromqMessageHandler(ZeroMqConsumerProperties properties, ZContext zContext, - @Autowired(required = false) Consumer socketConfigurer, - @Autowired(required = false) OutboundMessageMapper messageMapper) { - ZeroMqMessageHandler zeroMqMessageHandler = new ZeroMqMessageHandler(zContext, properties.getConnectUrl(), - properties.getSocketType()); - - if (properties.getTopic() != null) { - zeroMqMessageHandler.setTopicExpression(properties.getTopic()); - } - - if (socketConfigurer != null) { - zeroMqMessageHandler.setSocketConfigurer(socketConfigurer); - } - - if (messageMapper != null) { - zeroMqMessageHandler.setMessageMapper(messageMapper); - } - - return zeroMqMessageHandler; - } - - @Bean - public Function>, Mono> zeromqConsumer(ZeroMqMessageHandler zeromqMessageHandler) { - return input -> input.flatMap(zeromqMessageHandler::handleMessage) - .ignoreElements(); - } - -} diff --git a/functions/consumer/zeromq-consumer/src/main/java/org/springframework/cloud/fn/consumer/zeromq/ZeroMqConsumerProperties.java b/functions/consumer/zeromq-consumer/src/main/java/org/springframework/cloud/fn/consumer/zeromq/ZeroMqConsumerProperties.java deleted file mode 100644 index 330968f27..000000000 --- a/functions/consumer/zeromq-consumer/src/main/java/org/springframework/cloud/fn/consumer/zeromq/ZeroMqConsumerProperties.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.zeromq; - -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; -import org.zeromq.SocketType; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.validation.annotation.Validated; - -/** - * - * @author Daniel Frey - * @since 3.1.0 - */ -@ConfigurationProperties("zeromq.consumer") -@Validated -public class ZeroMqConsumerProperties { - - /** - * The Socket Type the connection should establish. - */ - private SocketType socketType = SocketType.PUB; - - /** - * Connection URL for connecting to the ZeroMQ Socket. - */ - private String connectUrl; - - /** - * A Topic SpEL expression to evaluate a topic before sending messages to subscribers. - */ - private Expression topic; - - @NotNull(message = "'socketType' is required") - public SocketType getSocketType() { - return socketType; - } - - /** - * @param socketType the {@link SocketType} to establish. - */ - public void setSocketType(SocketType socketType) { - this.socketType = socketType; - } - - @NotEmpty(message = "connectUrl is required like protocol://server:port") - public String getConnectUrl() { - return connectUrl; - } - - /** - * @param connectUrl The ZeroMQ socket to expose - */ - public void setConnectUrl(String connectUrl) { - this.connectUrl = connectUrl; - } - - public Expression getTopic() { - return topic; - } - - /** - * @param topic The 'topic' SpEL expression to set - */ - public void setTopic(Expression topic) { - this.topic = topic; - } - -} diff --git a/functions/consumer/zeromq-consumer/src/test/java/org/springframework/cloud/fn/consumer/zeromq/ZeroMqConsumerConfigurationTests.java b/functions/consumer/zeromq-consumer/src/test/java/org/springframework/cloud/fn/consumer/zeromq/ZeroMqConsumerConfigurationTests.java deleted file mode 100644 index 445c3986a..000000000 --- a/functions/consumer/zeromq-consumer/src/test/java/org/springframework/cloud/fn/consumer/zeromq/ZeroMqConsumerConfigurationTests.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2016-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.consumer.zeromq; - -import java.time.Duration; -import java.util.function.Function; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.zeromq.SocketType; -import org.zeromq.ZContext; -import org.zeromq.ZMQ; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; - -/** - * @author Daniel Frey - * @author Artem Bilan - * - * @since 3.1 - */ -@SpringBootTest(properties = "zeromq.consumer.topic='test-topic'") -@DirtiesContext -public class ZeroMqConsumerConfigurationTests { - - private static final ZContext CONTEXT = new ZContext(); - - private static ZMQ.Socket socket; - - @Autowired - Function>, Mono> subject; - - @BeforeAll - static void setup() { - - socket = CONTEXT.createSocket(SocketType.SUB); - socket.setReceiveTimeOut(10_000); - int bindPort = socket.bindToRandomPort("tcp://*"); - System.setProperty("zeromq.consumer.connectUrl", "tcp://localhost:" + bindPort); - } - - @AfterAll - static void tearDown() { - socket.close(); - CONTEXT.close(); - } - - @Test - void testMessageHandlerConfiguration() { - Message testMessage = MessageBuilder.withPayload("test").setHeader("topic", "test-topic").build(); - - await().atMost(Duration.ofSeconds(20)).pollDelay(Duration.ofMillis(100)) - .untilAsserted(() -> { - socket.subscribe("test-topic"); - subject.apply(Flux.just(testMessage)).subscribe(); - String topic = socket.recvStr(); - assertThat(topic).isEqualTo("test-topic"); - assertThat(socket.recvStr()).isEmpty(); - assertThat(socket.recvStr()).isEqualTo("test"); - }); - - } - - @SpringBootApplication - public static class ZeroMqConsumerTestApplication { - - } - -} diff --git a/functions/function-dependencies/pom.xml b/functions/function-dependencies/pom.xml deleted file mode 100644 index 30b0a52de..000000000 --- a/functions/function-dependencies/pom.xml +++ /dev/null @@ -1,310 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.stream.app - stream-applications-build - 5.0.0-SNAPSHOT - ../../stream-applications-build/pom.xml - - - org.springframework.cloud.fn - function-dependencies - pom - - - - - org.springframework.cloud.fn - mail-supplier - ${project.version} - - - org.springframework.cloud.fn - syslog-supplier - ${project.version} - - - org.springframework.cloud.fn - debezium-supplier - ${project.version} - - - org.springframework.cloud.fn - file-supplier - ${project.version} - - - org.springframework.cloud.fn - ftp-supplier - ${project.version} - - - org.springframework.cloud.fn - http-supplier - ${project.version} - - - org.springframework.cloud.fn - jdbc-supplier - ${project.version} - - - org.springframework.cloud.fn - jms-supplier - ${project.version} - - - org.springframework.cloud.fn - kafka-supplier - ${project.version} - - - org.springframework.cloud.fn - mongodb-supplier - ${project.version} - - - org.springframework.cloud.fn - mqtt-supplier - ${project.version} - - - org.springframework.cloud.fn - rabbit-supplier - ${project.version} - - - org.springframework.cloud.fn - s3-supplier - ${project.version} - - - org.springframework.cloud.fn - sftp-supplier - ${project.version} - - - org.springframework.cloud.fn - tcp-supplier - ${project.version} - - - org.springframework.cloud.fn - time-supplier - ${project.version} - - - org.springframework.cloud.fn - twitter-supplier - ${project.version} - - - org.springframework.cloud.fn - websocket-supplier - ${project.version} - - - org.springframework.cloud.fn - xmpp-supplier - ${project.version} - - - org.springframework.cloud.fn - zeromq-supplier - ${project.version} - - - org.springframework.cloud.fn - analytics-consumer - ${project.version} - - - org.springframework.cloud.fn - elasticsearch-consumer - ${project.version} - - - org.springframework.cloud.fn - rsocket-consumer - ${project.version} - - - org.springframework.cloud.fn - cassandra-consumer - ${project.version} - - - org.springframework.cloud.fn - file-consumer - ${project.version} - - - org.springframework.cloud.fn - ftp-consumer - ${project.version} - - - org.springframework.cloud.fn - jdbc-consumer - ${project.version} - - - org.springframework.cloud.fn - kafka-publisher - ${project.version} - - - org.springframework.cloud.fn - log-consumer - ${project.version} - - - org.springframework.cloud.fn - mongodb-consumer - ${project.version} - - - org.springframework.cloud.fn - mqtt-consumer - ${project.version} - - - org.springframework.cloud.fn - rabbit-consumer - ${project.version} - - - org.springframework.cloud.fn - redis-consumer - ${project.version} - - - org.springframework.cloud.fn - s3-consumer - ${project.version} - - - org.springframework.cloud.fn - sftp-consumer - ${project.version} - - - org.springframework.cloud.fn - tcp-consumer - ${project.version} - - - org.springframework.cloud.fn - twitter-consumer - ${project.version} - - - org.springframework.cloud.fn - xmpp-consumer - ${project.version} - - - org.springframework.cloud.fn - wavefront-consumer - ${project.version} - - - org.springframework.cloud.fn - websocket-consumer - ${project.version} - - - org.springframework.cloud.fn - zeromq-consumer - ${project.version} - - - org.springframework.cloud.fn - aggregator-function - ${project.version} - - - org.springframework.cloud.fn - filter-function - ${project.version} - - - org.springframework.cloud.fn - header-enricher-function - ${project.version} - - - org.springframework.cloud.fn - header-filter-function - ${project.version} - - - org.springframework.cloud.fn - http-request-function - ${project.version} - - - org.springframework.cloud.fn - image-recognition-function - ${project.version} - - - org.springframework.cloud.fn - object-detection-function - ${project.version} - - - org.springframework.cloud.fn - payload-converter-function - ${project.version} - - - org.springframework.cloud.fn - semantic-segmentation-function - ${project.version} - - - org.springframework.cloud.fn - spel-function - ${project.version} - - - org.springframework.cloud.fn - splitter-function - ${project.version} - - - org.springframework.cloud.fn - task-launch-request-function - ${project.version} - - - org.springframework.cloud.fn - twitter-function - ${project.version} - - - org.springframework.cloud.fn - function-test-support - ${project.version} - test - - - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.0.0 - - false - - - - - diff --git a/functions/function/aggregator-function/README.adoc b/functions/function/aggregator-function/README.adoc deleted file mode 100644 index 753fd1f73..000000000 --- a/functions/function/aggregator-function/README.adoc +++ /dev/null @@ -1,27 +0,0 @@ -# Aggregator Function - -This module provides an aggregation function that can be reused and composed in other applications. - -## Beans for injection - -You can import the `AggregatorFunctionConfiguration` in a Spring Boot application and then inject the following bean. - -`aggregatorFunction` - -You can use `aggregatorFunction` as a qualifier when injecting. - -Once injected, you can use the `apply` method of the `Function` to invoke it and get the result. - -## Configuration Options - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/aggregator/AggregatorFunctionProperties.java[AggregatorFunctionProperties.java] - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `AggregatorFactoryBean` configuration used by the `aggregatorFunction` definition. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/aggregator/AggregatorFunctionApplicationTests.java[test suite] for examples of how this function is used. - -## Other usage - -See this link:../../../applications/processor/aggregator-processor/README.adoc[README] where this function is used to create a Spring Cloud Stream application. diff --git a/functions/function/aggregator-function/pom.xml b/functions/function/aggregator-function/pom.xml deleted file mode 100644 index 4a038671d..000000000 --- a/functions/function/aggregator-function/pom.xml +++ /dev/null @@ -1,99 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - aggregator-function - aggregator-function - Spring Native Function for Aggregator - - - - org.springframework.cloud.fn - payload-converter-function - ${project.version} - - - org.springframework.integration - spring-integration-core - - - - org.springframework.integration - spring-integration-mongodb - - - org.springframework.boot - spring-boot-starter-data-mongodb - runtime - - - org.testcontainers - mongodb - ${testcontainers.version} - test - - - org.springframework.cloud.fn - mongodb-common - ${project.version} - test-jar - test - - - - org.springframework.integration - spring-integration-redis - - - org.springframework.boot - spring-boot-starter-data-redis - runtime - - - org.springframework.cloud.fn - redis-common - ${project.version} - test-jar - test - - - - org.springframework.integration - spring-integration-jdbc - - - org.springframework.boot - spring-boot-starter-jdbc - runtime - - - org.hsqldb - hsqldb - runtime - - - com.h2database - h2 - runtime - - - org.mariadb.jdbc - mariadb-java-client - runtime - - - org.postgresql - postgresql - runtime - - - - diff --git a/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/AggregatorFunctionConfiguration.java b/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/AggregatorFunctionConfiguration.java deleted file mode 100644 index d89077330..000000000 --- a/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/AggregatorFunctionConfiguration.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright 2020-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.aggregator; - -import java.util.function.Function; - -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.BeanFactoryAware; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.integration.aggregator.CorrelationStrategy; -import org.springframework.integration.aggregator.DefaultAggregatingMessageGroupProcessor; -import org.springframework.integration.aggregator.ExpressionEvaluatingCorrelationStrategy; -import org.springframework.integration.aggregator.ExpressionEvaluatingMessageGroupProcessor; -import org.springframework.integration.aggregator.ExpressionEvaluatingReleaseStrategy; -import org.springframework.integration.aggregator.MessageGroupProcessor; -import org.springframework.integration.aggregator.ReleaseStrategy; -import org.springframework.integration.annotation.ServiceActivator; -import org.springframework.integration.channel.FluxMessageChannel; -import org.springframework.integration.config.AggregatorFactoryBean; -import org.springframework.integration.store.MessageGroupStore; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageChannel; - -/** - * @author Artem Bilan - * @author Corneil du Plessis - */ -@AutoConfiguration -@EnableConfigurationProperties(AggregatorFunctionProperties.class) -public class AggregatorFunctionConfiguration { - - @Autowired - private AggregatorFunctionProperties properties; - - @Autowired - private BeanFactory beanFactory; - - @Bean - public Function>, Flux>> aggregatorFunction( - FluxMessageChannel inputChannel, - FluxMessageChannel outputChannel - ) { - return input -> Flux.from(outputChannel) - .doOnRequest((request) -> - inputChannel.subscribeTo( - input.map((inputMessage) -> - MessageBuilder.fromMessage(inputMessage) - .removeHeader("kafka_consumer") - .build()))); - } - - @Bean - public FluxMessageChannel inputChannel() { - return new FluxMessageChannel(); - } - - @Bean - public FluxMessageChannel outputChannel() { - return new FluxMessageChannel(); - } - - @Bean - @ServiceActivator(inputChannel = "inputChannel") - public AggregatorFactoryBean aggregator( - @Nullable CorrelationStrategy correlationStrategy, - @Nullable ReleaseStrategy releaseStrategy, - @Nullable MessageGroupProcessor messageGroupProcessor, - @Nullable MessageGroupStore messageStore, - @Qualifier("outputChannel") MessageChannel outputChannel, - @Nullable ComponentCustomizer aggregatorCustomizer) { - - AggregatorFactoryBean aggregator = new AggregatorFactoryBean(); - aggregator.setExpireGroupsUponCompletion(true); - aggregator.setSendPartialResultOnExpiry(true); - aggregator.setGroupTimeoutExpression(this.properties.getGroupTimeout()); - - if (correlationStrategy != null) { - aggregator.setCorrelationStrategy(correlationStrategy); - } - if (releaseStrategy != null) { - aggregator.setReleaseStrategy(releaseStrategy); - } - - MessageGroupProcessor groupProcessor = messageGroupProcessor; - - if (groupProcessor == null) { - groupProcessor = new DefaultAggregatingMessageGroupProcessor(); - ((BeanFactoryAware) groupProcessor).setBeanFactory(this.beanFactory); - } - aggregator.setProcessorBean(groupProcessor); - - if (messageStore != null) { - aggregator.setMessageStore(messageStore); - } - aggregator.setOutputChannel(outputChannel); - - if (aggregatorCustomizer != null) { - aggregatorCustomizer.customize(aggregator); - } - - return aggregator; - } - - @Bean - @ConditionalOnProperty(prefix = AggregatorFunctionProperties.PREFIX, name = "correlation") - @ConditionalOnMissingBean - public CorrelationStrategy correlationStrategy() { - return new ExpressionEvaluatingCorrelationStrategy(this.properties.getCorrelation()); - } - - @Bean - @ConditionalOnProperty(prefix = AggregatorFunctionProperties.PREFIX, name = "release") - @ConditionalOnMissingBean - public ReleaseStrategy releaseStrategy() { - return new ExpressionEvaluatingReleaseStrategy(this.properties.getRelease()); - } - - @Bean - @ConditionalOnProperty(prefix = AggregatorFunctionProperties.PREFIX, name = "aggregation") - @ConditionalOnMissingBean - public MessageGroupProcessor messageGroupProcessor() { - return new ExpressionEvaluatingMessageGroupProcessor(this.properties.getAggregation().getExpressionString()); - } - - - @Configuration - @ConditionalOnMissingBean(MessageGroupStore.class) - @Import({ - MessageStoreConfiguration.Mongo.class, - MessageStoreConfiguration.Redis.class, - MessageStoreConfiguration.Jdbc.class - }) - protected static class MessageStoreAutoConfiguration { - - } - -} diff --git a/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/AggregatorFunctionProperties.java b/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/AggregatorFunctionProperties.java deleted file mode 100644 index 0f41ff628..000000000 --- a/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/AggregatorFunctionProperties.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.aggregator; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; - -/** - * Configuration properties for the Aggregator function. - * - * @author Artem Bilan - */ -@ConfigurationProperties("aggregator") -public class AggregatorFunctionProperties { - - static final String PREFIX = "aggregator"; - - /** - * SpEL expression for correlation key. Default to correlationId header. - */ - private Expression correlation; - - /** - * SpEL expression for release strategy. Default is based on the sequenceSize header. - */ - private Expression release; - - /** - * SpEL expression for aggregation strategy. Default is collection of payloads. - */ - private Expression aggregation; - - /** - * SpEL expression for timeout to expiring uncompleted groups. - */ - private Expression groupTimeout; - - /** - * Message store type. - */ - private String messageStoreType = MessageStoreType.SIMPLE; - - /** - * Persistence message store entity: table prefix in RDBMS, collection name in MongoDb, etc. - */ - private String messageStoreEntity; - - public Expression getCorrelation() { - return this.correlation; - } - - public void setCorrelation(Expression correlation) { - this.correlation = correlation; - } - - public Expression getRelease() { - return this.release; - } - - public void setRelease(Expression release) { - this.release = release; - } - - public Expression getAggregation() { - return this.aggregation; - } - - public void setAggregation(Expression aggregation) { - this.aggregation = aggregation; - } - - public Expression getGroupTimeout() { - return this.groupTimeout; - } - - public void setGroupTimeout(Expression groupTimeout) { - this.groupTimeout = groupTimeout; - } - - public String getMessageStoreEntity() { - return this.messageStoreEntity; - } - - public void setMessageStoreEntity(String messageStoreEntity) { - this.messageStoreEntity = messageStoreEntity; - } - - public String getMessageStoreType() { - return this.messageStoreType; - } - - public void setMessageStoreType(String messageStoreType) { - this.messageStoreType = messageStoreType; - } - - static final class MessageStoreType { - - static final String SIMPLE = "simple"; - - static final String JDBC = "jdbc"; - - static final String MONGODB = "mongodb"; - - static final String REDIS = "redis"; - - } - -} diff --git a/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/ExcludeStoresAutoConfigurationEnvironmentPostProcessor.java b/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/ExcludeStoresAutoConfigurationEnvironmentPostProcessor.java deleted file mode 100644 index 60614059c..000000000 --- a/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/ExcludeStoresAutoConfigurationEnvironmentPostProcessor.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.aggregator; - -import java.util.Properties; - -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; -import org.springframework.boot.autoconfigure.data.mongo.MongoRepositoriesAutoConfiguration; -import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; -import org.springframework.boot.autoconfigure.data.redis.RedisRepositoriesAutoConfiguration; -import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; -import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; -import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; -import org.springframework.boot.env.EnvironmentPostProcessor; -import org.springframework.core.env.ConfigurableEnvironment; -import org.springframework.core.env.MutablePropertySources; -import org.springframework.core.env.PropertiesPropertySource; - -/** - * An {@link EnvironmentPostProcessor} to add {@code spring.autoconfigure.exclude} property - * since we can't use {@code application.properties} from the library perspective. - * - * @author Artem Bilan - * @author Corneil du Plessis - */ -public class ExcludeStoresAutoConfigurationEnvironmentPostProcessor implements EnvironmentPostProcessor { - - @Override - public void postProcessEnvironment(ConfigurableEnvironment environment, SpringApplication application) { - MutablePropertySources propertySources = environment.getPropertySources(); - Properties properties = new Properties(); - - properties.setProperty("spring.autoconfigure.exclude", - DataSourceAutoConfiguration.class.getName() + ", " + - DataSourceTransactionManagerAutoConfiguration.class.getName() + ", " + - MongoAutoConfiguration.class.getName() + ", " + - MongoDataAutoConfiguration.class.getName() + ", " + - MongoRepositoriesAutoConfiguration.class.getName() + ", " + - RedisAutoConfiguration.class.getName() + ", " + - RedisRepositoriesAutoConfiguration.class.getName()); - - propertySources.addLast(new PropertiesPropertySource("aggregator.exclude.stores.auto-configuration", properties)); - } - -} diff --git a/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/MessageStoreConfiguration.java b/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/MessageStoreConfiguration.java deleted file mode 100644 index c823a8c31..000000000 --- a/functions/function/aggregator-function/src/main/java/org/springframework/cloud/fn/aggregator/MessageStoreConfiguration.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.aggregator; - -import java.util.Arrays; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration; -import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; -import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; -import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration; -import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.convert.MongoCustomConversions; -import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.integration.jdbc.store.JdbcMessageStore; -import org.springframework.integration.mongodb.store.ConfigurableMongoDbMessageStore; -import org.springframework.integration.mongodb.support.BinaryToMessageConverter; -import org.springframework.integration.mongodb.support.MessageToBinaryConverter; -import org.springframework.integration.redis.store.RedisMessageStore; -import org.springframework.integration.store.MessageGroupStore; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.util.StringUtils; - - -/** - * A helper class containing configuration classes for particular technologies - * to expose an appropriate {@link org.springframework.integration.store.MessageStore} bean - * via matched configuration properties. - * - * @author Artem Bilan - * @author Corneil du Plessis - */ -class MessageStoreConfiguration { - - @ConditionalOnClass(ConfigurableMongoDbMessageStore.class) - @ConditionalOnProperty(prefix = AggregatorFunctionProperties.PREFIX, - name = "message-store-type", - havingValue = AggregatorFunctionProperties.MessageStoreType.MONGODB) - @Import({ MongoAutoConfiguration.class, - MongoDataAutoConfiguration.class }) - static class Mongo { - - @Bean - public MessageGroupStore messageStore(MongoTemplate mongoTemplate, AggregatorFunctionProperties properties) { - if (StringUtils.hasText(properties.getMessageStoreEntity())) { - return new ConfigurableMongoDbMessageStore(mongoTemplate, properties.getMessageStoreEntity()); - } - else { - return new ConfigurableMongoDbMessageStore(mongoTemplate); - } - } - - @Bean - @Primary - public MongoCustomConversions mongoDbCustomConversions() { - return new MongoCustomConversions(Arrays.asList( - new MessageToBinaryConverter(), new BinaryToMessageConverter())); - } - - } - - @ConditionalOnClass(RedisMessageStore.class) - @ConditionalOnProperty(prefix = AggregatorFunctionProperties.PREFIX, - name = "message-store-type", - havingValue = AggregatorFunctionProperties.MessageStoreType.REDIS) - @Import(RedisAutoConfiguration.class) - static class Redis { - - @Bean - public MessageGroupStore messageStore(RedisTemplate redisTemplate) { - return new RedisMessageStore(redisTemplate.getConnectionFactory()); - } - - } - - @ConditionalOnClass(JdbcMessageStore.class) - @ConditionalOnProperty(prefix = AggregatorFunctionProperties.PREFIX, - name = "message-store-type", - havingValue = AggregatorFunctionProperties.MessageStoreType.JDBC) - @Import({ - DataSourceAutoConfiguration.class, - DataSourceTransactionManagerAutoConfiguration.class }) - static class Jdbc { - - @Bean - public MessageGroupStore messageStore(JdbcTemplate jdbcTemplate, AggregatorFunctionProperties properties) { - JdbcMessageStore messageStore = new JdbcMessageStore(jdbcTemplate); - if (StringUtils.hasText(properties.getMessageStoreEntity())) { - messageStore.setTablePrefix(properties.getMessageStoreEntity()); - } - return messageStore; - } - - } - -} diff --git a/functions/function/aggregator-function/src/main/resources/META-INF/spring.factories b/functions/function/aggregator-function/src/main/resources/META-INF/spring.factories deleted file mode 100644 index 7211caa48..000000000 --- a/functions/function/aggregator-function/src/main/resources/META-INF/spring.factories +++ /dev/null @@ -1,2 +0,0 @@ -org.springframework.boot.env.EnvironmentPostProcessor=\ - org.springframework.cloud.fn.aggregator.ExcludeStoresAutoConfigurationEnvironmentPostProcessor diff --git a/functions/function/aggregator-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/function/aggregator-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index 8c697c53c..000000000 --- a/functions/function/aggregator-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.aggregator.AggregatorFunctionConfiguration diff --git a/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/AbstractAggregatorFunctionTests.java b/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/AbstractAggregatorFunctionTests.java deleted file mode 100644 index 362d71145..000000000 --- a/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/AbstractAggregatorFunctionTests.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.aggregator; - -import java.util.function.Function; - -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.integration.aggregator.AggregatingMessageHandler; -import org.springframework.integration.store.MessageGroupStore; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; - -/** - * @author Artem Bilan - * @author Corneil du Plessis - */ -@SpringBootTest -@DirtiesContext -public abstract class AbstractAggregatorFunctionTests { - - @Autowired - protected Function>, Flux>> aggregatorFunction; - - @Autowired(required = false) - protected MessageGroupStore messageGroupStore; - - @Autowired - protected AggregatingMessageHandler aggregatingMessageHandler; - - @SpringBootApplication - public static class AggregatorFunctionTestApplication { - public static void main(String[] args) { - SpringApplication.run(AggregatorFunctionTestApplication.class, args); - } - } - -} diff --git a/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/CustomPropsAndMongoMessageStoreAggregatorTests.java b/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/CustomPropsAndMongoMessageStoreAggregatorTests.java deleted file mode 100644 index abbc4b66f..000000000 --- a/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/CustomPropsAndMongoMessageStoreAggregatorTests.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.aggregator; - -import java.time.Duration; -import java.util.List; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo; -import org.springframework.cloud.fn.common.mongo.MongoDbTestContainerSupport; -import org.springframework.integration.mongodb.store.ConfigurableMongoDbMessageStore; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Artem Bilan - */ -@TestPropertySource(properties = { - "aggregator.correlation=T(Thread).currentThread().id", - "aggregator.release=!messages.?[payload == 'bar'].empty", - "aggregator.aggregation=#this.?[payload == 'foo'].![payload]", - "aggregator.messageStoreType=mongodb", - "aggregator.message-store-entity=aggregatorTest" -}) -@AutoConfigureDataMongo -public class CustomPropsAndMongoMessageStoreAggregatorTests extends AbstractAggregatorFunctionTests - implements MongoDbTestContainerSupport { - - @DynamicPropertySource - static void mongoDbProperties(DynamicPropertyRegistry registry) { - registry.add("spring.data.mongodb.port", MONGO_CONTAINER::getFirstMappedPort); - registry.add("spring.data.mongodb.database", () -> "test"); - } - - @Test - public void test() { - Flux> input = - Flux.just("foo", "bar") - .map(GenericMessage::new); - - Flux> output = this.aggregatorFunction.apply(input); - - output.as(StepVerifier::create) - .assertNext((message) -> - assertThat(message) - .extracting(Message::getPayload) - .isInstanceOf(List.class) - .asList() - .hasSize(1) - .element(0).isEqualTo("foo")) - .thenCancel() - .verify(Duration.ofSeconds(10)); - - assertThat(this.messageGroupStore).isInstanceOf(ConfigurableMongoDbMessageStore.class); - assertThat(TestUtils.getPropertyValue(this.messageGroupStore, "collectionName")).isEqualTo("aggregatorTest"); - assertThat(this.aggregatingMessageHandler.getMessageStore()).isSameAs(this.messageGroupStore); - } - -} diff --git a/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/DefaultAggregatorTests.java b/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/DefaultAggregatorTests.java deleted file mode 100644 index 06d365706..000000000 --- a/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/DefaultAggregatorTests.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.aggregator; - -import java.time.Duration; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.integration.IntegrationMessageHeaderAccessor; -import org.springframework.integration.store.SimpleMessageStore; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Artem Bilan - * @author Corneil du Plessis - */ -@Disabled("Fails on CI sporadically") -@TestPropertySource(properties = "aggregator.message-store-type=simple") -public class DefaultAggregatorTests extends AbstractAggregatorFunctionTests { - private static final Logger logger = LoggerFactory.getLogger(DefaultAggregatorTests.class); - - @Test - public void test() { - Flux> input = - Flux.just(MessageBuilder.withPayload("2") - .setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, "my_correlation") - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 2) - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 2) - .build(), - MessageBuilder.withPayload("1") - .setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, "my_correlation") - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 1) - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 2) - .build()); - - Flux> output = this.aggregatorFunction.apply(input.log("DefaultAggregatorTests:input")); - output.log("DefaultAggregatorTests:output") - .as(StepVerifier::create) - .assertNext((message) -> { - assertThat(message) - .extracting(Message::getPayload) - .asList() - .hasSize(2) - .contains("1", "2"); - }) - .thenCancel() - .verify(Duration.ofSeconds(30)); - - assertThat(this.messageGroupStore).isNull(); - assertThat(this.aggregatingMessageHandler.getMessageStore()).isInstanceOf(SimpleMessageStore.class); - - } -} diff --git a/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/JdbcMessageStoreAggregatorTests.java b/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/JdbcMessageStoreAggregatorTests.java deleted file mode 100644 index d009270fa..000000000 --- a/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/JdbcMessageStoreAggregatorTests.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.aggregator; - -import java.time.Duration; -import java.util.List; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.integration.IntegrationMessageHeaderAccessor; -import org.springframework.integration.jdbc.store.JdbcMessageStore; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Artem Bilan - * @author Corneil du Plessis - */ -@TestPropertySource(properties = "aggregator.message-store-type=jdbc") -public class JdbcMessageStoreAggregatorTests extends AbstractAggregatorFunctionTests { - - @Test - public void test() { - Flux> input = - Flux.just(MessageBuilder.withPayload("2") - .setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, "my_correlation") - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 2) - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 2) - .build(), - MessageBuilder.withPayload("1") - .setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, "my_correlation") - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 1) - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 2) - .build()); - - Flux> output = this.aggregatorFunction.apply(input); - - output.as(StepVerifier::create) - .assertNext((message) -> - assertThat(message) - .extracting(Message::getPayload) - .isInstanceOf(List.class) - .asList() - .hasSize(2) - .contains("1", "2")) - .thenCancel() - .verify(Duration.ofSeconds(10)); - - assertThat(this.messageGroupStore).isInstanceOf(JdbcMessageStore.class); - - assertThat(this.aggregatingMessageHandler.getMessageStore()).isSameAs(this.messageGroupStore); - } - -} diff --git a/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/RedisMessageStoreAggregatorTests.java b/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/RedisMessageStoreAggregatorTests.java deleted file mode 100644 index feb39c26a..000000000 --- a/functions/function/aggregator-function/src/test/java/org/springframework/cloud/fn/aggregator/RedisMessageStoreAggregatorTests.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2020-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.aggregator; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.time.Duration; -import java.util.List; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.aop.framework.ProxyFactory; -import org.springframework.cloud.fn.consumer.redis.RedisTestContainerSupport; -import org.springframework.integration.IntegrationMessageHeaderAccessor; -import org.springframework.integration.redis.store.RedisMessageStore; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Artem Bilan - */ -@TestPropertySource(properties = "aggregator.message-store-type=redis") -public class RedisMessageStoreAggregatorTests extends AbstractAggregatorFunctionTests implements RedisTestContainerSupport { - @DynamicPropertySource - static void redisProperties(DynamicPropertyRegistry registry) { - registry.add("spring.data.redis.url", RedisTestContainerSupport::getUri); - } - - @Test - public void test() { - InputStream fakeNonSerializableKafkaConsumer = new ByteArrayInputStream(new byte[0]); - - Flux> input = - Flux.just(MessageBuilder.withPayload("2") - .setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, "my_correlation") - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 2) - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 2) - .setHeader("kafka_consumer", new ProxyFactory(fakeNonSerializableKafkaConsumer).getProxy()) - .build(), - MessageBuilder.withPayload("1") - .setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, "my_correlation") - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 1) - .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 2) - .build()); - - Flux> output = this.aggregatorFunction.apply(input); - - output.as(StepVerifier::create) - .assertNext((message) -> - assertThat(message) - .extracting(Message::getPayload) - .isInstanceOf(List.class) - .asList() - .hasSize(2) - .contains("1", "2")) - .thenCancel() - .verify(Duration.ofSeconds(10)); - - assertThat(this.messageGroupStore).isInstanceOf(RedisMessageStore.class); - - assertThat(this.aggregatingMessageHandler.getMessageStore()).isSameAs(this.messageGroupStore); - } - -} diff --git a/functions/function/filter-function/README.adoc b/functions/function/filter-function/README.adoc deleted file mode 100644 index 2c9917cf0..000000000 --- a/functions/function/filter-function/README.adoc +++ /dev/null @@ -1,27 +0,0 @@ -# Filter Function - -This module provides a filter function that can be reused and composed in other applications. - -## Beans for injection - -You can import the `FilterFunctionConfiguration` in a Spring Boot application and then inject the following bean. - -`filterFunction` - -You can use `filterFunction` as a qualifier when injecting. - -Once injected, you can use the `apply` method of the `Function` to invoke it and get the result. - -## Configuration Options - -All configuration properties are prefixed with `filter.function`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/filter/FilterFunctionProperties.java[FilterFunctionProperties.java] - -## Examples - -See this link:src/test/java/org/springframework/cloud/fn/filter/FilterFunctionApplicationTests.java[test suite] for examples of how this function is used. - -## Other usage - -See this link:../../../applications/processor/filter-processor/README.adoc[README] where this function is used to create a Spring Cloud Stream application. \ No newline at end of file diff --git a/functions/function/filter-function/pom.xml b/functions/function/filter-function/pom.xml deleted file mode 100644 index ec8cc5d4d..000000000 --- a/functions/function/filter-function/pom.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - filter-function - filter-function - Spring Native Function for applying filter SpEL expressions - - - - org.springframework.cloud.fn - payload-converter-function - ${project.version} - - - - diff --git a/functions/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionConfiguration.java b/functions/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionConfiguration.java deleted file mode 100644 index cef6f6715..000000000 --- a/functions/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionConfiguration.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.filter; - -import java.util.function.Function; - -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.integration.transformer.ExpressionEvaluatingTransformer; -import org.springframework.messaging.Message; - -/** - * @author Artem Bilan - * @author David Turanski - */ -@AutoConfiguration -@EnableConfigurationProperties(FilterFunctionProperties.class) -public class FilterFunctionConfiguration { - - @Bean - public Function, Message> filterFunction( - ExpressionEvaluatingTransformer filterExpressionEvaluatingTransformer) { - - return message -> { - if ((Boolean) filterExpressionEvaluatingTransformer.transform(message).getPayload()) { - return message; - } - else { - return null; - } - }; - } - - @Bean - public ExpressionEvaluatingTransformer filterExpressionEvaluatingTransformer( - FilterFunctionProperties filterFunctionProperties) { - - return new ExpressionEvaluatingTransformer(filterFunctionProperties.getExpression()); - } - -} diff --git a/functions/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionProperties.java b/functions/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionProperties.java deleted file mode 100644 index ab5798d19..000000000 --- a/functions/function/filter-function/src/main/java/org/springframework/cloud/fn/filter/FilterFunctionProperties.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.filter; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.integration.expression.ValueExpression; - -/** - * Configuration properties for the SpEL function. - * - * @author Gary Russell - * @author Artem Bilan - */ -@ConfigurationProperties("filter.function") -public class FilterFunctionProperties { - - /** - * Boolean SpEL expression to apply against request message to filter. - */ - private Expression expression = new ValueExpression<>(true); - - public Expression getExpression() { - return this.expression; - } - - public void setExpression(Expression expression) { - this.expression = expression; - } - -} diff --git a/functions/function/filter-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/function/filter-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index 8973856fe..000000000 --- a/functions/function/filter-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.filter.FilterFunctionConfiguration diff --git a/functions/function/filter-function/src/test/java/org/springframework/cloud/fn/filter/FilterFunctionApplicationTests.java b/functions/function/filter-function/src/test/java/org/springframework/cloud/fn/filter/FilterFunctionApplicationTests.java deleted file mode 100644 index 6cf98aa65..000000000 --- a/functions/function/filter-function/src/test/java/org/springframework/cloud/fn/filter/FilterFunctionApplicationTests.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.filter; - -import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - - -import org.junit.jupiter.api.Test; - - - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Artem Bilan - * @author David Turanski - */ -@SpringBootTest(properties = "filter.function.expression=payload.length() > 5") -@DirtiesContext -public class FilterFunctionApplicationTests { - - @Autowired - @Qualifier("filterFunction") - Function, Message> filter; - - @Test - public void testFilter() { - Stream> messages = List.of("hello", "hello world") - .stream() - .map(GenericMessage::new); - List> result = messages.filter(message -> this.filter.apply(message) != null).collect(Collectors.toList()); - assertThat(result.size()).isEqualTo(1); - assertThat(result.get(0).getPayload()).isNotNull(); - assertThat(result.get(0).getPayload()).isEqualTo("hello world"); - } - - @SpringBootApplication - static class FilterFunctionTestApplication { - - } - -} diff --git a/functions/function/header-enricher-function/README.adoc b/functions/function/header-enricher-function/README.adoc deleted file mode 100644 index 29f4ff6db..000000000 --- a/functions/function/header-enricher-function/README.adoc +++ /dev/null @@ -1,25 +0,0 @@ -# Header Enricher Function - -This module provides a header enricher function that can be reused and composed in other applications. - -## Beans for injection - -You can import the `HeaderEnricherFunctionConfiguration` in a Spring Boot application and then inject the following bean. - -`headerEnricherFunction` - -You can use `headerEnricherFunction` as a qualifier when injecting. - -Once injected, you can use the `apply` method of the `Function` to invoke it and get the result. - -## Configuration Options - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/header/enricher/HeaderEnricherFunctionProperties.java[HeaderEnricherFunctionProperties.java] - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/header/enricher/HeaderEnricherFunctionApplicationTests.java[test suite] for examples of how this function is used. - -## Other usage - -See this link:../../../applications/processor/header-enricher-processor/README.adoc[README] where this function is used to create a Spring Cloud Stream application. \ No newline at end of file diff --git a/functions/function/header-enricher-function/pom.xml b/functions/function/header-enricher-function/pom.xml deleted file mode 100644 index 429c21854..000000000 --- a/functions/function/header-enricher-function/pom.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - header-enricher-function - header-enricher-function - Spring Native Function for applying message header enrichment - - - - org.springframework.cloud.fn - payload-converter-function - ${project.version} - - - - diff --git a/functions/function/header-enricher-function/src/main/java/org/springframework/cloud/fn/header/enricher/HeaderEnricherFunctionConfiguration.java b/functions/function/header-enricher-function/src/main/java/org/springframework/cloud/fn/header/enricher/HeaderEnricherFunctionConfiguration.java deleted file mode 100644 index 1f05aaf5f..000000000 --- a/functions/function/header-enricher-function/src/main/java/org/springframework/cloud/fn/header/enricher/HeaderEnricherFunctionConfiguration.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.header.enricher; - -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.function.Function; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.integration.transformer.HeaderEnricher; -import org.springframework.integration.transformer.support.ExpressionEvaluatingHeaderValueMessageProcessor; -import org.springframework.messaging.Message; - -/** - * @author Gary Russell - * @author Christian Tzolov - * @author Soby Chacko - * @author Artem Bilan - */ -@AutoConfiguration -@EnableConfigurationProperties(HeaderEnricherFunctionProperties.class) -@ConditionalOnProperty(prefix = "header.enricher", value = "headers") -public class HeaderEnricherFunctionConfiguration { - - @Autowired - private HeaderEnricherFunctionProperties properties; - - @Bean - public Function, Message> headerEnricherFunction(HeaderEnricher headerEnricher) { - return headerEnricher::transform; - } - - @Bean - public HeaderEnricher headerEnricher(BeanFactory beanFactory) { - Map> headersToAdd = new HashMap<>(); - Properties props = this.properties.getHeaders(); - Enumeration enumeration = props.propertyNames(); - while (enumeration.hasMoreElements()) { - String propertyName = (String) enumeration.nextElement(); - ExpressionEvaluatingHeaderValueMessageProcessor headerValueMessageProcessor = - new ExpressionEvaluatingHeaderValueMessageProcessor<>(props.getProperty(propertyName), null); - headerValueMessageProcessor.setBeanFactory(beanFactory); - headersToAdd.put(propertyName, headerValueMessageProcessor); - } - HeaderEnricher headerEnricher = new HeaderEnricher(headersToAdd); - headerEnricher.setDefaultOverwrite(this.properties.isOverwrite()); - return headerEnricher; - } - -} diff --git a/functions/function/header-enricher-function/src/main/java/org/springframework/cloud/fn/header/enricher/HeaderEnricherFunctionProperties.java b/functions/function/header-enricher-function/src/main/java/org/springframework/cloud/fn/header/enricher/HeaderEnricherFunctionProperties.java deleted file mode 100644 index 7ca3d2291..000000000 --- a/functions/function/header-enricher-function/src/main/java/org/springframework/cloud/fn/header/enricher/HeaderEnricherFunctionProperties.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.header.enricher; - -import java.util.Properties; - -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * Configuration properties for the Header Enricher Processor application. - * - * @author Gary Russell - * @author Artem Bilan - * @author Soby Chacko - */ -@ConfigurationProperties("header.enricher") -@Validated -public class HeaderEnricherFunctionProperties { - - /** - * \n separated properties representing headers in which values are SpEL expressions, e.g - * foo='bar' \n baz=payload.baz. - */ - private Properties headers; - - /** - * set to true to overwrite any existing message headers. - */ - private boolean overwrite = false; - - @NotNull - public Properties getHeaders() { - return this.headers; - } - - public void setHeaders(Properties headers) { - this.headers = headers; - } - - public boolean isOverwrite() { - return this.overwrite; - } - - public void setOverwrite(boolean overwrite) { - this.overwrite = overwrite; - } - -} diff --git a/functions/function/header-enricher-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/function/header-enricher-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index 9e5532de8..000000000 --- a/functions/function/header-enricher-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.header.enricher.HeaderEnricherFunctionConfiguration diff --git a/functions/function/header-enricher-function/src/test/java/org/springframework/cloud/fn/header/enricher/HeaderEnricherFunctionApplicationTests.java b/functions/function/header-enricher-function/src/test/java/org/springframework/cloud/fn/header/enricher/HeaderEnricherFunctionApplicationTests.java deleted file mode 100644 index bf81d8ec6..000000000 --- a/functions/function/header-enricher-function/src/test/java/org/springframework/cloud/fn/header/enricher/HeaderEnricherFunctionApplicationTests.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.header.enricher; - -import java.util.function.Function; - -import org.junit.jupiter.api.Test; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.integration.test.matcher.HeaderMatcher; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.annotation.DirtiesContext; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - -/** - * - * @author Gary Russell - * @author Soby Chacko - */ -@SpringBootTest(properties = { - "header.enricher.headers=foo='bar' \\n baz='fiz' \\n buz=payload \\n jaz=@value", - "header.enricher.overwrite = true" }) -@DirtiesContext -public class HeaderEnricherFunctionApplicationTests { - - @Autowired - Function, Message> headerEnricherFunction; - - @Test - public void testDefault() { - final Message message = MessageBuilder.withPayload("hello") - .setHeader("baz", "qux").build(); - final Message enriched = headerEnricherFunction.apply(message); - - assertThat(enriched, HeaderMatcher.hasHeader("foo", equalTo("bar"))); - assertThat(enriched, HeaderMatcher.hasHeader("baz", equalTo("fiz"))); - assertThat(enriched, HeaderMatcher.hasHeader("buz", equalTo("hello"))); - assertThat(enriched, HeaderMatcher.hasHeader("jaz", equalTo("beanValue"))); - } - - @SpringBootApplication - static class HeaderEnricherFunctionTestApplication { - - @Bean - public String value() { - return "beanValue"; - } - - } - -} diff --git a/functions/function/header-filter-function/README.adoc b/functions/function/header-filter-function/README.adoc deleted file mode 100644 index b9c9f93eb..000000000 --- a/functions/function/header-filter-function/README.adoc +++ /dev/null @@ -1,25 +0,0 @@ -= Header Enricher Function - -This module provides a header enricher function that can be reused and composed in other applications. - -== Beans for injection - -You can import the `HeaderEnricherFunctionConfiguration` in a Spring Boot application and then inject the following bean. - -`headerFilterFunction` - -You can use `headerFilterFunction` as a qualifier when injecting. - -Once injected, you can use the `apply` method of the `Function` to invoke it and get the result. - -== Configuration Options - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionProperties.java[HeaderFilterFunctionProperties.java] - -== Tests - -See this link:src/test/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionApplicationTests.java[test suite] for examples of how this function is used. - -== Other usage - -See this link:../../../applications/processor/header-filter-processor/README.adoc[README] where this function is used to create a Spring Cloud Stream application. \ No newline at end of file diff --git a/functions/function/header-filter-function/pom.xml b/functions/function/header-filter-function/pom.xml deleted file mode 100644 index ead2c9c41..000000000 --- a/functions/function/header-filter-function/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - header-filter-function - header-filter-function - Spring Native Function for applying message filters - - - - org.springframework.cloud.fn - payload-converter-function - ${project.version} - - - org.assertj - assertj-core - test - - - - diff --git a/functions/function/header-filter-function/src/main/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionConfiguration.java b/functions/function/header-filter-function/src/main/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionConfiguration.java deleted file mode 100644 index 2a5fdd2fb..000000000 --- a/functions/function/header-filter-function/src/main/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionConfiguration.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.header.filter; - -import java.util.HashSet; -import java.util.function.Function; - -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.integration.IntegrationMessageHeaderAccessor; -import org.springframework.integration.transformer.HeaderFilter; -import org.springframework.messaging.Message; -import org.springframework.util.StringUtils; - -/** - * Configure a function using {@link HeaderFilter}. - * - * @author Corneil du Plessis - */ -@AutoConfiguration -@EnableConfigurationProperties(HeaderFilterFunctionProperties.class) -@ConditionalOnExpression("'${header.filter.remove}'!='' or '${header.filter.delete-all}' != ''") -public class HeaderFilterFunctionConfiguration { - private final HeaderFilterFunctionProperties properties; - public HeaderFilterFunctionConfiguration(HeaderFilterFunctionProperties properties) { - this.properties = properties; - } - - @Bean - public Function, Message> headerFilterFunction() { - if (properties.isDeleteAll()) { - return (message) -> { - var accessor = new IntegrationMessageHeaderAccessor(message); - var headers = new HashSet<>(message.getHeaders().keySet()); - headers.removeIf(accessor::isReadOnly); - HeaderFilter filter = new HeaderFilter(headers.toArray(new String[0])); - return filter.transform(message); - }; - } - else { - return headerFilter()::transform; - } - } - - @Bean - public HeaderFilter headerFilter() { - if (properties.getRemove() != null) { - String[] remove = StringUtils.tokenizeToStringArray(properties.getRemove(), ", ", true, true); - HeaderFilter filter = new HeaderFilter(remove); - if (properties.getRemove().contains("*")) { - filter.setPatternMatch(true); - } - return filter; - } - else { - return new HeaderFilter(""); - } - } - -} diff --git a/functions/function/header-filter-function/src/main/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionProperties.java b/functions/function/header-filter-function/src/main/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionProperties.java deleted file mode 100644 index e0a873eca..000000000 --- a/functions/function/header-filter-function/src/main/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionProperties.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.header.filter; - -import org.springframework.boot.context.properties.ConfigurationProperties; - -/** - * Properties for configuration of header-filter-function. - * @author Corneil du Plessis - */ -@ConfigurationProperties("header.filter") -public class HeaderFilterFunctionProperties { - /** - * Indicates the need to remove all headers. - */ - private boolean deleteAll = false; - - /** - * Remove all headers named. A comma, space separated list of header names. - * The names may contain patterns. - */ - private String remove; - - public boolean isDeleteAll() { - return deleteAll; - } - - public void setDeleteAll(boolean deleteAll) { - this.deleteAll = deleteAll; - } - - public String getRemove() { - return remove; - } - - public void setRemove(String remove) { - this.remove = remove; - } -} diff --git a/functions/function/header-filter-function/src/test/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionApplicationDeleteAllTests.java b/functions/function/header-filter-function/src/test/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionApplicationDeleteAllTests.java deleted file mode 100644 index df7debf1a..000000000 --- a/functions/function/header-filter-function/src/test/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionApplicationDeleteAllTests.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.header.filter; - -import java.util.Set; -import java.util.function.Function; - -import org.junit.jupiter.api.Test; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; - -import static org.assertj.core.api.Assertions.assertThat; - -@SpringBootTest(classes = { - HeaderFilterFunctionApplicationDeleteAllTests.HeaderFilterFunctionTestApplication.class, - HeaderFilterFunctionConfiguration.class -}, - properties = {"header.filter.delete-all=true"} -) -public class HeaderFilterFunctionApplicationDeleteAllTests { - @Autowired - protected Function, Message> headerFilter; - - @Test - public void testRemoveLeavesIdTimestampAll() { - // given - final Message message = MessageBuilder.withPayload("hello") - .setHeader("foo", "bar") - .setHeader("bar", "foo") - .build(); - Message result = headerFilter.apply(message); - var headers = result.getHeaders().keySet(); - assertThat(headers).isEqualTo(Set.of("id", "timestamp")); - } - - @SpringBootApplication - static class HeaderFilterFunctionTestApplication { - public static void main(String[] args) throws Exception { - SpringApplication.main(args); - } - } - -} diff --git a/functions/function/header-filter-function/src/test/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionApplicationTests.java b/functions/function/header-filter-function/src/test/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionApplicationTests.java deleted file mode 100644 index 6c7ca30f0..000000000 --- a/functions/function/header-filter-function/src/test/java/org/springframework/cloud/fn/header/filter/HeaderFilterFunctionApplicationTests.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.header.filter; - -import java.util.Set; -import java.util.function.Function; - -import org.junit.jupiter.api.Test; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; - -import static org.assertj.core.api.Assertions.assertThat; - -@SpringBootTest(classes = { - HeaderFilterFunctionApplicationTests.HeaderFilterFunctionTestApplication.class, - HeaderFilterFunctionConfiguration.class -}, - properties = {"header.filter.remove=foo,bar,pf-*"} -) -public class HeaderFilterFunctionApplicationTests { - @Autowired - protected Function, Message> headerFilter; - - @Test - public void testRemoveAll() { - // given - final Message message = MessageBuilder.withPayload("hello") - .setHeader("foo", "bar") - .setHeader("bar", "foo") - .build(); - Message result = headerFilter.apply(message); - var headers = HeaderUtils.getNonReadOnlyHeaders(result); - assertThat(headers).isEmpty(); - } - - @Test - public void testRemoveSome() { - // given - final Message message = MessageBuilder.withPayload("hello") - .setHeader("foo", "bar") - .setHeader("foo-bar", "bar") - .setHeader("bar", "foo") - .build(); - Message result = headerFilter.apply(message); - var headers = HeaderUtils.getNonReadOnlyHeaders(result); - assertThat(headers).isEqualTo(Set.of("foo-bar")); - } - - @Test - public void testRemoveSomeWithWildcard() { - // given - final Message message = MessageBuilder.withPayload("hello") - .setHeader("foo", "bar") - .setHeader("pf-foo", "bar") - .setHeader("pf-bar", "bar") - .setHeader("pfBar", "bar") - .setHeader("bar", "foo") - .build(); - Message result = headerFilter.apply(message); - var headers = HeaderUtils.getNonReadOnlyHeaders(result); - assertThat(headers).isEqualTo(Set.of("pfBar")); - } - - @Test - public void testRemoveLeavesIdTimestampAll() { - // given - final Message message = MessageBuilder.withPayload("hello") - .setHeader("foo", "bar") - .setHeader("bar", "foo") - .build(); - Message result = headerFilter.apply(message); - - assertThat(result.getHeaders().keySet()).isEqualTo(Set.of("id", "timestamp")); - } - - @SpringBootApplication - static class HeaderFilterFunctionTestApplication { - public static void main(String[] main) { - - } - } - -} diff --git a/functions/function/header-filter-function/src/test/java/org/springframework/cloud/fn/header/filter/HeaderUtils.java b/functions/function/header-filter-function/src/test/java/org/springframework/cloud/fn/header/filter/HeaderUtils.java deleted file mode 100644 index d5ac05cef..000000000 --- a/functions/function/header-filter-function/src/test/java/org/springframework/cloud/fn/header/filter/HeaderUtils.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.header.filter; - -import java.util.HashSet; -import java.util.Set; - -import org.jetbrains.annotations.NotNull; - -import org.springframework.integration.IntegrationMessageHeaderAccessor; -import org.springframework.messaging.Message; - -final public class HeaderUtils { - private HeaderUtils() { - } - - @NotNull - public static Set getNonReadOnlyHeaders(Message message) { - var headers = new HashSet<>(message.getHeaders().keySet()); - var accessor = new IntegrationMessageHeaderAccessor(message); - headers.removeIf(accessor::isReadOnly); - return headers; - } -} diff --git a/functions/function/header-filter-function/src/test/resources/test.properties b/functions/function/header-filter-function/src/test/resources/test.properties deleted file mode 100644 index 4f937e4c5..000000000 --- a/functions/function/header-filter-function/src/test/resources/test.properties +++ /dev/null @@ -1 +0,0 @@ -logging.level.root=debug diff --git a/functions/function/http-request-function/README.adoc b/functions/function/http-request-function/README.adoc deleted file mode 100644 index e3f4bf01b..000000000 --- a/functions/function/http-request-function/README.adoc +++ /dev/null @@ -1,32 +0,0 @@ -# HTTP Request Function - -This module provides an HTTP request function that can be reused and composed in other applications. -The `Function` uses the reactive `WebClient` from `Spring WebFlux` and is implemented as a `java.util.function.Function`. -This function gives you a reactive stream of `ResponseEntity` given a stream of request messages as the function a signature of `Function,ResponseEntity>`. -Users have to subscribe to the returned `Flux` to receive the data. - -## Beans for injection - -You can import the `HttpRequestFunction` configuration in a Spring Boot application and then inject the following bean. - -`httpRequestFunction` - -You may inject this as `HttpRequestFunction`. - -You can use `httpRequestFunction` as a qualifier when injecting. - -Once injected, you can use the `apply` method of the `Function` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `http.request`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/http/request/HttpRequestFunctionProperties.java[HttpRequestFunctionProperties.java] - -## Examples - -See this link:src/test/java/org/springframework/cloud/fn/http/request/HttpRequestFunctionApplicationTests.java[test suite] for examples of how this function is used. - -## Other usage - -See this link:../../../applications/processor/http-request-processor/README.adoc[README] where this function is used to create a Spring Cloud Stream application to process HTTP requests. \ No newline at end of file diff --git a/functions/function/http-request-function/pom.xml b/functions/function/http-request-function/pom.xml deleted file mode 100644 index b1ad75700..000000000 --- a/functions/function/http-request-function/pom.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - http-request-function - http-request-function - Spring Native Function for Submitting an HTTP request - - - - org.springframework.boot - spring-boot-starter-webflux - - - com.squareup.okhttp3 - mockwebserver - test - - - - diff --git a/functions/function/http-request-function/src/main/java/org/springframework/cloud/fn/http/request/HttpRequestFunctionConfiguration.java b/functions/function/http-request-function/src/main/java/org/springframework/cloud/fn/http/request/HttpRequestFunctionConfiguration.java deleted file mode 100644 index 1b5e6c866..000000000 --- a/functions/function/http-request-function/src/main/java/org/springframework/cloud/fn/http/request/HttpRequestFunctionConfiguration.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2018-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.http.request; - -import java.time.Duration; -import java.util.Map; -import java.util.function.Function; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.convert.converter.Converter; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.ResponseEntity; -import org.springframework.integration.config.IntegrationConverter; -import org.springframework.messaging.Message; -import org.springframework.web.reactive.function.client.WebClient; -import org.springframework.web.util.DefaultUriBuilderFactory; -import org.springframework.web.util.UriBuilderFactory; - - -/** - * Configuration for a {@link Function} that makes HTTP requests to a resource and for - * each request, returns a {@link ResponseEntity}. - * - * @author David Turanski - * @author Sunny Hemdev - * @author Corneil du Plessis - **/ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties(HttpRequestFunctionProperties.class) -public class HttpRequestFunctionConfiguration { - - @Bean - public HttpRequestFunction httpRequestFunction(WebClient.Builder webClientBuilder, HttpRequestFunctionProperties properties) { - return new HttpRequestFunction(webClientBuilder.build(), properties); - } - - @Bean - @IntegrationConverter - public Converter httpMethodConverter() { - return new HttpMethodConverter(); - } - - /** - * Function that accepts a {@code Flux>} containing body and headers and - * returns a {@code Flux>}. - */ - public static class HttpRequestFunction implements Function, Object> { - - private final WebClient webClient; - - private final UriBuilderFactory uriBuilderFactory = new DefaultUriBuilderFactory(); - - private final HttpRequestFunctionProperties properties; - - public HttpRequestFunction(WebClient webClient, HttpRequestFunctionProperties properties) { - this.webClient = webClient; - this.properties = properties; - } - - @Override - public Object apply(Message message) { - return this.webClient - .method(resolveHttpMethod(message)) - .uri(uriBuilderFactory.uriString(resolveUrl(message)).build()) - .bodyValue(resolveBody(message)) - .headers(httpHeaders -> httpHeaders.addAll(resolveHeaders(message))) - .retrieve() - .toEntity(properties.getExpectedResponseType()) - .map(responseEntity -> properties.getReplyExpression().getValue(responseEntity)) - .timeout(Duration.ofMillis(properties.getTimeout())) - .block(); - } - - private String resolveUrl(Message message) { - return properties.getUrlExpression().getValue(message, String.class); - } - - private HttpMethod resolveHttpMethod(Message message) { - return properties.getHttpMethodExpression().getValue(message, HttpMethod.class); - } - - private Object resolveBody(Message message) { - return properties.getBodyExpression() != null ? properties.getBodyExpression().getValue(message) - : message.getPayload(); - } - - private HttpHeaders resolveHeaders(Message message) { - HttpHeaders headers = new HttpHeaders(); - if (properties.getHeadersExpression() != null) { - Map headersMap = properties.getHeadersExpression().getValue(message, Map.class); - for (Map.Entry header : headersMap.entrySet()) { - if (header.getKey() != null && header.getValue() != null) { - headers.add(header.getKey().toString(), - header.getValue().toString()); - } - } - } - return headers; - } - - } - - public static class HttpMethodConverter implements Converter { - - @Override - public HttpMethod convert(String source) { - return HttpMethod.valueOf(source); - } - } -} diff --git a/functions/function/http-request-function/src/main/java/org/springframework/cloud/fn/http/request/HttpRequestFunctionProperties.java b/functions/function/http-request-function/src/main/java/org/springframework/cloud/fn/http/request/HttpRequestFunctionProperties.java deleted file mode 100644 index 50608ede0..000000000 --- a/functions/function/http-request-function/src/main/java/org/springframework/cloud/fn/http/request/HttpRequestFunctionProperties.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright 2015-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.http.request; - -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.http.HttpMethod; -import org.springframework.http.ResponseEntity; -import org.springframework.integration.expression.FunctionExpression; -import org.springframework.integration.expression.ValueExpression; -import org.springframework.validation.annotation.Validated; - -/** - * Configuration properties for the Http Client Processor module. - * - * @author Waldemar Hummer - * @author Mark Fisher - * @author Christian Tzolov - * @author Artem Bilan - * @author David Turanski - * @author Sunny Hemdev - */ -@Validated -@ConfigurationProperties("http.request") -public class HttpRequestFunctionProperties { - - private static final Class DEFAULT_RESPONSE_TYPE = String.class; - - /** - * The type used to interpret the response. - */ - private Class expectedResponseType = DEFAULT_RESPONSE_TYPE; - - /** - * Request timeout in milliseconds. - */ - private long timeout = 30_000; - - /** - * A SpEL expression against incoming message to determine the URL to use. - */ - private Expression urlExpression; - - /** - * A SpEL expression to derive the request method from the incoming message. - */ - private Expression httpMethodExpression = new ValueExpression(HttpMethod.GET); - - /** - * A SpEL expression to derive the request body from the incoming message. - */ - private Expression bodyExpression; - - /** - * A SpEL expression used to derive the http headers map to use. - */ - private Expression headersExpression; - - /** - * A SpEL expression used to compute the final result, applied against the whole http - * {@link org.springframework.http.ResponseEntity}. - */ - private Expression replyExpression = new FunctionExpression(ResponseEntity::getBody); - - @NotNull - public Expression getUrlExpression() { - return urlExpression; - } - - public void setUrlExpression(Expression urlExpression) { - this.urlExpression = urlExpression; - } - - public Expression getHttpMethodExpression() { - return httpMethodExpression; - } - - public void setHttpMethodExpression(Expression httpMethodExpression) { - this.httpMethodExpression = httpMethodExpression; - } - - @NotNull - public Class getExpectedResponseType() { - return expectedResponseType; - } - - public void setExpectedResponseType(Class expectedResponseType) { - this.expectedResponseType = expectedResponseType; - } - - public long getTimeout() { - return this.timeout; - } - - public void setTimeout(long timeout) { - this.timeout = timeout; - } - - public Expression getBodyExpression() { - return bodyExpression; - } - - public void setBodyExpression(Expression bodyExpression) { - this.bodyExpression = bodyExpression; - } - - public Expression getHeadersExpression() { - return headersExpression; - } - - public void setHeadersExpression(Expression headersExpression) { - this.headersExpression = headersExpression; - } - - @NotNull - public Expression getReplyExpression() { - return replyExpression; - } - - public void setReplyExpression(Expression replyExpression) { - this.replyExpression = replyExpression; - } - -} diff --git a/functions/function/http-request-function/src/test/java/org/springframework/cloud/fn/http/request/HttpRequestFunctionTestApplicationTests.java b/functions/function/http-request-function/src/test/java/org/springframework/cloud/fn/http/request/HttpRequestFunctionTestApplicationTests.java deleted file mode 100644 index 1747d2171..000000000 --- a/functions/function/http-request-function/src/test/java/org/springframework/cloud/fn/http/request/HttpRequestFunctionTestApplicationTests.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.http.request; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -import okhttp3.mockwebserver.Dispatcher; -import okhttp3.mockwebserver.MockResponse; -import okhttp3.mockwebserver.MockWebServer; -import okhttp3.mockwebserver.RecordedRequest; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.cloud.fn.http.request.HttpRequestFunctionConfiguration.HttpRequestFunction; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.messaging.support.MessageBuilder; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.fail; - -public class HttpRequestFunctionTestApplicationTests { - private MockWebServer server = new MockWebServer(); - - private ApplicationContextRunner runner; - - @BeforeEach - void setup() { - this.runner = new ApplicationContextRunner() - .withUserConfiguration(HttpRequestFunctionTestApplication.class) - .withPropertyValues( - "http.request.reply-expression=#root", - "http.request.url-expression='" + url() + "'"); - } - - @Test - void shouldReturnString() { - - server.enqueue(new MockResponse() - .setResponseCode(HttpStatus.OK.value()) - .setBody("hello")); - - runner.withPropertyValues("http.request.http-method-expression='POST'").run(context -> { - HttpRequestFunction httpRequestFunction = context.getBean(HttpRequestFunction.class); - Message message = MessageBuilder.withPayload("").build(); - ResponseEntity r = (ResponseEntity) httpRequestFunction.apply(message); - assertThat(r.getBody()).isEqualTo("hello"); - assertThat(r.getStatusCode().is2xxSuccessful()).isTrue(); - }); - } - - @Test - void shouldPostJson() { - - server.setDispatcher(new Dispatcher() { - @Override - public MockResponse dispatch(RecordedRequest recordedRequest) { - return new MockResponse().setHeader(HttpHeaders.CONTENT_TYPE, - recordedRequest.getHeader(HttpHeaders.CONTENT_TYPE)) - .setBody(recordedRequest.getBody()) - .setResponseCode(HttpStatus.CREATED.value()); - } - }); - - runner.withPropertyValues("http.request.http-method-expression='POST'", - "http.request.headers-expression={'Content-Type':'application/json'}").run(context -> { - HttpRequestFunction httpRequestFunction = context.getBean(HttpRequestFunction.class); - String json = "{\"hello\":\"world\"}"; - Message message = MessageBuilder.withPayload(json) - .build(); - ResponseEntity r = (ResponseEntity) httpRequestFunction.apply(message); - assertThat(r.getBody()).isEqualTo(json); - assertThat(r.getStatusCode().is2xxSuccessful()).isTrue(); - assertThat(r.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); - RecordedRequest request = server.takeRequest(100, TimeUnit.MILLISECONDS); - assertThat(request.getMethod()).isEqualTo("POST"); - }); - } - - @Test - void shouldPostPojoAsJson() { - - server.setDispatcher(new Dispatcher() { - @Override - public MockResponse dispatch(RecordedRequest recordedRequest) { - return new MockResponse().setHeader(HttpHeaders.CONTENT_TYPE, - recordedRequest.getHeader(HttpHeaders.CONTENT_TYPE)) - .setBody(recordedRequest.getBody()) - .setResponseCode(HttpStatus.CREATED.value()); - } - }); - - runner.withPropertyValues("http.request.http-method-expression='POST'", - "http.request.expected-response-type=" + Map.class.getName()).run(context -> { - HttpRequestFunction httpRequestFunction = context.getBean(HttpRequestFunction.class); - Map json = Collections.singletonMap("hello", "world"); - Message message = MessageBuilder.withPayload(json) - .build(); - ResponseEntity r = (ResponseEntity) httpRequestFunction.apply(message); - assertThat(r.getBody()).isEqualTo(json); - assertThat(r.getStatusCode().is2xxSuccessful()).isTrue(); - assertThat(r.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_JSON); - RecordedRequest request = server.takeRequest(100, TimeUnit.MILLISECONDS); - assertThat(request.getMethod()).isEqualTo("POST"); - }); - } - - @Test - void shouldDelete() { - server.setDispatcher(new Dispatcher() { - @Override - public MockResponse dispatch(RecordedRequest recordedRequest) { - return new MockResponse().setHeader(HttpHeaders.CONTENT_TYPE, - recordedRequest.getHeader(HttpHeaders.CONTENT_TYPE)) - .setBody(recordedRequest.getBody()) - .setResponseCode(HttpStatus.ACCEPTED.value()); - } - }); - - runner.withPropertyValues("http.request.http-method-expression='DELETE'", - "http.request.expected-response-type=" + Void.class.getName()).run(context -> { - HttpRequestFunction httpRequestFunction = context.getBean(HttpRequestFunction.class); - Message message = MessageBuilder.withPayload("") - .build(); - ResponseEntity r = (ResponseEntity) httpRequestFunction.apply(message); - assertThat(r.getBody()).isNull(); - assertThat(r.getStatusCode().is2xxSuccessful()).isTrue(); - RecordedRequest request = server.takeRequest(100, TimeUnit.MILLISECONDS); - assertThat(request.getMethod()).isEqualTo("DELETE"); - }); - } - - @Test - void shouldThrowErrorIfCannotConnect() throws IOException { - server.shutdown(); - runner.run(context -> { - HttpRequestFunction httpRequestFunction = context.getBean(HttpRequestFunction.class); - try { - httpRequestFunction.apply(new GenericMessage("")); - fail("Expected exception"); - } - catch (Throwable throwable) { - assertThat(throwable.getMessage()).contains("Connection refused"); - } - }); - } - - @Test - void requestUsingExpressions() { - server.setDispatcher(new Dispatcher() { - @Override - public MockResponse dispatch(RecordedRequest recordedRequest) { - return new MockResponse().setHeader(HttpHeaders.CONTENT_TYPE, - recordedRequest.getHeader(HttpHeaders.ACCEPT)) - .setBody(recordedRequest.getBody()) - .setResponseCode(HttpStatus.OK.value()); - } - }); - - runner.withPropertyValues( - "http.request.url-expression=headers['url']", - "http.request.http-method-expression=headers['method']", - "http.request.body-expression=headers['body']", - "http.request.headers-expression={Accept:'application/json'}") - .run(context -> { - Message message = MessageBuilder.withPayload("") - .setHeader("url", url()) - .setHeader("method", "POST") - .setHeader("body", "{\"hello\":\"world\"}") - .build(); - - HttpRequestFunction httpRequestFunction = context.getBean(HttpRequestFunction.class); - ResponseEntity responseEntity = (ResponseEntity) httpRequestFunction.apply(message); - assertThat(responseEntity.getStatusCode()).isEqualTo(HttpStatus.OK); - assertThat(responseEntity.getBody()).isEqualTo(message.getHeaders().get("body")); - assertThat(responseEntity.getHeaders().getContentType()) - .isEqualTo(MediaType.APPLICATION_JSON); - }); - } - - @Test - void requestUsingReturnType() { - server.setDispatcher(new Dispatcher() { - @Override - public MockResponse dispatch(RecordedRequest recordedRequest) { - return new MockResponse().setHeader(HttpHeaders.CONTENT_TYPE, - recordedRequest.getHeader(HttpHeaders.CONTENT_TYPE)) - .setBody(recordedRequest.getBody()) - .setResponseCode(HttpStatus.OK.value()); - } - }); - runner.withPropertyValues( - "http.request..http-method-expression='POST'", - "http.request.headers-expression={'Content-Type':'application/octet-stream'}", - "http.request.expected-response-type=byte[]") - .run(context -> { - Message message = MessageBuilder.withPayload("hello") - .build(); - HttpRequestFunction httpRequestFunction = context.getBean(HttpRequestFunction.class); - ResponseEntity responseEntity = (ResponseEntity) httpRequestFunction.apply(message); - assertThat(responseEntity.getStatusCode()).isEqualTo(HttpStatus.OK); - assertThat(responseEntity.getBody()).isEqualTo("hello".getBytes()); - assertThat(responseEntity.getHeaders().getContentType()) - .isEqualTo(MediaType.APPLICATION_OCTET_STREAM); - }); - } - - @Test - void requestUsingJsonPathMethodExpression() { - server.setDispatcher(new Dispatcher() { - @Override - public MockResponse dispatch(RecordedRequest recordedRequest) { - return new MockResponse() - .setBody(recordedRequest.getBody()) - .setHeader("method", recordedRequest.getMethod()) - .setResponseCode(HttpStatus.OK.value()); - } - }); - runner.withPropertyValues( - "http.request.http-method-expression=#jsonPath(payload,'$.myMethod')") - .run(context -> { - Message message = MessageBuilder - .withPayload("{\"name\":\"Fred\",\"age\":41, \"myMethod\":\"POST\"}") - .build(); - HttpRequestFunction httpRequestFunction = context.getBean(HttpRequestFunction.class); - ResponseEntity responseEntity = (ResponseEntity) httpRequestFunction.apply(message); - assertThat(responseEntity.getStatusCode()).isEqualTo(HttpStatus.OK); - assertThat(responseEntity.getBody()).isEqualTo(message.getPayload()); - }); - } - - private String url() { - return String.format("http://localhost:%d", server.getPort()); - } - - @SpringBootApplication - static class HttpRequestFunctionTestApplication { - } -} diff --git a/functions/function/image-recognition-function/README.adoc b/functions/function/image-recognition-function/README.adoc deleted file mode 100644 index 141b8b52e..000000000 --- a/functions/function/image-recognition-function/README.adoc +++ /dev/null @@ -1,103 +0,0 @@ -:images-asciidoc: https://raw.githubusercontent.com/spring-cloud/stream-applications/master/functions/function/image-recognition-function/src/main/resources/images/ - -# Image Recognition - -[.lead] -Java model inference library for the https://github.com/tensorflow/models/tree/master/research/slim#pre-trained-models[Inception], https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md#pre-trained-models[MobileNetV1] and https://github.com/tensorflow/models/tree/master/research/slim/nets/mobilenet#pretrained-models[MobileNetV2] image recognition architectures. -Provides real-time recognition of the https://dl.bintray.com/big-data/generic/imagenet_comp_graph_label_strings.txt[LSVRC-2012-CLS categories] in the input images. - -[cols="1,2",frame=none,grid=none] -|=== -| image:{images-asciidoc}/image-augmented.jpg[alt=Inception 1,width=100%] -|The https://github.com/spring-cloud/stream-applications/blob/master/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/ImageRecognition.java[ImageRecognition] takes an image and outputs a list of probable categories the image contains. The response is represented by https://github.com/spring-cloud/stream-applications/blob/master/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/RecognitionResponse.java[RecognitionResponse] class. - -The https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/JsonMapperFunction.java[JsonMapperFunction] permits -converting the `RecognitionResponse` into JSON objects and the -https://github.com/spring-cloud/stream-applications/blob/master/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/ImageRecognitionAugmenter.java[ImageRecognitionAugmenter] can augment the input image with the detected categories (as shown in pic. 1). -|=== - -## Usage - -Add the `image-recognition` dependency to the pom (use the latest version available): - -[source,xml] ----- - - org.springframework.cloud.fn - image-recognition-function - ${revision} - ----- - -#### Example 1: Image Recognition - -The following snippet demonstrates how to use the `ImageRecognition` for detecting the categories present in an input image. -It also shows how to convert the result into JSON format and augment the input image with the detected category labels. - -[source,java,linenums] ----- -ImageRecognition recognitionService = ImageRecognition.mobilenetModeV2( - "https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.4_224.tgz#mobilenet_v2_1.4_224_frozen.pb", //<1> - 224, //<2> - 5, //<3> - true); //<4> - -byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/giant_panda_in_beijing_zoo_1.jpg"); //<5> - -List recognizedObjects = recognitionService.recognize(inputImage); //<6> ----- -<1> Downloads and loads a pre-trained `mobilenet_v2_1.4_224_frozen.pb` model. -Mind that on first attempt it will download few hundreds of MBs. -The consecutive runs will use the cached copy (5) instead. -The category labels for the MobileNetV2 are resolved from `src/main/resources/labels/mobilenet_labels.txt`. -<2> The wxh seize of the input normalized image. -<3> Top K result to return. -<4> Cache the model on the local file system. -<5> Load the image to recognise. -<6> Return a map of the top-k most probable category names and their probabilities. - -The `ImageRecognition.mobilenetModeV1` and `ImageRecognition.inception` factory methods help to load and configure pre-trained mobilenetModeV1 and and Inception models. - -Next you can convert the result in JSON format. - -[source,java,linenums] ----- -String jsonRecognizedObjects = new JsonMapperFunction().apply(recognizedObjects); ----- - -.Sample Image Recognition JSON representation -[source,json] ----- -[{"label":"giant panda","probability":0.9946687817573547},{"label":"Arctic fox","probability":0.0036631098482757807},{"label":"ice bear","probability":3.3782739774324E-4},{"label":"American black bear","probability":2.3452856112271547E-4},{"label":"skunk","probability":1.6454080468975008E-4}] ----- - -Use the `ImageRecognitionAugmenter` to draw the recognise categories on top of the input image. - -[source,java,linenums] ----- -byte[] augmentedImage = new ImageRecognitionAugmenter().apply(inputImage, recognizedObjects); //<1> -IOUtils.write(augmentedImage, new FileOutputStream("./image-recognition/target/image-augmented.jpg"));//<2> ----- -<1> Augment the image with the recognized categories (uses Java2D internally). -<2> Stores the augmented image as `image-augmented.jpg` image file. - -.Augmented image-augmented.jpg file -image:{images-asciidoc}/image-recognition-panda-augmented.jpg[alt=Augmented,width=30%] - -## Models - -This implementation supports all pre-trained https://github.com/tensorflow/models/tree/master/research/slim#pre-trained-models[Inception], https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md#pre-trained-models[MobileNetV1] and https://github.com/tensorflow/models/tree/master/research/slim/nets/mobilenet#pretrained-models[MobileNetV2] models. -Following URI notation can be used to download any of the models directly from the zoo. - ----- -http://# ----- - -The `` is the frozen model file name within the archive. - - -TIP: To speedup the bootstrap performance you may consider extracting the model and caching it locally. -Then you can use the `file://path-to-my-local-copy` URI schema to access it. - -NOTE: It is important to use the labels that correspond to the model being used! -Table below highlights this mapping. diff --git a/functions/function/image-recognition-function/pom.xml b/functions/function/image-recognition-function/pom.xml deleted file mode 100644 index e8ad6fd9a..000000000 --- a/functions/function/image-recognition-function/pom.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - image-recognition-function - image-recognition-function - Spring Native Function for Tensorflow Integration - - - 1.3.2 - - - - - org.springframework.cloud.fn - tensorflow-common - ${project.version} - - - - - - - tensorflow-snapshots - https://oss.sonatype.org/content/repositories/snapshots/ - - true - - - - - - diff --git a/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/ImageRecognition.java b/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/ImageRecognition.java deleted file mode 100644 index f93f39331..000000000 --- a/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/ImageRecognition.java +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.image.recognition; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.tensorflow.Operand; -import org.tensorflow.Tensor; -import org.tensorflow.op.core.Placeholder; -import org.tensorflow.op.image.DecodeJpeg; -import org.tensorflow.op.nn.TopK; - -import org.springframework.cloud.fn.common.tensorflow.GraphRunner; -import org.springframework.cloud.fn.common.tensorflow.GraphRunnerMemory; -import org.springframework.cloud.fn.common.tensorflow.ProtoBufGraphDefinition; -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.core.io.Resource; -import org.springframework.util.StreamUtils; - -/** - * @author Christian Tzolov - */ -public class ImageRecognition implements AutoCloseable { - - private final List labels; - private final GraphRunner imageNormalization; - private final GraphRunner imageRecognition; - private final GraphRunner maxProbability; - private final GraphRunner topKProbabilities; - - /** - * Instead of creating the {@link ImageRecognition} service explicitly via the constructor, - * you should consider the convenience factory methods below. E.g. - * - * {@link #inception(String, int, int, boolean)} - * {@link #mobileNetV1(String, int, int, boolean)} - * {@link #mobileNetV2(String, int, int, boolean)} - * - * @param modelUri location of the pre-trained model to use. - * @param labelsUri location of the list fromMemory pre-trained categories used by the model. - * @param imageRecognitionGraphInputName name of the Model's input node to send the input image to. - * @param imageRecognitionGraphOutputName name of the Model's output node to retrieve the predictions from. - * @param imageHeight normalized image height. - * @param imageWidth normalized image width. - * @param mean mean value to normalize the input image. - * @param scale scale to normalize the input image. - * @param responseSize Max number of predictions per recognize. - * @param cacheModel if true the pre-trained model is cached on the local file system. - */ - public ImageRecognition(String modelUri, String labelsUri, int imageHeight, int imageWidth, float mean, float scale, - String imageRecognitionGraphInputName, String imageRecognitionGraphOutputName, int responseSize, boolean cacheModel) { - - this.labels = labels(labelsUri); - - /** - * Normalizes the raw input image into format expected by the pre-trained Inception/MobileNetV1/MobileNetV2 models. - * Typically the model is trained fromMemory images scaled to certain size. Usually it is 224x224 pixels, but can be - * also 192x192, 160x160, 128128, 92x92. Use the (imageHeight, imageWidth) to set the desired size. - * The colors, represented as R, G, B in 1-byte each were converted to float using (Value - Mean)/Scale. - * - * imageHeight normalized image height. - * imageWidth normalized image width. - * mean mean value to normalize the input image. - * scale scale to normalize the input image. - */ - this.imageNormalization = new GraphRunner("raw_image", "normalized_image") - .withGraphDefinition(tf -> { - Placeholder input = tf.withName("raw_image").placeholder(String.class); - final Operand decodedImage = - tf.dtypes.cast(tf.image.decodeJpeg(input, DecodeJpeg.channels(3L)), Float.class); - final Operand resizedImage = tf.image.resizeBilinear( - tf.expandDims(decodedImage, tf.constant(0)), - tf.constant(new int[] { imageHeight, imageWidth })); - tf.withName("normalized_image").math.div(tf.math.sub(resizedImage, tf.constant(mean)), tf.constant(scale)); - }); - - this.imageRecognition = new GraphRunner(imageRecognitionGraphInputName, imageRecognitionGraphOutputName) - .withGraphDefinition(new ProtoBufGraphDefinition(toResource(modelUri), cacheModel)); - - this.maxProbability = new GraphRunner(Arrays.asList("recognition_result"), Arrays.asList("category", "probability")) - .withGraphDefinition(tf -> { - Placeholder input = tf.withName("recognition_result").placeholder(Float.class); - tf.withName("category").math.argMax(input, tf.constant(1)); - tf.withName("probability").max(input, tf.constant(1)); - }); - - this.topKProbabilities = new GraphRunner("recognition_result", "topK") - .withGraphDefinition(tf -> { - Placeholder input = tf.withName("recognition_result").placeholder(Float.class); - tf.withName("topK").nn.topK(input, tf.constant(responseSize), TopK.sorted(true)); - }); - } - - /** - * Takes an byte encoded image and returns the most probable category recognized in the image along fromMemory its probability. - * @param inputImage Byte array encoded image to recognize. - * @return Returns a single map entry containing the names of the recognized categories as key and the confidence as value. - */ - public Map recognizeMax(byte[] inputImage) { - - try (Tensor inputTensor = Tensor.create(inputImage); GraphRunnerMemory memorize = new GraphRunnerMemory()) { - - Map> max = this.imageNormalization.andThen(memorize) - .andThen(this.imageRecognition).andThen(memorize) - .andThen(this.maxProbability).andThen(memorize) - .apply(Collections.singletonMap("raw_image", inputTensor)); - - long[] category = new long[1]; - max.get("category").copyTo(category); - float[] probability = new float[1]; - max.get("probability").copyTo(probability); - - return Collections.singletonMap(labels.get((int) category[0]), Double.valueOf(probability[0])); - } - } - - /** - * Takes an byte encoded input image and returns the top K most probable categories recognized in the image - * along fromMemory their probabilities. - * - * @param inputImage Byte array encoded image to recognize. - * @return Returns a list of key-value pairs. Every key-value pair represents a single category recognized. - * The key stands for the name(s) of the category while the value states the confidence that there is an - * object of this category. The entries in the Map are ordered from the higher to the lower confidences. - */ - public Map recognizeTopK(byte[] inputImage) { - - try (Tensor inputTensor = Tensor.create(inputImage); GraphRunnerMemory memorize = new GraphRunnerMemory()) { - - Map> topKResults = - this.imageNormalization.andThen(memorize) - .andThen(this.imageRecognition).andThen(memorize) - .andThen(this.topKProbabilities).andThen(memorize) - .apply(Collections.singletonMap("raw_image", inputTensor)); - - Tensor recognizedImagesTensor = memorize.getTensorMap().get(this.imageRecognition.getSingleFetchName()); - float[][] results = new float[(int) recognizedImagesTensor.shape()[0]][(int) recognizedImagesTensor.shape()[1]]; - recognizedImagesTensor.copyTo(results); - - Tensor topKTensor = topKResults.get("topK").expect(Float.class); - float[][] topK = new float[(int) topKTensor.shape()[0]][(int) topKTensor.shape()[1]]; - topKTensor.copyTo(topK); - - float min = topK[0][topK[0].length - 1]; - - Map valueToIndex = new HashMap<>(); - for (int i = 0; i < results[0].length; i++) { - if (results[0][i] >= min) { - valueToIndex.put(results[0][i], i); - } - } - - Map map = new LinkedHashMap<>(); - for (float tk : topK[0]) { - map.put(labels.get(valueToIndex.get(tk)), (double) tk); - } - - return map; - } - } - - private Resource toResource(String uri) { - return new DefaultResourceLoader().getResource(uri); - } - - /** - * Converts a labels resources into string list. - * @return Returns string lists. One line per different category. - */ - private List labels(String labelsUri) { - try (InputStream is = toResource(labelsUri).getInputStream()) { - return Arrays.asList(StreamUtils.copyToString(is, Charset.forName("UTF-8")).split("\n")); - } - catch (IOException e) { - throw new RuntimeException("Failed to initialize the Vocabulary", e); - } - } - - /** - * - * The Inception graph uses "input" as input and "output" as output. - * - */ - public static ImageRecognition inception(String inceptionModelUri, - int normalizedImageSize, int responseSize, boolean cacheModel) { - return new ImageRecognition(inceptionModelUri, "classpath:/labels/inception_labels.txt", - normalizedImageSize, normalizedImageSize, 117f, 1f, - "input", "output", - responseSize, cacheModel); - } - - /** - * Convenience for MobileNetV2 pre-trained models: - * https://github.com/tensorflow/models/tree/master/research/slim/nets/mobilenet#pretrained-models - * - * The normalized image size is always square (e.g. H=W) - * - * The MobileNetV2 graph uses "input" as input and "MobilenetV2/Predictions/Reshape_1" as output. - * - * @param mobileNetV2ModelUri model uri - * @param normalizedImageSize Depends on the pre-trained model used. Usually 224px is used. - * @param responseSize Number of responses fot topK requests. - * @param cacheModel cache model - * @return ImageRecognition instance configured fromMemory a MobileNetV2 pre-trained model. - */ - public static ImageRecognition mobileNetV2(String mobileNetV2ModelUri, - int normalizedImageSize, int responseSize, boolean cacheModel) { - return new ImageRecognition(mobileNetV2ModelUri, "classpath:/labels/mobilenet_labels.txt", - normalizedImageSize, normalizedImageSize, 0f, 127f, - "input", "MobilenetV2/Predictions/Reshape_1", - responseSize, cacheModel); - } - - /** - * Convenience for MobileNetV1 pre-trained models: - * https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md#pre-trained-models - * - * The MobileNetV1 graph uses "input" as input and "MobilenetV1/Predictions/Reshape_1" as output. - * - */ - public static ImageRecognition mobileNetV1(String mobileNetV1ModelUri, - int normalizedImageSize, int responseSize, boolean cacheModel) { - return new ImageRecognition(mobileNetV1ModelUri, "classpath:/labels/mobilenet_labels.txt", - normalizedImageSize, normalizedImageSize, - 0f, 127f, - "input", "MobilenetV1/Predictions/Reshape_1", - responseSize, cacheModel); - } - - /** - * Convert image recognition results into {@link RecognitionResponse} domain list. - * @param recognitionMap map containing the category mames and its probability. Returned by the - * {@link ImageRecognition#recognizeMax(byte[])} and the ImageRecognition{@link #recognizeTopK(byte[])} methods - * @return List of {@link RecognitionResponse} objects representing the name-to-probability pairs in the input map. - */ - public static List toRecognitionResponse(Map recognitionMap) { - return recognitionMap.entrySet().stream() - .map(nameProbabilityPair -> new RecognitionResponse(nameProbabilityPair.getKey(), nameProbabilityPair.getValue())) - .collect(Collectors.toList()); - } - - @Override - public void close() { - this.imageNormalization.close(); - this.imageRecognition.close(); - this.maxProbability.close(); - this.topKProbabilities.close(); - } -} diff --git a/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/ImageRecognitionAugmenter.java b/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/ImageRecognitionAugmenter.java deleted file mode 100644 index 6ffae73f9..000000000 --- a/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/ImageRecognitionAugmenter.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.image.recognition; - -import java.awt.Color; -import java.awt.FontMetrics; -import java.awt.Graphics2D; -import java.awt.RenderingHints; -import java.awt.geom.Rectangle2D; -import java.awt.image.BufferedImage; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.List; -import java.util.function.BiFunction; - -import javax.imageio.ImageIO; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - - -/** - * Ability to to augment the input image fromMemory the recognized labels. - * - * @author Christian Tzolov - */ -public class ImageRecognitionAugmenter implements BiFunction, byte[]> { - - private static final Log logger = LogFactory.getLog(ImageRecognitionAugmenter.class); - - /** IMAGE_FORMAT. */ - public static final String IMAGE_FORMAT = "jpg"; - - private final Color textColor = Color.BLACK; - private final Color bgColor = new Color(167, 252, 0); - - public ImageRecognitionAugmenter() { - } - - /** - * Augment the input image by adding the recognized classes. - * - * @param imageBytes input image as byte array - * @param result computed recognition labels - * @return the image augmented fromMemory recognized labels. - */ - @Override - public byte[] apply(byte[] imageBytes, List result) { - try { - if (result != null) { - BufferedImage originalImage = ImageIO.read(new ByteArrayInputStream(imageBytes)); - - Graphics2D g = originalImage.createGraphics(); - g.setRenderingHint(RenderingHints.KEY_TEXT_ANTIALIASING, RenderingHints.VALUE_TEXT_ANTIALIAS_ON); - FontMetrics fm = g.getFontMetrics(); - - int x = 1; - int y = 1; - for (RecognitionResponse r : result) { - - String labelName = r.getLabel(); - int probability = (int) (100 * r.getProbability()); - String title = labelName + ": " + probability + "%"; - - Rectangle2D rect = fm.getStringBounds(title, g); - - g.setColor(bgColor); - g.fillRect(x, y, (int) rect.getWidth() + 6, (int) rect.getHeight()); - - g.setColor(textColor); - g.drawString(title, x + 3, (int) (y + rect.getHeight() - 3)); - y = (int) (y + rect.getHeight() + 1); - } - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ImageIO.write(originalImage, IMAGE_FORMAT, baos); - baos.flush(); - imageBytes = baos.toByteArray(); - baos.close(); - } - } - catch (IOException e) { - logger.error("Failed to draw labels in the input image", e); - } - - return imageBytes; - } - -} diff --git a/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/RecognitionResponse.java b/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/RecognitionResponse.java deleted file mode 100644 index e0169a5b5..000000000 --- a/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/RecognitionResponse.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.image.recognition; - -/** - * @author Christian Tzolov - */ -public class RecognitionResponse { - private String label; - private Double probability; - - public RecognitionResponse() { - } - - public RecognitionResponse(String label, Double probability) { - this.label = label; - this.probability = probability; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public Double getProbability() { - return probability; - } - - public void setProbability(Double probability) { - this.probability = probability; - } - - @Override - public String toString() { - return "{label='" + label + ", probability=" + probability + '}'; - } -} diff --git a/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/util/ImageNetReadableNamesWriter.java b/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/util/ImageNetReadableNamesWriter.java deleted file mode 100644 index 889d111f4..000000000 --- a/functions/function/image-recognition-function/src/main/java/org/springframework/cloud/fn/image/recognition/util/ImageNetReadableNamesWriter.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.image.recognition.util; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import org.apache.commons.io.FileUtils; - -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.core.io.Resource; -import org.springframework.util.Assert; -import org.springframework.util.StreamUtils; - -/** - * Create a text file mapping label id to human readable string. - * - * Produces a text file where every line represents single category. The line number represents the category id, while - * the line text is human-readable names for the categories fromMemory this imagenet id. - * - * Based on https://github.com/tensorflow/models/blob/master/research/slim/datasets/imagenet.py#L66 - * - * We retrieve a synset file, which contains a list of valid synset labels used - * by ILSVRC competition. There is one synset one per line, eg. - * # n01440764 - * # n01443537 - * We also retrieve a synset_to_human_file, which contains a mapping from synsets - * to human-readable names for every synset in Imagenet. These are stored in a - * tsv format, as follows: - * # n02119247 black fox - * # n02119359 silver fox - * We assign each synset (in alphabetical order) an integer, starting from 1 - * (since 0 is reserved for the background class) - * - * @author Christian Tzolov - */ -public final class ImageNetReadableNamesWriter { - - private ImageNetReadableNamesWriter() { - } - - /** BASE_URL. */ - public final static String BASE_URL = "https://raw.githubusercontent.com/tensorflow/models/master/research/inception/inception/data/"; - /** SYNSET_URI. */ - public final static String SYNSET_URI = BASE_URL + "imagenet_lsvrc_2015_synsets.txt"; - /** SYNSET_TO_HUMAN_URI. */ - public final static String SYNSET_TO_HUMAN_URI = BASE_URL + "imagenet_metadata.txt"; - - public static void main(String[] args) { - Charset utf8 = Charset.forName("UTF-8"); - - try (InputStream synsetIs = toResource(SYNSET_URI).getInputStream(); - InputStream synsetToHumanIs = toResource(SYNSET_TO_HUMAN_URI).getInputStream()) { - - List synsetList = Arrays.asList(StreamUtils.copyToString(synsetIs, utf8) - .split("\n")).stream().map(l -> l.trim()).collect(Collectors.toList()); - Assert.notNull(synsetList, "Failed to initialize the labels list"); - Assert.isTrue(synsetList.size() == 1000, "Labels list is expected to be of " + - "size 1000 but was:" + synsetList.size()); - - Map synsetToHuman = Arrays.asList(StreamUtils.copyToString(synsetToHumanIs, utf8) - .split("\n")).stream().map(s2h -> s2h.split("\t")).collect(Collectors.toMap(s -> s[0], s -> s[1])); - Assert.notNull(synsetToHuman, "Failed to initialize the synsetToHuman"); - Assert.isTrue(synsetToHuman.size() == 21842, "synsetToHuman is expected to be of " + - "size 21842 but was:" + synsetToHuman.size()); - - List l = synsetList.stream().map(id -> synsetToHuman.get(id)).collect(Collectors.toList()); - - List ll = new ArrayList<>(); - ll.add("dummy"); - ll.addAll(l); - System.out.println(ll.get(389)); - FileUtils.writeLines(new File("labels.txt"), ll); - - } - catch (IOException e) { - throw new RuntimeException("Failed to initialize the Vocabulary", e); - } - } - - public static Resource toResource(String uri) { - return new DefaultResourceLoader().getResource(uri); - } -} diff --git a/functions/function/image-recognition-function/src/main/resources/images/image-augmented.jpg b/functions/function/image-recognition-function/src/main/resources/images/image-augmented.jpg deleted file mode 100644 index 472a3532b..000000000 Binary files a/functions/function/image-recognition-function/src/main/resources/images/image-augmented.jpg and /dev/null differ diff --git a/functions/function/image-recognition-function/src/main/resources/images/image-recognition-panda-augmented.jpg b/functions/function/image-recognition-function/src/main/resources/images/image-recognition-panda-augmented.jpg deleted file mode 100644 index 3cfc4ae64..000000000 Binary files a/functions/function/image-recognition-function/src/main/resources/images/image-recognition-panda-augmented.jpg and /dev/null differ diff --git a/functions/function/image-recognition-function/src/main/resources/labels/inception_labels.txt b/functions/function/image-recognition-function/src/main/resources/labels/inception_labels.txt deleted file mode 100644 index 0ac5a169d..000000000 --- a/functions/function/image-recognition-function/src/main/resources/labels/inception_labels.txt +++ /dev/null @@ -1,1001 +0,0 @@ -dummy -kit fox -English setter -Siberian husky -Australian terrier -English springer -grey whale -lesser panda -Egyptian cat -ibex -Persian cat -cougar -gazelle -porcupine -sea lion -malamute -badger -Great Dane -Walker hound -Welsh springer spaniel -whippet -Scottish deerhound -killer whale -mink -African elephant -Weimaraner -soft-coated wheaten terrier -Dandie Dinmont -red wolf -Old English sheepdog -jaguar -otterhound -bloodhound -Airedale -hyena -meerkat -giant schnauzer -titi -three-toed sloth -sorrel -black-footed ferret -dalmatian -black-and-tan coonhound -papillon -skunk -Staffordshire bullterrier -Mexican hairless -Bouvier des Flandres -weasel -miniature poodle -Cardigan -malinois -bighorn -fox squirrel -colobus -tiger cat -Lhasa -impala -coyote -Yorkshire terrier -Newfoundland -brown bear -red fox -Norwegian elkhound -Rottweiler -hartebeest -Saluki -grey fox -schipperke -Pekinese -Brabancon griffon -West Highland white terrier -Sealyham terrier -guenon -mongoose -indri -tiger -Irish wolfhound -wild boar -EntleBucher -zebra -ram -French bulldog -orangutan -basenji -leopard -Bernese mountain dog -Maltese dog -Norfolk terrier -toy terrier -vizsla -cairn -squirrel monkey -groenendael -clumber -Siamese cat -chimpanzee -komondor -Afghan hound -Japanese spaniel -proboscis monkey -guinea pig -white wolf -ice bear -gorilla -borzoi -toy poodle -Kerry blue terrier -ox -Scotch terrier -Tibetan mastiff -spider monkey -Doberman -Boston bull -Greater Swiss Mountain dog -Appenzeller -Shih-Tzu -Irish water spaniel -Pomeranian -Bedlington terrier -warthog -Arabian camel -siamang -miniature schnauzer -collie -golden retriever -Irish terrier -affenpinscher -Border collie -hare -boxer -silky terrier -beagle -Leonberg -German short-haired pointer -patas -dhole -baboon -macaque -Chesapeake Bay retriever -bull mastiff -kuvasz -capuchin -pug -curly-coated retriever -Norwich terrier -flat-coated retriever -hog -keeshond -Eskimo dog -Brittany spaniel -standard poodle -Lakeland terrier -snow leopard -Gordon setter -dingo -standard schnauzer -hamster -Tibetan terrier -Arctic fox -wire-haired fox terrier -basset -water buffalo -American black bear -Angora -bison -howler monkey -hippopotamus -chow -giant panda -American Staffordshire terrier -Shetland sheepdog -Great Pyrenees -Chihuahua -tabby -marmoset -Labrador retriever -Saint Bernard -armadillo -Samoyed -bluetick -redbone -polecat -marmot -kelpie -gibbon -llama -miniature pinscher -wood rabbit -Italian greyhound -lion -cocker spaniel -Irish setter -dugong -Indian elephant -beaver -Sussex spaniel -Pembroke -Blenheim spaniel -Madagascar cat -Rhodesian ridgeback -lynx -African hunting dog -langur -Ibizan hound -timber wolf -cheetah -English foxhound -briard -sloth bear -Border terrier -German shepherd -otter -koala -tusker -echidna -wallaby -platypus -wombat -revolver -umbrella -schooner -soccer ball -accordion -ant -starfish -chambered nautilus -grand piano -laptop -strawberry -airliner -warplane -airship -balloon -space shuttle -fireboat -gondola -speedboat -lifeboat -canoe -yawl -catamaran -trimaran -container ship -liner -pirate -aircraft carrier -submarine -wreck -half track -tank -missile -bobsled -dogsled -bicycle-built-for-two -mountain bike -freight car -passenger car -barrow -shopping cart -motor scooter -forklift -electric locomotive -steam locomotive -amphibian -ambulance -beach wagon -cab -convertible -jeep -limousine -minivan -Model T -racer -sports car -go-kart -golfcart -moped -snowplow -fire engine -garbage truck -pickup -tow truck -trailer truck -moving van -police van -recreational vehicle -streetcar -snowmobile -tractor -mobile home -tricycle -unicycle -horse cart -jinrikisha -oxcart -bassinet -cradle -crib -four-poster -bookcase -china cabinet -medicine chest -chiffonier -table lamp -file -park bench -barber chair -throne -folding chair -rocking chair -studio couch -toilet seat -desk -pool table -dining table -entertainment center -wardrobe -Granny Smith -orange -lemon -fig -pineapple -banana -jackfruit -custard apple -pomegranate -acorn -hip -ear -rapeseed -corn -buckeye -organ -upright -chime -drum -gong -maraca -marimba -steel drum -banjo -cello -violin -harp -acoustic guitar -electric guitar -cornet -French horn -trombone -harmonica -ocarina -panpipe -bassoon -oboe -sax -flute -daisy -yellow lady's slipper -cliff -valley -alp -volcano -promontory -sandbar -coral reef -lakeside -seashore -geyser -hatchet -cleaver -letter opener -plane -power drill -lawn mower -hammer -corkscrew -can opener -plunger -screwdriver -shovel -plow -chain saw -cock -hen -ostrich -brambling -goldfinch -house finch -junco -indigo bunting -robin -bulbul -jay -magpie -chickadee -water ouzel -kite -bald eagle -vulture -great grey owl -black grouse -ptarmigan -ruffed grouse -prairie chicken -peacock -quail -partridge -African grey -macaw -sulphur-crested cockatoo -lorikeet -coucal -bee eater -hornbill -hummingbird -jacamar -toucan -drake -red-breasted merganser -goose -black swan -white stork -black stork -spoonbill -flamingo -American egret -little blue heron -bittern -crane -limpkin -American coot -bustard -ruddy turnstone -red-backed sandpiper -redshank -dowitcher -oystercatcher -European gallinule -pelican -king penguin -albatross -great white shark -tiger shark -hammerhead -electric ray -stingray -barracouta -coho -tench -goldfish -eel -rock beauty -anemone fish -lionfish -puffer -sturgeon -gar -loggerhead -leatherback turtle -mud turtle -terrapin -box turtle -banded gecko -common iguana -American chameleon -whiptail -agama -frilled lizard -alligator lizard -Gila monster -green lizard -African chameleon -Komodo dragon -triceratops -African crocodile -American alligator -thunder snake -ringneck snake -hognose snake -green snake -king snake -garter snake -water snake -vine snake -night snake -boa constrictor -rock python -Indian cobra -green mamba -sea snake -horned viper -diamondback -sidewinder -European fire salamander -common newt -eft -spotted salamander -axolotl -bullfrog -tree frog -tailed frog -whistle -wing -paintbrush -hand blower -oxygen mask -snorkel -loudspeaker -microphone -screen -mouse -electric fan -oil filter -strainer -space heater -stove -guillotine -barometer -rule -odometer -scale -analog clock -digital clock -wall clock -hourglass -sundial -parking meter -stopwatch -digital watch -stethoscope -syringe -magnetic compass -binoculars -projector -sunglasses -loupe -radio telescope -bow -cannon [ground] -assault rifle -rifle -projectile -computer keyboard -typewriter keyboard -crane -lighter -abacus -cash machine -slide rule -desktop computer -hand-held computer -notebook -web site -harvester -thresher -printer -slot -vending machine -sewing machine -joystick -switch -hook -car wheel -paddlewheel -pinwheel -potter's wheel -gas pump -carousel -swing -reel -radiator -puck -hard disc -sunglass -pick -car mirror -solar dish -remote control -disk brake -buckle -hair slide -knot -combination lock -padlock -nail -safety pin -screw -muzzle -seat belt -ski -candle -jack-o'-lantern -spotlight -torch -neck brace -pier -tripod -maypole -mousetrap -spider web -trilobite -harvestman -scorpion -black and gold garden spider -barn spider -garden spider -black widow -tarantula -wolf spider -tick -centipede -isopod -Dungeness crab -rock crab -fiddler crab -king crab -American lobster -spiny lobster -crayfish -hermit crab -tiger beetle -ladybug -ground beetle -long-horned beetle -leaf beetle -dung beetle -rhinoceros beetle -weevil -fly -bee -grasshopper -cricket -walking stick -cockroach -mantis -cicada -leafhopper -lacewing -dragonfly -damselfly -admiral -ringlet -monarch -cabbage butterfly -sulphur butterfly -lycaenid -jellyfish -sea anemone -brain coral -flatworm -nematode -conch -snail -slug -sea slug -chiton -sea urchin -sea cucumber -iron -espresso maker -microwave -Dutch oven -rotisserie -toaster -waffle iron -vacuum -dishwasher -refrigerator -washer -Crock Pot -frying pan -wok -caldron -coffeepot -teapot -spatula -altar -triumphal arch -patio -steel arch bridge -suspension bridge -viaduct -barn -greenhouse -palace -monastery -library -apiary -boathouse -church -mosque -stupa -planetarium -restaurant -cinema -home theater -lumbermill -coil -obelisk -totem pole -castle -prison -grocery store -bakery -barbershop -bookshop -butcher shop -confectionery -shoe shop -tobacco shop -toyshop -fountain -cliff dwelling -yurt -dock -brass -megalith -bannister -breakwater -dam -chainlink fence -picket fence -worm fence -stone wall -grille -sliding door -turnstile -mountain tent -scoreboard -honeycomb -plate rack -pedestal -beacon -mashed potato -bell pepper -head cabbage -broccoli -cauliflower -zucchini -spaghetti squash -acorn squash -butternut squash -cucumber -artichoke -cardoon -mushroom -shower curtain -jean -carton -handkerchief -sandal -ashcan -safe -plate -necklace -croquet ball -fur coat -thimble -pajama -running shoe -cocktail shaker -chest -manhole cover -modem -tub -tray -balance beam -bagel -prayer rug -kimono -hot pot -whiskey jug -knee pad -book jacket -spindle -ski mask -beer bottle -crash helmet -bottlecap -tile roof -mask -maillot -Petri dish -football helmet -bathing cap -teddy bear -holster -pop bottle -photocopier -vestment -crossword puzzle -golf ball -trifle -suit -water tower -feather boa -cloak -red wine -drumstick -shield -Christmas stocking -hoopskirt -menu -stage -bonnet -meat loaf -baseball -face powder -scabbard -sunscreen -beer glass -hen-of-the-woods -guacamole -lampshade -wool -hay -bow tie -mailbag -water jug -bucket -dishrag -soup bowl -eggnog -mortar -trench coat -paddle -chain -swab -mixing bowl -potpie -wine bottle -shoji -bulletproof vest -drilling platform -binder -cardigan -sweatshirt -pot -birdhouse -hamper -ping-pong ball -pencil box -pay-phone -consomme -apron -punching bag -backpack -groom -bearskin -pencil sharpener -broom -mosquito net -abaya -mortarboard -poncho -crutch -Polaroid camera -space bar -cup -racket -traffic light -quill -radio -dough -cuirass -military uniform -lipstick -shower cap -monitor -oscilloscope -mitten -brassiere -French loaf -vase -milk can -rugby ball -paper towel -earthstar -envelope -miniskirt -cowboy hat -trolleybus -perfume -bathtub -hotdog -coral fungus -bullet train -pillow -toilet tissue -cassette -carpenter's kit -ladle -stinkhorn -lotion -hair spray -academic gown -dome -crate -wig -burrito -pill bottle -chain mail -theater curtain -window shade -barrel -washbasin -ballpoint -basketball -bath towel -cowboy boot -gown -window screen -agaric -cellular telephone -nipple -barbell -mailbox -lab coat -fire screen -minibus -packet -maze -pole -horizontal bar -sombrero -pickelhaube -rain barrel -wallet -cassette player -comic book -piggy bank -street sign -bell cote -fountain pen -Windsor tie -volleyball -overskirt -sarong -purse -bolo tie -bib -parachute -sleeping bag -television -swimming trunks -measuring cup -espresso -pizza -breastplate -shopping basket -wooden spoon -saltshaker -chocolate sauce -ballplayer -goblet -gyromitra -stretcher -water bottle -dial telephone -soap dispenser -jersey -school bus -jigsaw puzzle -plastic bag -reflex camera -diaper -Band Aid -ice lolly -velvet -tennis ball -gasmask -doormat -Loafer -ice cream -pretzel -quilt -maillot -tape player -clog -iPod -bolete -scuba diver -pitcher -matchstick -bikini -sock -CD player -lens cap -thatch -vault -beaker -bubble -cheeseburger -parallel bars -flagpole -coffee mug -rubber eraser -stole -carbonara -dumbbell \ No newline at end of file diff --git a/functions/function/image-recognition-function/src/main/resources/labels/mobilenet_labels.txt b/functions/function/image-recognition-function/src/main/resources/labels/mobilenet_labels.txt deleted file mode 100644 index a6d95b46b..000000000 --- a/functions/function/image-recognition-function/src/main/resources/labels/mobilenet_labels.txt +++ /dev/null @@ -1,1001 +0,0 @@ -dummy -tench, Tinca tinca -goldfish, Carassius auratus -great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias -tiger shark, Galeocerdo cuvieri -hammerhead, hammerhead shark -electric ray, crampfish, numbfish, torpedo -stingray -cock -hen -ostrich, Struthio camelus -brambling, Fringilla montifringilla -goldfinch, Carduelis carduelis -house finch, linnet, Carpodacus mexicanus -junco, snowbird -indigo bunting, indigo finch, indigo bird, Passerina cyanea -robin, American robin, Turdus migratorius -bulbul -jay -magpie -chickadee -water ouzel, dipper -kite -bald eagle, American eagle, Haliaeetus leucocephalus -vulture -great grey owl, great gray owl, Strix nebulosa -European fire salamander, Salamandra salamandra -common newt, Triturus vulgaris -eft -spotted salamander, Ambystoma maculatum -axolotl, mud puppy, Ambystoma mexicanum -bullfrog, Rana catesbeiana -tree frog, tree-frog -tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui -loggerhead, loggerhead turtle, Caretta caretta -leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea -mud turtle -terrapin -box turtle, box tortoise -banded gecko -common iguana, iguana, Iguana iguana -American chameleon, anole, Anolis carolinensis -whiptail, whiptail lizard -agama -frilled lizard, Chlamydosaurus kingi -alligator lizard -Gila monster, Heloderma suspectum -green lizard, Lacerta viridis -African chameleon, Chamaeleo chamaeleon -Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis -African crocodile, Nile crocodile, Crocodylus niloticus -American alligator, Alligator mississipiensis -triceratops -thunder snake, worm snake, Carphophis amoenus -ringneck snake, ring-necked snake, ring snake -hognose snake, puff adder, sand viper -green snake, grass snake -king snake, kingsnake -garter snake, grass snake -water snake -vine snake -night snake, Hypsiglena torquata -boa constrictor, Constrictor constrictor -rock python, rock snake, Python sebae -Indian cobra, Naja naja -green mamba -sea snake -horned viper, cerastes, sand viper, horned asp, Cerastes cornutus -diamondback, diamondback rattlesnake, Crotalus adamanteus -sidewinder, horned rattlesnake, Crotalus cerastes -trilobite -harvestman, daddy longlegs, Phalangium opilio -scorpion -black and gold garden spider, Argiope aurantia -barn spider, Araneus cavaticus -garden spider, Aranea diademata -black widow, Latrodectus mactans -tarantula -wolf spider, hunting spider -tick -centipede -black grouse -ptarmigan -ruffed grouse, partridge, Bonasa umbellus -prairie chicken, prairie grouse, prairie fowl -peacock -quail -partridge -African grey, African gray, Psittacus erithacus -macaw -sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita -lorikeet -coucal -bee eater -hornbill -hummingbird -jacamar -toucan -drake -red-breasted merganser, Mergus serrator -goose -black swan, Cygnus atratus -tusker -echidna, spiny anteater, anteater -platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus -wallaby, brush kangaroo -koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus -wombat -jellyfish -sea anemone, anemone -brain coral -flatworm, platyhelminth -nematode, nematode worm, roundworm -conch -snail -slug -sea slug, nudibranch -chiton, coat-of-mail shell, sea cradle, polyplacophore -chambered nautilus, pearly nautilus, nautilus -Dungeness crab, Cancer magister -rock crab, Cancer irroratus -fiddler crab -king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica -American lobster, Northern lobster, Maine lobster, Homarus americanus -spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish -crayfish, crawfish, crawdad, crawdaddy -hermit crab -isopod -white stork, Ciconia ciconia -black stork, Ciconia nigra -spoonbill -flamingo -little blue heron, Egretta caerulea -American egret, great white heron, Egretta albus -bittern -crane -limpkin, Aramus pictus -European gallinule, Porphyrio porphyrio -American coot, marsh hen, mud hen, water hen, Fulica americana -bustard -ruddy turnstone, Arenaria interpres -red-backed sandpiper, dunlin, Erolia alpina -redshank, Tringa totanus -dowitcher -oystercatcher, oyster catcher -pelican -king penguin, Aptenodytes patagonica -albatross, mollymawk -grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus -killer whale, killer, orca, grampus, sea wolf, Orcinus orca -dugong, Dugong dugon -sea lion -Chihuahua -Japanese spaniel -Maltese dog, Maltese terrier, Maltese -Pekinese, Pekingese, Peke -Shih-Tzu -Blenheim spaniel -papillon -toy terrier -Rhodesian ridgeback -Afghan hound, Afghan -basset, basset hound -beagle -bloodhound, sleuthhound -bluetick -black-and-tan coonhound -Walker hound, Walker foxhound -English foxhound -redbone -borzoi, Russian wolfhound -Irish wolfhound -Italian greyhound -whippet -Ibizan hound, Ibizan Podenco -Norwegian elkhound, elkhound -otterhound, otter hound -Saluki, gazelle hound -Scottish deerhound, deerhound -Weimaraner -Staffordshire bullterrier, Staffordshire bull terrier -American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier -Bedlington terrier -Border terrier -Kerry blue terrier -Irish terrier -Norfolk terrier -Norwich terrier -Yorkshire terrier -wire-haired fox terrier -Lakeland terrier -Sealyham terrier, Sealyham -Airedale, Airedale terrier -cairn, cairn terrier -Australian terrier -Dandie Dinmont, Dandie Dinmont terrier -Boston bull, Boston terrier -miniature schnauzer -giant schnauzer -standard schnauzer -Scotch terrier, Scottish terrier, Scottie -Tibetan terrier, chrysanthemum dog -silky terrier, Sydney silky -soft-coated wheaten terrier -West Highland white terrier -Lhasa, Lhasa apso -flat-coated retriever -curly-coated retriever -golden retriever -Labrador retriever -Chesapeake Bay retriever -German short-haired pointer -vizsla, Hungarian pointer -English setter -Irish setter, red setter -Gordon setter -Brittany spaniel -clumber, clumber spaniel -English springer, English springer spaniel -Welsh springer spaniel -cocker spaniel, English cocker spaniel, cocker -Sussex spaniel -Irish water spaniel -kuvasz -schipperke -groenendael -malinois -briard -kelpie -komondor -Old English sheepdog, bobtail -Shetland sheepdog, Shetland sheep dog, Shetland -collie -Border collie -Bouvier des Flandres, Bouviers des Flandres -Rottweiler -German shepherd, German shepherd dog, German police dog, alsatian -Doberman, Doberman pinscher -miniature pinscher -Greater Swiss Mountain dog -Bernese mountain dog -Appenzeller -EntleBucher -boxer -bull mastiff -Tibetan mastiff -French bulldog -Great Dane -Saint Bernard, St Bernard -Eskimo dog, husky -malamute, malemute, Alaskan malamute -Siberian husky -dalmatian, coach dog, carriage dog -affenpinscher, monkey pinscher, monkey dog -basenji -pug, pug-dog -Leonberg -Newfoundland, Newfoundland dog -Great Pyrenees -Samoyed, Samoyede -Pomeranian -chow, chow chow -keeshond -Brabancon griffon -Pembroke, Pembroke Welsh corgi -Cardigan, Cardigan Welsh corgi -toy poodle -miniature poodle -standard poodle -Mexican hairless -timber wolf, grey wolf, gray wolf, Canis lupus -white wolf, Arctic wolf, Canis lupus tundrarum -red wolf, maned wolf, Canis rufus, Canis niger -coyote, prairie wolf, brush wolf, Canis latrans -dingo, warrigal, warragal, Canis dingo -dhole, Cuon alpinus -African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus -hyena, hyaena -red fox, Vulpes vulpes -kit fox, Vulpes macrotis -Arctic fox, white fox, Alopex lagopus -grey fox, gray fox, Urocyon cinereoargenteus -tabby, tabby cat -tiger cat -Persian cat -Siamese cat, Siamese -Egyptian cat -cougar, puma, catamount, mountain lion, painter, panther, Felis concolor -lynx, catamount -leopard, Panthera pardus -snow leopard, ounce, Panthera uncia -jaguar, panther, Panthera onca, Felis onca -lion, king of beasts, Panthera leo -tiger, Panthera tigris -cheetah, chetah, Acinonyx jubatus -brown bear, bruin, Ursus arctos -American black bear, black bear, Ursus americanus, Euarctos americanus -ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus -sloth bear, Melursus ursinus, Ursus ursinus -mongoose -meerkat, mierkat -tiger beetle -ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle -ground beetle, carabid beetle -long-horned beetle, longicorn, longicorn beetle -leaf beetle, chrysomelid -dung beetle -rhinoceros beetle -weevil -fly -bee -ant, emmet, pismire -grasshopper, hopper -cricket -walking stick, walkingstick, stick insect -cockroach, roach -mantis, mantid -cicada, cicala -leafhopper -lacewing, lacewing fly -dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk -damselfly -admiral -ringlet, ringlet butterfly -monarch, monarch butterfly, milkweed butterfly, Danaus plexippus -cabbage butterfly -sulphur butterfly, sulfur butterfly -lycaenid, lycaenid butterfly -starfish, sea star -sea urchin -sea cucumber, holothurian -wood rabbit, cottontail, cottontail rabbit -hare -Angora, Angora rabbit -hamster -porcupine, hedgehog -fox squirrel, eastern fox squirrel, Sciurus niger -marmot -beaver -guinea pig, Cavia cobaya -sorrel -zebra -hog, pig, grunter, squealer, Sus scrofa -wild boar, boar, Sus scrofa -warthog -hippopotamus, hippo, river horse, Hippopotamus amphibius -ox -water buffalo, water ox, Asiatic buffalo, Bubalus bubalis -bison -ram, tup -bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis -ibex, Capra ibex -hartebeest -impala, Aepyceros melampus -gazelle -Arabian camel, dromedary, Camelus dromedarius -llama -weasel -mink -polecat, fitch, foulmart, foumart, Mustela putorius -black-footed ferret, ferret, Mustela nigripes -otter -skunk, polecat, wood pussy -badger -armadillo -three-toed sloth, ai, Bradypus tridactylus -orangutan, orang, orangutang, Pongo pygmaeus -gorilla, Gorilla gorilla -chimpanzee, chimp, Pan troglodytes -gibbon, Hylobates lar -siamang, Hylobates syndactylus, Symphalangus syndactylus -guenon, guenon monkey -patas, hussar monkey, Erythrocebus patas -baboon -macaque -langur -colobus, colobus monkey -proboscis monkey, Nasalis larvatus -marmoset -capuchin, ringtail, Cebus capucinus -howler monkey, howler -titi, titi monkey -spider monkey, Ateles geoffroyi -squirrel monkey, Saimiri sciureus -Madagascar cat, ring-tailed lemur, Lemur catta -indri, indris, Indri indri, Indri brevicaudatus -Indian elephant, Elephas maximus -African elephant, Loxodonta africana -lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens -giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca -barracouta, snoek -eel -coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch -rock beauty, Holocanthus tricolor -anemone fish -sturgeon -gar, garfish, garpike, billfish, Lepisosteus osseus -lionfish -puffer, pufferfish, blowfish, globefish -abacus -abaya -academic gown, academic robe, judge's robe -accordion, piano accordion, squeeze box -acoustic guitar -aircraft carrier, carrier, flattop, attack aircraft carrier -airliner -airship, dirigible -altar -ambulance -amphibian, amphibious vehicle -analog clock -apiary, bee house -apron -ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin -assault rifle, assault gun -backpack, back pack, knapsack, packsack, rucksack, haversack -bakery, bakeshop, bakehouse -balance beam, beam -balloon -ballpoint, ballpoint pen, ballpen, Biro -Band Aid -banjo -bannister, banister, balustrade, balusters, handrail -barbell -barber chair -barbershop -barn -barometer -barrel, cask -barrow, garden cart, lawn cart, wheelbarrow -baseball -basketball -bassinet -bassoon -bathing cap, swimming cap -bath towel -bathtub, bathing tub, bath, tub -beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon -beacon, lighthouse, beacon light, pharos -beaker -bearskin, busby, shako -beer bottle -beer glass -bell cote, bell cot -bib -bicycle-built-for-two, tandem bicycle, tandem -bikini, two-piece -binder, ring-binder -binoculars, field glasses, opera glasses -birdhouse -boathouse -bobsled, bobsleigh, bob -bolo tie, bolo, bola tie, bola -bonnet, poke bonnet -bookcase -bookshop, bookstore, bookstall -bottlecap -bow -bow tie, bow-tie, bowtie -brass, memorial tablet, plaque -brassiere, bra, bandeau -breakwater, groin, groyne, mole, bulwark, seawall, jetty -breastplate, aegis, egis -broom -bucket, pail -buckle -bulletproof vest -bullet train, bullet -butcher shop, meat market -cab, hack, taxi, taxicab -caldron, cauldron -candle, taper, wax light -cannon -canoe -can opener, tin opener -cardigan -car mirror -carousel, carrousel, merry-go-round, roundabout, whirligig -carpenter's kit, tool kit -carton -car wheel -cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM -cassette -cassette player -castle -catamaran -CD player -cello, violoncello -cellular telephone, cellular phone, cellphone, cell, mobile phone -chain -chainlink fence -chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour -chain saw, chainsaw -chest -chiffonier, commode -chime, bell, gong -china cabinet, china closet -Christmas stocking -church, church building -cinema, movie theater, movie theatre, movie house, picture palace -cleaver, meat cleaver, chopper -cliff dwelling -cloak -clog, geta, patten, sabot -cocktail shaker -coffee mug -coffeepot -coil, spiral, volute, whorl, helix -combination lock -computer keyboard, keypad -confectionery, confectionary, candy store -container ship, containership, container vessel -convertible -corkscrew, bottle screw -cornet, horn, trumpet, trump -cowboy boot -cowboy hat, ten-gallon hat -cradle -crane -crash helmet -crate -crib, cot -Crock Pot -croquet ball -crutch -cuirass -dam, dike, dyke -desk -desktop computer -dial telephone, dial phone -diaper, nappy, napkin -digital clock -digital watch -dining table, board -dishrag, dishcloth -dishwasher, dish washer, dishwashing machine -disk brake, disc brake -dock, dockage, docking facility -dogsled, dog sled, dog sleigh -dome -doormat, welcome mat -drilling platform, offshore rig -drum, membranophone, tympan -drumstick -dumbbell -Dutch oven -electric fan, blower -electric guitar -electric locomotive -entertainment center -envelope -espresso maker -face powder -feather boa, boa -file, file cabinet, filing cabinet -fireboat -fire engine, fire truck -fire screen, fireguard -flagpole, flagstaff -flute, transverse flute -folding chair -football helmet -forklift -fountain -fountain pen -four-poster -freight car -French horn, horn -frying pan, frypan, skillet -fur coat -garbage truck, dustcart -gasmask, respirator, gas helmet -gas pump, gasoline pump, petrol pump, island dispenser -goblet -go-kart -golf ball -golfcart, golf cart -gondola -gong, tam-tam -gown -grand piano, grand -greenhouse, nursery, glasshouse -grille, radiator grille -grocery store, grocery, food market, market -guillotine -hair slide -hair spray -half track -hammer -hamper -hand blower, blow dryer, blow drier, hair dryer, hair drier -hand-held computer, hand-held microcomputer -handkerchief, hankie, hanky, hankey -hard disc, hard disk, fixed disk -harmonica, mouth organ, harp, mouth harp -harp -harvester, reaper -hatchet -holster -home theater, home theatre -honeycomb -hook, claw -hoopskirt, crinoline -horizontal bar, high bar -horse cart, horse-cart -hourglass -iPod -iron, smoothing iron -jack-o'-lantern -jean, blue jean, denim -jeep, landrover -jersey, T-shirt, tee shirt -jigsaw puzzle -jinrikisha, ricksha, rickshaw -joystick -kimono -knee pad -knot -lab coat, laboratory coat -ladle -lampshade, lamp shade -laptop, laptop computer -lawn mower, mower -lens cap, lens cover -letter opener, paper knife, paperknife -library -lifeboat -lighter, light, igniter, ignitor -limousine, limo -liner, ocean liner -lipstick, lip rouge -Loafer -lotion -loudspeaker, speaker, speaker unit, loudspeaker system, speaker system -loupe, jeweler's loupe -lumbermill, sawmill -magnetic compass -mailbag, postbag -mailbox, letter box -maillot -maillot, tank suit -manhole cover -maraca -marimba, xylophone -mask -matchstick -maypole -maze, labyrinth -measuring cup -medicine chest, medicine cabinet -megalith, megalithic structure -microphone, mike -microwave, microwave oven -military uniform -milk can -minibus -miniskirt, mini -minivan -missile -mitten -mixing bowl -mobile home, manufactured home -Model T -modem -monastery -monitor -moped -mortar -mortarboard -mosque -mosquito net -motor scooter, scooter -mountain bike, all-terrain bike, off-roader -mountain tent -mouse, computer mouse -mousetrap -moving van -muzzle -nail -neck brace -necklace -nipple -notebook, notebook computer -obelisk -oboe, hautboy, hautbois -ocarina, sweet potato -odometer, hodometer, mileometer, milometer -oil filter -organ, pipe organ -oscilloscope, scope, cathode-ray oscilloscope, CRO -overskirt -oxcart -oxygen mask -packet -paddle, boat paddle -paddlewheel, paddle wheel -padlock -paintbrush -pajama, pyjama, pj's, jammies -palace -panpipe, pandean pipe, syrinx -paper towel -parachute, chute -parallel bars, bars -park bench -parking meter -passenger car, coach, carriage -patio, terrace -pay-phone, pay-station -pedestal, plinth, footstall -pencil box, pencil case -pencil sharpener -perfume, essence -Petri dish -photocopier -pick, plectrum, plectron -pickelhaube -picket fence, paling -pickup, pickup truck -pier -piggy bank, penny bank -pill bottle -pillow -ping-pong ball -pinwheel -pirate, pirate ship -pitcher, ewer -plane, carpenter's plane, woodworking plane -planetarium -plastic bag -plate rack -plow, plough -plunger, plumber's helper -Polaroid camera, Polaroid Land camera -pole -police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria -poncho -pool table, billiard table, snooker table -pop bottle, soda bottle -pot, flowerpot -potter's wheel -power drill -prayer rug, prayer mat -printer -prison, prison house -projectile, missile -projector -puck, hockey puck -punching bag, punch bag, punching ball, punchball -purse -quill, quill pen -quilt, comforter, comfort, puff -racer, race car, racing car -racket, racquet -radiator -radio, wireless -radio telescope, radio reflector -rain barrel -recreational vehicle, RV, R.V. -reel -reflex camera -refrigerator, icebox -remote control, remote -restaurant, eating house, eating place, eatery -revolver, six-gun, six-shooter -rifle -rocking chair, rocker -rotisserie -rubber eraser, rubber, pencil eraser -rugby ball -rule, ruler -running shoe -safe -safety pin -saltshaker, salt shaker -sandal -sarong -sax, saxophone -scabbard -scale, weighing machine -school bus -schooner -scoreboard -screen, CRT screen -screw -screwdriver -seat belt, seatbelt -sewing machine -shield, buckler -shoe shop, shoe-shop, shoe store -shoji -shopping basket -shopping cart -shovel -shower cap -shower curtain -ski -ski mask -sleeping bag -slide rule, slipstick -sliding door -slot, one-armed bandit -snorkel -snowmobile -snowplow, snowplough -soap dispenser -soccer ball -sock -solar dish, solar collector, solar furnace -sombrero -soup bowl -space bar -space heater -space shuttle -spatula -speedboat -spider web, spider's web -spindle -sports car, sport car -spotlight, spot -stage -steam locomotive -steel arch bridge -steel drum -stethoscope -stole -stone wall -stopwatch, stop watch -stove -strainer -streetcar, tram, tramcar, trolley, trolley car -stretcher -studio couch, day bed -stupa, tope -submarine, pigboat, sub, U-boat -suit, suit of clothes -sundial -sunglass -sunglasses, dark glasses, shades -sunscreen, sunblock, sun blocker -suspension bridge -swab, swob, mop -sweatshirt -swimming trunks, bathing trunks -swing -switch, electric switch, electrical switch -syringe -table lamp -tank, army tank, armored combat vehicle, armoured combat vehicle -tape player -teapot -teddy, teddy bear -television, television system -tennis ball -thatch, thatched roof -theater curtain, theatre curtain -thimble -thresher, thrasher, threshing machine -throne -tile roof -toaster -tobacco shop, tobacconist shop, tobacconist -toilet seat -torch -totem pole -tow truck, tow car, wrecker -toyshop -tractor -trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi -tray -trench coat -tricycle, trike, velocipede -trimaran -tripod -triumphal arch -trolleybus, trolley coach, trackless trolley -trombone -tub, vat -turnstile -typewriter keyboard -umbrella -unicycle, monocycle -upright, upright piano -vacuum, vacuum cleaner -vase -vault -velvet -vending machine -vestment -viaduct -violin, fiddle -volleyball -waffle iron -wall clock -wallet, billfold, notecase, pocketbook -wardrobe, closet, press -warplane, military plane -washbasin, handbasin, washbowl, lavabo, wash-hand basin -washer, automatic washer, washing machine -water bottle -water jug -water tower -whiskey jug -whistle -wig -window screen -window shade -Windsor tie -wine bottle -wing -wok -wooden spoon -wool, woolen, woollen -worm fence, snake fence, snake-rail fence, Virginia fence -wreck -yawl -yurt -web site, website, internet site, site -comic book -crossword puzzle, crossword -street sign -traffic light, traffic signal, stoplight -book jacket, dust cover, dust jacket, dust wrapper -menu -plate -guacamole -consomme -hot pot, hotpot -trifle -ice cream, icecream -ice lolly, lolly, lollipop, popsicle -French loaf -bagel, beigel -pretzel -cheeseburger -hotdog, hot dog, red hot -mashed potato -head cabbage -broccoli -cauliflower -zucchini, courgette -spaghetti squash -acorn squash -butternut squash -cucumber, cuke -artichoke, globe artichoke -bell pepper -cardoon -mushroom -Granny Smith -strawberry -orange -lemon -fig -pineapple, ananas -banana -jackfruit, jak, jack -custard apple -pomegranate -hay -carbonara -chocolate sauce, chocolate syrup -dough -meat loaf, meatloaf -pizza, pizza pie -potpie -burrito -red wine -espresso -cup -eggnog -alp -bubble -cliff, drop, drop-off -coral reef -geyser -lakeside, lakeshore -promontory, headland, head, foreland -sandbar, sand bar -seashore, coast, seacoast, sea-coast -valley, vale -volcano -ballplayer, baseball player -groom, bridegroom -scuba diver -rapeseed -daisy -yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum -corn -acorn -hip, rose hip, rosehip -buckeye, horse chestnut, conker -coral fungus -agaric -gyromitra -stinkhorn, carrion fungus -earthstar -hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa -bolete -ear, spike, capitulum -toilet tissue, toilet paper, bathroom tissue diff --git a/functions/function/image-recognition-function/src/test/java/org/springframework/cloud/fn/image/recognition/ImageRecognitionExample.java b/functions/function/image-recognition-function/src/test/java/org/springframework/cloud/fn/image/recognition/ImageRecognitionExample.java deleted file mode 100644 index 4f441f536..000000000 --- a/functions/function/image-recognition-function/src/test/java/org/springframework/cloud/fn/image/recognition/ImageRecognitionExample.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.image.recognition; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.List; - -import org.apache.commons.io.IOUtils; - -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; -import org.springframework.cloud.fn.common.tensorflow.deprecated.JsonMapperFunction; - -/** - * @author Christian Tzolov - */ -public final class ImageRecognitionExample { - - private ImageRecognitionExample() { - - } - - public static void main(String[] args) throws IOException { - - // You can use file:, http: or classpath: to provide the path to the input image. - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/giant_panda_in_beijing_zoo_1.jpg"); - - // MmobileNetV2 models - // https://github.com/tensorflow/models/tree/master/research/slim/nets/mobilenet#pretrained-models - String mobilenet_v2_modelUri = "https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.4_224.tgz#mobilenet_v2_1.4_224_frozen.pb"; - //String mobilenet_v2_modelUri = "https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_0.35_96.tgz#mobilenet_v2_0.35_96_frozen.pb"; - try (ImageRecognition imageRecognition = ImageRecognition.mobileNetV2( - mobilenet_v2_modelUri, - 224, - 5, - true)) { - - List recognizedObjects = - ImageRecognition.toRecognitionResponse(imageRecognition.recognizeTopK(inputImage)); - - // Draw the predicted labels on top of the input image. - byte[] augmentedImage = new ImageRecognitionAugmenter().apply(inputImage, recognizedObjects); - IOUtils.write(augmentedImage, new FileOutputStream("./image-recognition/target/image-augmented-mobilnetV2.jpg")); - - - String jsonRecognizedObjects = new JsonMapperFunction().apply(recognizedObjects); - System.out.println("mobilnetV2 result:" + jsonRecognizedObjects); - } - - - String mobilenet_v1_modelUri = "https://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224.tgz#mobilenet_v1_1.0_224_frozen.pb"; - try (ImageRecognition recognitionService = ImageRecognition.mobileNetV1( - mobilenet_v1_modelUri, - 224, - 5, - true)) { - - List recognizedObjects = - ImageRecognition.toRecognitionResponse(recognitionService.recognizeTopK(inputImage)); - - // Draw the predicted labels on top of the input image. - byte[] augmentedImage = new ImageRecognitionAugmenter().apply(inputImage, recognizedObjects); - IOUtils.write(augmentedImage, new FileOutputStream("./image-recognition/target/image-augmented-mobilnetV1.jpg")); - - - String jsonRecognizedObjects = new JsonMapperFunction().apply(recognizedObjects); - System.out.println("mobilnetV1 result:" + jsonRecognizedObjects); - } - - String inception_modelUri = "https://storage.googleapis.com/scdf-tensorflow-models/image-recognition/tensorflow_inception_graph.pb"; - try (ImageRecognition recognitionService = ImageRecognition.inception( - inception_modelUri, - 224, - 5, - true)) { - - List recognizedObjects = - ImageRecognition.toRecognitionResponse(recognitionService.recognizeTopK(inputImage)); - - // Draw the predicted labels on top of the input image. - byte[] augmentedImage = new ImageRecognitionAugmenter().apply(inputImage, recognizedObjects); - IOUtils.write(augmentedImage, new FileOutputStream("./image-recognition/target/image-augmented-inception.jpg")); - - - String jsonRecognizedObjects = new JsonMapperFunction().apply(recognizedObjects); - System.out.println("inception result:" + jsonRecognizedObjects); - } - } -} diff --git a/functions/function/image-recognition-function/src/test/java/org/springframework/cloud/fn/image/recognition/ImageRecognitionExample2.java b/functions/function/image-recognition-function/src/test/java/org/springframework/cloud/fn/image/recognition/ImageRecognitionExample2.java deleted file mode 100644 index 089927b3a..000000000 --- a/functions/function/image-recognition-function/src/test/java/org/springframework/cloud/fn/image/recognition/ImageRecognitionExample2.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.image.recognition; - -import java.io.FileOutputStream; -import java.io.IOException; - -import org.apache.commons.io.IOUtils; - -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; - -/** - * @author Christian Tzolov - */ -public final class ImageRecognitionExample2 { - - private ImageRecognitionExample2() { - - } - - public static void main(String[] args) throws IOException { - - ImageRecognitionAugmenter augmenter = new ImageRecognitionAugmenter(); - - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/giant_panda_in_beijing_zoo_1.jpg"); - - ImageRecognition inceptions = ImageRecognition.inception( - "https://storage.googleapis.com/scdf-tensorflow-models/image-recognition/tensorflow_inception_graph.pb", - 224, 10, true); - System.out.println(inceptions.recognizeMax(inputImage)); - System.out.println(inceptions.recognizeTopK(inputImage)); - System.out.println(ImageRecognition.toRecognitionResponse(inceptions.recognizeTopK(inputImage))); - - IOUtils.write(augmenter.apply(inputImage, ImageRecognition.toRecognitionResponse(inceptions.recognizeTopK(inputImage))), - new FileOutputStream("./functions/function/image-recognition-function/target/image-augmented-inceptions.jpg")); - inceptions.close(); - - ImageRecognition mobileNetV2 = ImageRecognition.mobileNetV2( - "https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.4_224.tgz#mobilenet_v2_1.4_224_frozen.pb", - 224, 10, true); - System.out.println(mobileNetV2.recognizeMax(inputImage)); - System.out.println(mobileNetV2.recognizeTopK(inputImage)); - IOUtils.write(augmenter.apply(inputImage, ImageRecognition.toRecognitionResponse(mobileNetV2.recognizeTopK(inputImage))), - new FileOutputStream("./functions/function/image-recognition-function/target/image-augmented-mobilnetV2.jpg")); - mobileNetV2.close(); - - ImageRecognition mobileNetV1 = ImageRecognition.mobileNetV1( - "https://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224.tgz#mobilenet_v1_1.0_224_frozen.pb", - 224, 10, true); - System.out.println(mobileNetV1.recognizeMax(inputImage)); - System.out.println(mobileNetV1.recognizeTopK(inputImage)); - IOUtils.write(augmenter.apply(inputImage, ImageRecognition.toRecognitionResponse(mobileNetV1.recognizeTopK(inputImage))), - new FileOutputStream("./functions/function/image-recognition-function/target/image-augmented-mobilnetV1.jpg")); - mobileNetV1.close(); - } -} diff --git a/functions/function/image-recognition-function/src/test/java/org/springframework/cloud/fn/image/recognition/SavedModelTest.java b/functions/function/image-recognition-function/src/test/java/org/springframework/cloud/fn/image/recognition/SavedModelTest.java deleted file mode 100644 index 8af8c42bd..000000000 --- a/functions/function/image-recognition-function/src/test/java/org/springframework/cloud/fn/image/recognition/SavedModelTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.image.recognition; - -import java.util.Map; - - -import com.google.protobuf.InvalidProtocolBufferException; -import org.tensorflow.SavedModelBundle; -import org.tensorflow.framework.MetaGraphDef; -import org.tensorflow.framework.SignatureDef; - -/** - * @author Christian Tzolov - */ -public final class SavedModelTest { - - private SavedModelTest() { - - } - - /** - * https://medium.com/@jsflo.dev/saving-and-loading-a-tensorflow-model-using-the-savedmodel-api-17645576527 - * - * https://www.tensorflow.org/alpha/guide/saved_model - * - */ - public static void main(String[] args) throws InvalidProtocolBufferException { - SavedModelBundle savedModelBundle = - SavedModelBundle.load("/Users/ctzolov/Downloads/ssd_mobilenet_v1_coco_2017_11_17/saved_model", "serve"); - //SavedModelBundle.load("/Users/ctzolov/Downloads/aiy_vision_classifier_plants_V1_1/", "serve"); - //SavedModelBundle savedModelBundle = - // SavedModelBundle.load("/Users/ctzolov/Downloads/mnasnet-a1/saved_model", "serve"); - - MetaGraphDef meta = MetaGraphDef.parseFrom(savedModelBundle.metaGraphDef()); - - Map signatures = meta.getSignatureDefMap(); - - System.out.println(signatures); - - savedModelBundle.session(); - - //Iterator itr = savedModelBundle.graph().operations(); - // - //while (itr.hasNext()) { - // System.out.println("Operation: " + itr.next()); - //} - } -} diff --git a/functions/function/image-recognition-function/src/test/resources/images/giant_panda_in_beijing_zoo_1.jpg b/functions/function/image-recognition-function/src/test/resources/images/giant_panda_in_beijing_zoo_1.jpg deleted file mode 100644 index e7a44cae2..000000000 Binary files a/functions/function/image-recognition-function/src/test/resources/images/giant_panda_in_beijing_zoo_1.jpg and /dev/null differ diff --git a/functions/function/image-recognition-function/src/test/resources/images/panda.jpeg b/functions/function/image-recognition-function/src/test/resources/images/panda.jpeg deleted file mode 100644 index f912b9440..000000000 Binary files a/functions/function/image-recognition-function/src/test/resources/images/panda.jpeg and /dev/null differ diff --git a/functions/function/object-detection-function/README.adoc b/functions/function/object-detection-function/README.adoc deleted file mode 100644 index 5df992ad0..000000000 --- a/functions/function/object-detection-function/README.adoc +++ /dev/null @@ -1,189 +0,0 @@ -:images-asciidoc: https://raw.githubusercontent.com/spring-cloud/stream-applications/master/functions/function/object-detection-function/src/main/resources/images/ - -# Object Detection Function - -Java model inference library for the https://github.com/tensorflow/models/blob/master/research/object_detection/README.md[TensorFlow Object Detection API]. Allows real-time localization and identification of multiple objects in a single or batch of images. Works with all https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md[pre-trained zoo models] and ttps://github.com/tensorflow/models/tree/865c14c/research/object_detection/data[object labels]. - -[cols="1,2", frame=none, grid=none] -|=== -| image:{images-asciidoc}/object_detection_1.jpg[alt=Object Detection 1, width=100%] -|The https://github.com/spring-cloud/stream-applications/blob/master/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionService.java[ObjectDetectionService] -takes an image or a batch of images and outputs a list of predicted objects bounding boxes -represented by https://github.com/spring-cloud/stream-applications/blob/master/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/domain/ObjectDetection.java[ObjectDetection]. -For the models supporting https://github.com/tensorflow/models/tree/master/research/object_detection#february-9-2018[Instance Segmentation], -the `ObjectDetectionService` can predict the instance segmentation `masks` in addition to object bounding boxes. - -The https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/JsonMapperFunction.java[JsonMapperFunction] permits -converting the `List` into JSON objects and the -https://github.com/spring-cloud/stream-applications/blob/master/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionImageAugmenter.java[ObjectDetectionImageAugmenter] -allow to augment the input image with the detected bounding boxes and segmentation masks. -|=== - -## Usage - -Add the `object-detection` dependency to the pom (use the latest version available): - -[source,xml] ----- - - org.springframework.cloud.fn - object-detection-function - ${revision} - ----- - -#### Example 1: Object Detection - -The https://github.com/spring-cloud/stream-applications/blob/master/functions/function/object-detection-function/src/test/java/org/springframework/cloud/fn/object/detection/examples/ExampleObjectDetection.java[ExampleObjectDetection.java] -sample demonstrates how to use the `ObjectDetectionService` for detecting objects in input images. It also shows how to -convert the result into JSON format and augment the input image with the detected object bounding boxes. - -[source,java,linenums] ----- -ObjectDetectionService detectionService = new ObjectDetectionService( - "https://download.tensorflow.org/models/object_detection/faster_rcnn_nas_coco_2018_01_28.tar.gz#frozen_inference_graph.pb", //<1> - "https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt", //<2> - 0.4f, //<3> - false, //<4> - true); //<5> - -byte[] image = GraphicsUtils.loadAsByteArray("classpath:/images/object-detection.jpg"); //<6> - -List detectedObjects = detectionService.detect(image); //<7> ----- -<1> Downloads and loads a pre-trained `frozen_inference_graph.pb` model directly from the `faster_rcnn_nas_coco.tar.gz` archive in the -Tensorflow model zoo. Mind that on first attempt it will download few hundreds of MBs. The consecutive runs will use the -cached copy (5) instead. -<2> Object category labels (e.g. names) for the model -<3> Confidence threshold - Only object with estimate above the threshold are returned -<4> Indicate that this is not a `mask` (e.g. not an instance segmentation) model type -<5> Cache the model on the local file system. -<6> Load the input image to evaluate -<7> Detect the objects in the image and represent the result as a list of ObjectDetection instances. - -Next you can convert the result in JSON format. - -[source,java,linenums] ----- -String jsonObjectDetections = new JsonMapperFunction().apply(detectedObjects); -System.out.println(jsonObjectDetections); ----- - -.Sample Object Detection JSON representation -[source,json] ----- -[{"name":"person","estimate":0.998,"x1":0.160,"y1":0.774,"x2":0.201,"y2":0.946,"cid":1}, - {"name":"kite","estimate":0.998,"x1":0.437,"y1":0.089,"x2":0.495,"y2":0.169,"cid":38}, - {"name":"person","estimate":0.997,"x1":0.084,"y1":0.681,"x2":0.121,"y2":0.848,"cid":1}, - {"name":"kite","estimate":0.988,"x1":0.206,"y1":0.263,"x2":0.225,"y2":0.314,"cid":38}]] ----- - -Use the https://github.com/spring-cloud/stream-applications/blob/master/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionImageAugmenter.java[ObjectDetectionImageAugmenter] -to draw the detected objects on top of the input image. - -[source,java,linenums] ----- -byte[] annotatedImage = new ObjectDetectionImageAugmenter().apply(image, detectedObjects); // <1> -IOUtils.write(annotatedImage, new FileOutputStream("./object-detection-function/target/object-detection-augmented.jpg")); //<2> ----- -<1> Augment the image with the detected object bounding boxes (Uses Java2D internally). -<2> Stores the augmented image as `object-detection-augmented.jpg` image file. - -.Augmented object-detection-augmented.jpg file -image:{images-asciidoc}/object-detection-augmented.jpg[alt=Object Detection, width=60%] - -TIP: Set the `ObjectDetectionImageAugmenter#agnosticColors` property to `true` to use a monochrome color schema. - -#### Example 2: Instance Segmentation - -The https://github.com/spring-cloud/stream-applications/blob/master/functions/function/object-detection-function/src/test/java/org/springframework/cloud/fn/object/detection/examples/ExampleInstanceSegmentation.java[ExampleInstanceSegmentation.java] -sample shows how to use the `ObjectDetectionService` for `Instance Segmentation`. -NOTE: It requires a trained model that supports `Masks` as well as setting the instance segmentation (e.g. `useMasks`) flag to `true`. - -[source,java,linenums] ----- -ObjectDetectionService detectionService = new ObjectDetectionService( - "https://download.tensorflow.org/models/object_detection/mask_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz#frozen_inference_graph.pb", // <1> - "https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt", // <2> - 0.4f, // <3> - true, // <4> - true); // <5> - -byte[] image = GraphicsUtils.loadAsByteArray("classpath:/images/object-detection.jpg"); - -List detectedObjects = detectionService.detect(image); // <6> - -String jsonObjectDetections = new JsonMapperFunction().apply(detectedObjects); // <7> -System.out.println(jsonObjectDetections); - -byte[] annotatedImage = new ObjectDetectionImageAugmenter(true) // <8> - .apply(image, detectedObjects); -IOUtils.write(annotatedImage, new FileOutputStream("./object-detection-function/target/object-detection-segmentation-augmented.jpg")); ----- -<1> Uses one of the 4 MASK pre-trained models -<2> Object category labels (e.g. names) for the model -<3> Confidence threshold - Only object with estimate above the threshold are returned. -<4> Use masks output - For the pre-trained models instruct to use the extended fetch names that include instance segmentation masks as well. -<5> Cache model - Create a local copy of the model to speed up consecutive runs. -<6> Evaluate the model to predict the object in the input image. -<7> Convert the detected object in to JSON array. NOTE: that with mask there is an additional field: `mask` -<8> Draw the detected object on top of the input image. Mind the `true` constructor parameter stands for draw detected masks. -If false only the bounding boxes will be shown. - -.Result augmented object-detection-segmentation-augmented.jpg file -image:{images-asciidoc}/object-detection-segmentation-augmented.jpg[alt=Object Detection Augmented, width=60%] - -## Models -All pre-trained https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md[detection_model_zoo.md] -models are supported. Following URI notation can be used to download any of the models directly from the zoo. - ----- -http://#frozen_inference_graph.pb ----- - -The `frozen_inference_graph.pb` is the frozen model file name within the archive. - -NOTE: For some models this name may differ. You have to download and open the archive to find the real name. - -TIP: To speedup the bootstrap performance you may consider extracting the `frozen_inference_graph.pb` and caching it -locally. Then you can use the `file://path-to-my-local-copy` URI schema to access it. - -Following models can be used for `Instance Segmentation` as well: - -[frame=none, grid=none] -|=== -| https://download.tensorflow.org/models/object_detection/mask_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz[mask_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz] -| https://download.tensorflow.org/models/object_detection/mask_rcnn_inception_v2_coco_2018_01_28.tar.gz[mask_rcnn_inception_v2_coco_2018_01_28.tar.gz] -| https://download.tensorflow.org/models/object_detection/mask_rcnn_resnet101_atrous_coco_2018_01_28.tar.gz[mask_rcnn_resnet101_atrous_coco_2018_01_28.tar.gz] -| https://download.tensorflow.org/models/object_detection/mask_rcnn_resnet50_atrous_coco_2018_01_28.tar.gz[mask_rcnn_resnet50_atrous_coco_2018_01_28.tar.gz] -|=== - -In addition to the model, the `ObjectDetectionService` requires a list of labels that correspond to the categories detectable by the selected model. -All labels files are available in the https://github.com/tensorflow/models/tree/master/research/object_detection/data[object_detection/data] folder. - -NOTE: It is important to use the labels that correspond to the model being used! Table below highlights this mapping. - -.Relationsip between trained model types and category labels -[%header, cols="1,2", frame=none, grid=none] -|=== -| Model -| Labels - -| https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md#coco-trained-models[coco] -| https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt[mscoco_label_map.pbtxt] - -| https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md#kitti-trained-models[kitti] -| https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/kitti_label_map.pbtxt[kitti_label_map.pbtxt] - -| https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md#open-images-trained-models[open-images] -| https://github.com/tensorflow/models/blob/master/research/object_detection/data/oid_bbox_trainable_label_map.pbtxt[oid_bbox_trainable_label_map.pbtxt] - -| https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md#inaturalist-species-trained-models[inaturalist-species] -| https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/fgvc_2854_classes_label_map.pbtxt[fgvc_2854_classes_label_map.pbtxt] - -| https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md#ava-v21-trained-models[ava] -| https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/ava_label_map_v2.1.pbtxt[ava_label_map_v2.1.pbtxt] - -|=== - -TIP: For performance reasons you may consider downloading the required label files to the local file system. diff --git a/functions/function/object-detection-function/pom.xml b/functions/function/object-detection-function/pom.xml deleted file mode 100644 index 8d71f5c77..000000000 --- a/functions/function/object-detection-function/pom.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - object-detection-function - object-detection-function - Spring Native Function for Tensorflow Integration - - 3.19.6 - - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - org.springframework.cloud.fn - tensorflow-common - ${project.version} - - - - - - - com.github.os72 - protoc-jar-maven-plugin - - 3.11.4 - - - generate-sources - - run - - - - - src/main/proto - - - - - java - src/main/java - - - com.google.protobuf:protoc:${protobuf.version} - - - - - - - - - - - - - - - - - diff --git a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionImageAugmenter.java b/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionImageAugmenter.java deleted file mode 100644 index 76fd15d3f..000000000 --- a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionImageAugmenter.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.object.detection; - -import java.awt.Color; -import java.awt.image.BufferedImage; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.util.List; -import java.util.function.BiFunction; - -import javax.imageio.ImageIO; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; -import org.springframework.cloud.fn.object.detection.domain.ObjectDetection; -import org.springframework.util.CollectionUtils; - -/** - * Augment the input image fromMemory detected object bounding boxes and categories. - * For mask models and withMask set to true it draws the instance segmentation image as well. - * - * @author Christian Tzolov - */ -public class ObjectDetectionImageAugmenter implements BiFunction, byte[]> { - - private static final Log logger = LogFactory.getLog(ObjectDetectionImageAugmenter.class); - - /** Make checkstyle happy. **/ - public static final String DEFAULT_IMAGE_FORMAT = "jpg"; - - private String imageFormat = DEFAULT_IMAGE_FORMAT; - - private final boolean withMask; - private boolean agnosticColors = false; - - public ObjectDetectionImageAugmenter() { - this(false); - } - - public ObjectDetectionImageAugmenter(boolean withMask) { - this.withMask = withMask; - } - - public boolean isAgnosticColors() { - return agnosticColors; - } - - public void setAgnosticColors(boolean agnosticColors) { - this.agnosticColors = agnosticColors; - } - - public String getImageFormat() { - return imageFormat; - } - - public void setImageFormat(String imageFormat) { - this.imageFormat = imageFormat; - } - - @Override - public byte[] apply(byte[] imageBytes, List objectDetections) { - - if (!CollectionUtils.isEmpty(objectDetections)) { - try { - BufferedImage bufferedImage = ImageIO.read(new ByteArrayInputStream(imageBytes)); - - for (ObjectDetection od : objectDetections) { - int y1 = (int) (od.getY1() * (float) bufferedImage.getHeight()); - int x1 = (int) (od.getX1() * (float) bufferedImage.getWidth()); - int y2 = (int) (od.getY2() * (float) bufferedImage.getHeight()); - int x2 = (int) (od.getX2() * (float) bufferedImage.getWidth()); - - int cid = od.getCid(); - - String labelName = od.getName(); - int probability = (int) (100 * od.getConfidence()); - String title = labelName + ": " + probability + "%"; - - GraphicsUtils.drawBoundingBox(bufferedImage, cid, title, x1, y1, x2, y2, this.agnosticColors); - - if (this.withMask && od.getMask() != null) { - float[][] mask = od.getMask(); - if (mask != null) { - Color maskColor = this.agnosticColors ? null : GraphicsUtils.getClassColor(cid); - BufferedImage maskImage = GraphicsUtils.createMaskImage( - mask, x2 - x1, y2 - y1, maskColor); - GraphicsUtils.overlayImages(bufferedImage, maskImage, x1, y1); - } - } - } - - imageBytes = GraphicsUtils.toImageByteArray(bufferedImage, this.getImageFormat()); - } - catch (IOException e) { - logger.error(e); - } - } - - // Null mend that QR image is found and not output message will be send. - return imageBytes; - } -} diff --git a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionInputAdapter.java b/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionInputAdapter.java deleted file mode 100644 index 1dfff4e40..000000000 --- a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionInputAdapter.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.object.detection; - -import java.util.Collections; -import java.util.Map; -import java.util.function.Function; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.tensorflow.Operand; -import org.tensorflow.Tensor; -import org.tensorflow.op.core.Placeholder; -import org.tensorflow.op.image.DecodeJpeg; -import org.tensorflow.types.UInt8; - -import org.springframework.cloud.fn.common.tensorflow.GraphRunner; - -/** - * Converts byte array image into a input Tensor for the Object Detection API. - * - * @author Christian Tzolov - */ -public class ObjectDetectionInputAdapter implements Function>>, AutoCloseable { - - private static final Log logger = LogFactory.getLog(ObjectDetectionInputAdapter.class); - - /** Make checkstyle happy. **/ - public static final String RAW_IMAGE = "raw_image"; - /** Make checkstyle happy. **/ - public static final String NORMALIZED_IMAGE = "normalized_image"; - /** Make checkstyle happy. **/ - public static final long CHANNELS = 3; - - private final GraphRunner imageLoaderGraph; - - public ObjectDetectionInputAdapter() { - - this.imageLoaderGraph = new GraphRunner(RAW_IMAGE, NORMALIZED_IMAGE) - .withGraphDefinition(tf -> { - Placeholder rawImage = tf.withName(RAW_IMAGE).placeholder(String.class); - Operand decodedImage = tf.dtypes.cast( - tf.image.decodeJpeg(rawImage, DecodeJpeg.channels(CHANNELS)), UInt8.class); - // Expand dimensions since the model expects images to have shape: [1, H, W, 3] - tf.withName(NORMALIZED_IMAGE).expandDims(decodedImage, tf.constant(0)); - }); - } - - @Override - public Map> apply(byte[] inputImage) { - try (Tensor inputTensor = Tensor.create(inputImage)) { - return this.imageLoaderGraph.apply(Collections.singletonMap(RAW_IMAGE, inputTensor)); - } - } - - @Override - public void close() { - if (this.imageLoaderGraph != null) { - this.imageLoaderGraph.close(); - } - } -} diff --git a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionInputConverter.java b/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionInputConverter.java deleted file mode 100644 index 7f55e5939..000000000 --- a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionInputConverter.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.object.detection; - -import java.awt.image.BufferedImage; -import java.awt.image.DataBufferByte; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.Map; -import java.util.function.Function; - -import javax.imageio.ImageIO; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.tensorflow.Tensor; -import org.tensorflow.types.UInt8; - -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; - -/** - * Converts byte array image into a input Tensor for the Object Detection API. The computed image tensors uses the - * 'image_tensor' model placeholder. - * - * @author Christian Tzolov - */ -public class ObjectDetectionInputConverter implements Function>> { - - private static final Log logger = LogFactory.getLog(ObjectDetectionInputConverter.class); - - private static final long CHANNELS = 3; - - /** Make checkstyle happy. **/ - public static final String IMAGE_TENSOR_FEED_NAME = "image_tensor"; - - @Override - public Map> apply(byte[][] input) { - return Collections.singletonMap(IMAGE_TENSOR_FEED_NAME, makeImageTensor(input)); - } - - private static Tensor makeImageTensor(byte[][] imageBytesArray) { - try { - int batchSize = imageBytesArray.length; - ByteBuffer byteBuffer = null; - long[] shape = null; - for (int batchIndex = 0; batchIndex < batchSize; batchIndex++) { - byte[] imageBytes = imageBytesArray[batchIndex]; - ByteArrayInputStream is = new ByteArrayInputStream(imageBytes); - BufferedImage img = ImageIO.read(is); - - if (img.getType() != BufferedImage.TYPE_3BYTE_BGR) { - img = GraphicsUtils.toBufferedImageType(img, BufferedImage.TYPE_3BYTE_BGR); - } - - if (byteBuffer == null) { - byteBuffer = ByteBuffer.allocate((int) (batchSize * img.getHeight() * img.getWidth() * CHANNELS)); - shape = new long[] { batchSize, img.getHeight(), img.getWidth(), CHANNELS }; - } - - byte[] data = ((DataBufferByte) img.getData().getDataBuffer()).getData(); - // ImageIO.read produces BGR-encoded images, while the model expects RGB. - bgrToRgb(data); - byteBuffer.put(data); - } - byteBuffer.flip(); - - return Tensor.create(UInt8.class, shape, byteBuffer); - } - catch (IOException e) { - throw new IllegalArgumentException("Incorrect image format", e); - } - - } - - private static void bgrToRgb(byte[] data) { - for (int i = 0; i < data.length; i += 3) { - byte tmp = data[i]; - data[i] = data[i + 2]; - data[i + 2] = tmp; - } - } -} diff --git a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionOutputConverter.java b/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionOutputConverter.java deleted file mode 100644 index f913c6ad9..000000000 --- a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionOutputConverter.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.object.detection; - -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -import com.google.protobuf.TextFormat; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.tensorflow.Tensor; - -import org.springframework.cloud.fn.object.detection.domain.ObjectDetection; -import org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass; -import org.springframework.core.io.Resource; -import org.springframework.util.Assert; -import org.springframework.util.StreamUtils; -import org.springframework.util.StringUtils; - -/** - * Converts the Tensorflow Object Detection result into {@link ObjectDetection} list. - * The pre-trained Object Detection models (http://bit.ly/2osxMAY) produce 3 tensor outputs: - * (1) detection_classes - containing the ids of detected objects, (2) detection_scores - confidence probabilities of the - * detected object and (3) detection_boxes - the object bounding boxes withing the images. - * - * The MASK based models provide to 2 additional tensors: (4) num_detections and (5) detection_masks. - * - * All outputs tensors are float arrays, having: - * - 1 as the first dimension - * - maxObjects as the second dimension - * While boxesT will have 4 as the third dimension (2 sets of (x, y) coordinates). - * This can be verified by looking at scoresT.shape() etc. - * - * The format detected classes (e.g. labels) names is defined by the 'string_int_labels_map.proto'. The input list - * is available at: https://github.com/tensorflow/models/tree/master/research/object_detection/data - * - * @author Christian Tzolov - */ -public class ObjectDetectionOutputConverter implements Function>, List>> { - - private static final Log logger = LogFactory.getLog(ObjectDetectionOutputConverter.class); - - /** DETECTION_CLASSES. */ - public static final String DETECTION_CLASSES = "detection_classes"; - /** DETECTION_SCORES. */ - public static final String DETECTION_SCORES = "detection_scores"; - /** DETECTION_BOXES. */ - public static final String DETECTION_BOXES = "detection_boxes"; - /** DETECTION_MASKS. */ - public static final String DETECTION_MASKS = "detection_masks"; - /** NUM_DETECTIONS. */ - public static final String NUM_DETECTIONS = "num_detections"; - - private final String[] labels; - private float confidence; - private List modelFetch; - - public ObjectDetectionOutputConverter(Resource labelsResource, float confidence, List modelFetch) { - this.confidence = confidence; - this.modelFetch = modelFetch; - try { - this.labels = loadLabels(labelsResource); - Assert.notNull(this.labels, String.format("Failed to initialize object labels [%s].", labelsResource)); - } - catch (Exception e) { - throw new RuntimeException(String.format("Failed to initialize object labels [%s].", labelsResource), e); - } - - logger.info(String.format("Object labels [%s] loaded.", labelsResource)); - } - - /** - * Loads object labels in the string_int_label_map.proto. - * @param labelsResource location of the labels as a resource - * @return String[] of labels - */ - private static String[] loadLabels(Resource labelsResource) throws Exception { - try (InputStream is = labelsResource.getInputStream()) { - String text = StreamUtils.copyToString(is, Charset.forName("UTF-8")); - StringIntLabelMapOuterClass.StringIntLabelMap.Builder builder = - StringIntLabelMapOuterClass.StringIntLabelMap.newBuilder(); - TextFormat.merge(text, builder); - StringIntLabelMapOuterClass.StringIntLabelMap proto = builder.build(); - - int maxLabelId = proto.getItemList().stream() - .map(StringIntLabelMapOuterClass.StringIntLabelMapItem::getId) - .max(Comparator.comparing(i -> i)) - .orElse(-1); - - String[] labelIdToNameMap = new String[maxLabelId + 1]; - for (StringIntLabelMapOuterClass.StringIntLabelMapItem item : proto.getItemList()) { - if (!StringUtils.isEmpty(item.getDisplayName())) { - labelIdToNameMap[item.getId()] = item.getDisplayName(); - } - else { - // Common practice is to set the name to a MID or Synsets Id. Synset is a set of synonyms that - // share a common meaning: https://en.wikipedia.org/wiki/WordNet - labelIdToNameMap[item.getId()] = item.getName(); - } - } - return labelIdToNameMap; - } - } - - @Override - public List> apply(Map> tensorMap) { - - try (Tensor scoresTensor = tensorMap.get(DETECTION_SCORES).expect(Float.class); - Tensor classesTensor = tensorMap.get(DETECTION_CLASSES).expect(Float.class); - Tensor boxesTensor = tensorMap.get(DETECTION_BOXES).expect(Float.class) - ) { - // All these tensors have: - // - 1 as the first dimension - // - maxObjects as the second dimension - // While boxesT will have 4 as the third dimension (2 sets of (x, y) coordinates). - // This can be verified by looking at scoresT.shape() etc. - int batchSize = (int) scoresTensor.shape()[0]; - int maxObjects = (int) scoresTensor.shape()[1]; - float[][] scores = scoresTensor.copyTo(new float[batchSize][maxObjects]); - float[][] classes = classesTensor.copyTo(new float[batchSize][maxObjects]); - float[][][] boxes = boxesTensor.copyTo(new float[batchSize][maxObjects][4]); - - List> batchObjectDetections = new ArrayList<>(); - - for (int batchIndex = 0; batchIndex < batchSize; batchIndex++) { - - - List objectDetections = new ArrayList<>(); - - // Collect only the objects whose scores are at above the configured confidence threshold. - for (int i = 0; i < scores[batchIndex].length; ++i) { - if (scores[batchIndex][i] >= confidence) { - String category = labels[(int) classes[batchIndex][i]]; - float score = scores[batchIndex][i]; - - ObjectDetection od = new ObjectDetection(); - od.setName(category); - od.setConfidence(score); - od.setX1(boxes[batchIndex][i][1]); - od.setY1(boxes[batchIndex][i][0]); - od.setX2(boxes[batchIndex][i][3]); - od.setY2(boxes[batchIndex][i][2]); - od.setCid((int) classes[batchIndex][i]); - - // Mask allows image-segmentation - if (modelFetch.contains(DETECTION_MASKS) && modelFetch.contains(NUM_DETECTIONS)) { - Tensor masksTensor = tensorMap.get(DETECTION_MASKS).expect(Float.class); - Tensor numDetections = tensorMap.get(NUM_DETECTIONS).expect(Float.class); - float nd = numDetections.copyTo(new float[batchSize])[0]; - - if (masksTensor != null) { - long[] shape = masksTensor.shape(); - float[][][][] masks = masksTensor.copyTo(new float[(int) shape[0]][(int) shape[1]][(int) shape[2]][(int) shape[3]]); - od.setMask(masks[batchIndex][i]); - logger.debug(String.format("Num detections: %s, Masks: %s", nd, masks)); - } - } - - objectDetections.add(od); - } - } - - batchObjectDetections.add(objectDetections); - } - return batchObjectDetections; - } - } -} diff --git a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionService.java b/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionService.java deleted file mode 100644 index 2b3fa10b6..000000000 --- a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionService.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.object.detection; - -import java.awt.image.BufferedImage; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.List; - -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; -import org.springframework.cloud.fn.common.tensorflow.deprecated.TensorFlowService; -import org.springframework.cloud.fn.object.detection.domain.ObjectDetection; -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.util.StreamUtils; - -/** - * Convenience class that leverages the the {@link ObjectDetectionInputConverter}, {@link ObjectDetectionOutputConverter} and {@link TensorFlowService} - * in combination fromMemory the Tensorflow Object Detection API (https://github.com/tensorflow/models/tree/master/research/object_detection) - * models for detection objects in input images. - * - * All pre-trained models (https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md) and labels are supported. - * - * You can download pre-trained models directly from the zoo: https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md - * Just use the URI notation: (zoo model tar.gz url)#(name of the frozen model file name). To speedup the bootstrap - * performance you should consider downloading the models locally and use the file:/"path to my model" URI instead! - * - * The object category labels for the pre-trained models are available at: https://github.com/tensorflow/models/tree/master/research/object_detection/data - * Use the labels applicable for the model. Also, for performance reasons you may consider to download the labels - * and load them from file: instead. - * - * @author Christian Tzolov - */ -public class ObjectDetectionService { - - /** Default list of fetch names for Box models. */ - public static List FETCH_NAMES = Arrays.asList( - ObjectDetectionOutputConverter.DETECTION_SCORES, ObjectDetectionOutputConverter.DETECTION_CLASSES, - ObjectDetectionOutputConverter.DETECTION_BOXES, ObjectDetectionOutputConverter.NUM_DETECTIONS); - - /** Default list of fetch names for mask supporting models. */ - public static List FETCH_NAMES_WITH_MASKS = Arrays.asList( - ObjectDetectionOutputConverter.DETECTION_SCORES, ObjectDetectionOutputConverter.DETECTION_CLASSES, - ObjectDetectionOutputConverter.DETECTION_BOXES, ObjectDetectionOutputConverter.DETECTION_MASKS, - ObjectDetectionOutputConverter.NUM_DETECTIONS); - - private final ObjectDetectionInputConverter inputConverter; - private final ObjectDetectionOutputConverter outputConverter; - private final TensorFlowService tensorFlowService; - - public ObjectDetectionService() { - this("https://download.tensorflow.org/models/object_detection/ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz#frozen_inference_graph.pb", - "https://storage.googleapis.com/scdf-tensorflow-models/object-detection/mscoco_label_map.pbtxt", - 0.4f, false, true); - } - - /** - * Convenience constructor that would initialize all necessary internal components. - * @param modelUri URI of the pre-trained, frozen Tensorflow model. - * @param labelsUri URI of the pre-trained category labels. - * @param confidence Confidence threshold. Only objects detected wth confidence above this threshold will be returned. - * @param withMasks If a Mask model is selected then you can use this flag to extract the instance segmentation masks as well. - */ - public ObjectDetectionService(String modelUri, String labelsUri, - float confidence, boolean withMasks, boolean cacheModel) { - this.inputConverter = new ObjectDetectionInputConverter(); - List fetchNames = withMasks ? FETCH_NAMES_WITH_MASKS : FETCH_NAMES; - this.outputConverter = new ObjectDetectionOutputConverter( - new DefaultResourceLoader().getResource(labelsUri), confidence, fetchNames); - this.tensorFlowService = new TensorFlowService( - new DefaultResourceLoader().getResource(modelUri), fetchNames, cacheModel); - } - - /** - * Generic constructor thea allow the converter to be pre-configured before passed to the service. - * @param inputConverter Converter from byte array to object detection input image tensor - * @param outputConverter Covets the object detection output tensors into {@link ObjectDetection } list - * @param tensorFlowService Java tensorflow runner instance - */ - public ObjectDetectionService(ObjectDetectionInputConverter inputConverter, - ObjectDetectionOutputConverter outputConverter, TensorFlowService tensorFlowService) { - this.inputConverter = inputConverter; - this.outputConverter = outputConverter; - this.tensorFlowService = tensorFlowService; - } - - /** - * Detects objects in a single input image identified by its URI. - * - * @param imageUri input image's URI - * @return Returns a list of {@link ObjectDetection} domain objects representing detected objects - */ - public List detect(String imageUri) { - try (InputStream is = new DefaultResourceLoader().getResource(imageUri).getInputStream()) { - return this.detect(StreamUtils.copyToByteArray(is)); - } - catch (IOException e) { - e.printStackTrace(); - throw new IllegalStateException("Failed to detect the image:" + imageUri, e); - } - } - - /** - * Detects objects in a single {@link BufferedImage}. - * - * @param image Input image to detect objects from. - * @param format Image format (e.g. jpg, png ...) to use when converting the buffer into byte array. - * @return Returns a list of {@link ObjectDetection} domain objects representing detected objects in the input image - */ - public List detect(BufferedImage image, String format) { - return this.detect(GraphicsUtils.toImageByteArray(image, format)); - } - - /** - * Detects objects from a single input image encoded as byte array. - * - * @param image Input image encoded as byte array - * @return Returns a list of {@link ObjectDetection} domain objects representing detected objects in the input image - */ - public List detect(byte[] image) { - return this.inputConverter.andThen(this.tensorFlowService).andThen(this.outputConverter).apply(new byte[][] { image }).get(0); - } - - /** - * Uses detects objects from a batch of input images encoded as byte array. - * - * @param batchedImages Batch of input images encoded as byte arrays. First dimension is the batch size and second the image bytes. - * @return Returns list of lists. For every input image in the batch a list of {@link ObjectDetection} domain objects representing detected objects in the input image. - */ - public List> detect(byte[][] batchedImages) { - return this.inputConverter.andThen(this.tensorFlowService).andThen(this.outputConverter).apply(batchedImages); - } -} diff --git a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionService2.java b/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionService2.java deleted file mode 100644 index 09c3cc921..000000000 --- a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionService2.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.object.detection; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import org.tensorflow.Operand; -import org.tensorflow.Tensor; -import org.tensorflow.op.core.Placeholder; -import org.tensorflow.op.image.DecodeJpeg; -import org.tensorflow.types.UInt8; - -import org.springframework.cloud.fn.common.tensorflow.GraphRunner; -import org.springframework.cloud.fn.common.tensorflow.GraphRunnerMemory; -import org.springframework.cloud.fn.common.tensorflow.ProtoBufGraphDefinition; -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; -import org.springframework.cloud.fn.object.detection.domain.ObjectDetection; -import org.springframework.core.io.DefaultResourceLoader; - -/** - * @author Christian Tzolov - */ -public class ObjectDetectionService2 implements AutoCloseable { - - /** Default Box models fetch names. */ - public static List FETCH_NAMES = Arrays.asList( - ObjectDetectionOutputConverter.DETECTION_SCORES, ObjectDetectionOutputConverter.DETECTION_CLASSES, - ObjectDetectionOutputConverter.DETECTION_BOXES, ObjectDetectionOutputConverter.NUM_DETECTIONS); - - /** Default Models models fetch names. */ - public static List FETCH_NAMES_WITH_MASKS = Arrays.asList( - ObjectDetectionOutputConverter.DETECTION_SCORES, ObjectDetectionOutputConverter.DETECTION_CLASSES, - ObjectDetectionOutputConverter.DETECTION_BOXES, ObjectDetectionOutputConverter.DETECTION_MASKS, - ObjectDetectionOutputConverter.NUM_DETECTIONS); - - private final GraphRunner imageNormalization; - private final GraphRunner objectDetection; - private final ObjectDetectionOutputConverter outputConverter; - - - public ObjectDetectionService2(String modelUri, ObjectDetectionOutputConverter outputConverter) { - - this.imageNormalization = new GraphRunner("raw_image", "normalized_image") - .withGraphDefinition(tf -> { - Placeholder rawImage = tf.withName("raw_image").placeholder(String.class); - Operand decodedImage = tf.dtypes.cast( - tf.image.decodeJpeg(rawImage, DecodeJpeg.channels(3L)), UInt8.class); - // Expand dimensions since the model expects images to have shape: [1, H, W, 3] - tf.withName("normalized_image").expandDims(decodedImage, tf.constant(0)); - }); - - this.objectDetection = new GraphRunner(Arrays.asList("image_tensor"), FETCH_NAMES) - .withGraphDefinition(new ProtoBufGraphDefinition( - new DefaultResourceLoader().getResource(modelUri), true)); - - this.outputConverter = outputConverter; - } - - public List detect(byte[] image) { - try (Tensor inputTensor = Tensor.create(image); GraphRunnerMemory memorize = new GraphRunnerMemory()) { - - List> out = this.imageNormalization.andThen(memorize) - .andThen(this.objectDetection).andThen(memorize) - .andThen(outputConverter) - .apply(Collections.singletonMap("raw_image", inputTensor)); - - return out.get(0); - - } - } - - @Override - public void close() { - this.imageNormalization.close(); - this.objectDetection.close(); - //this.outputConverter.close(); - } - - public static void main(String[] args) throws IOException { - String modelUri = "https://dl.bintray.com/big-data/generic/ssdlite_mobilenet_v2_coco_2018_05_09_frozen_inference_graph.pb"; - String labelUri = "https://dl.bintray.com/big-data/generic/mscoco_label_map.pbtxt"; - - ObjectDetectionOutputConverter outputAdapter = new ObjectDetectionOutputConverter( - new DefaultResourceLoader().getResource(labelUri), 0.4f, FETCH_NAMES); - - //byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/object-detection.jpg"); - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/wild-animals-15.jpg"); - - try (ObjectDetectionService2 objectDetectionService2 = new ObjectDetectionService2(modelUri, outputAdapter)) { - - List boza = objectDetectionService2.detect(inputImage); - - System.out.println(boza); - } - } -} diff --git a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/domain/ObjectDetection.java b/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/domain/ObjectDetection.java deleted file mode 100644 index 15e107586..000000000 --- a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/domain/ObjectDetection.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.object.detection.domain; - -import java.util.Arrays; - -import com.fasterxml.jackson.annotation.JsonInclude; - -/** - * @author Christian Tzolov - */ - -@JsonInclude(JsonInclude.Include.NON_NULL) -public class ObjectDetection { - - private String name; - private float confidence; - private float x1; - private float y1; - private float x2; - private float y2; - private float[][] mask; - private int cid; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public float getConfidence() { - return confidence; - } - - public void setConfidence(float confidence) { - this.confidence = confidence; - } - - public float getX1() { - return x1; - } - - public void setX1(float x1) { - this.x1 = x1; - } - - public float getY1() { - return y1; - } - - public void setY1(float y1) { - this.y1 = y1; - } - - public float getX2() { - return x2; - } - - public void setX2(float x2) { - this.x2 = x2; - } - - public float getY2() { - return y2; - } - - public void setY2(float y2) { - this.y2 = y2; - } - - public int getCid() { - return cid; - } - - public void setCid(int cid) { - this.cid = cid; - } - - public float[][] getMask() { - return mask; - } - - public void setMask(float[][] mask) { - this.mask = mask; - } - - @Override - public String toString() { - return "ObjectDetection{" + - "name='" + name + '\'' + - ", confidence=" + confidence + - ", x1=" + x1 + - ", y1=" + y1 + - ", x2=" + x2 + - ", y2=" + y2 + - ", mask=" + Arrays.toString(mask) + - ", cid=" + cid + - '}'; - } -} diff --git a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/protos/StringIntLabelMapOuterClass.java b/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/protos/StringIntLabelMapOuterClass.java deleted file mode 100644 index ab7fe9a91..000000000 --- a/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/protos/StringIntLabelMapOuterClass.java +++ /dev/null @@ -1,1864 +0,0 @@ -// Generated by the protocol buffer compiler. DO NOT EDIT! -// source: string_int_label_map.proto - -package org.springframework.cloud.fn.object.detection.protos; - -public final class StringIntLabelMapOuterClass { - private StringIntLabelMapOuterClass() {} - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistryLite registry) { - } - - public static void registerAllExtensions( - com.google.protobuf.ExtensionRegistry registry) { - registerAllExtensions( - (com.google.protobuf.ExtensionRegistryLite) registry); - } - public interface StringIntLabelMapItemOrBuilder extends - // @@protoc_insertion_point(interface_extends:org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem) - com.google.protobuf.MessageOrBuilder { - - /** - * 

-     * String name. The most common practice is to set this to a MID or synsets
-     * id. Synset: a set of synonyms that share a common meaning.
-     * https://en.wikipedia.org/wiki/WordNet
-     *
- * - * optional string name = 1; - * @return Whether the name field is set. - */ - boolean hasName(); - /** - * 
-     * String name. The most common practice is to set this to a MID or synsets
-     * id. Synset: a set of synonyms that share a common meaning.
-     * https://en.wikipedia.org/wiki/WordNet
-     *
- * - * optional string name = 1; - * @return The name. - */ - java.lang.String getName(); - /** - * 
-     * String name. The most common practice is to set this to a MID or synsets
-     * id. Synset: a set of synonyms that share a common meaning.
-     * https://en.wikipedia.org/wiki/WordNet
-     *
- * - * optional string name = 1; - * @return The bytes for name. - */ - com.google.protobuf.ByteString - getNameBytes(); - - /** - * 
-     * Integer id that maps to the string name above. Label ids should start
-     * from 1.
-     *
- * - * optional int32 id = 2; - * @return Whether the id field is set. - */ - boolean hasId(); - /** - * 
-     * Integer id that maps to the string name above. Label ids should start
-     * from 1.
-     *
- * - * optional int32 id = 2; - * @return The id. - */ - int getId(); - - /** - * 
-     * Human readable string label.
-     *
- * - * optional string display_name = 3; - * @return Whether the displayName field is set. - */ - boolean hasDisplayName(); - /** - * 
-     * Human readable string label.
-     *
- * - * optional string display_name = 3; - * @return The displayName. - */ - java.lang.String getDisplayName(); - /** - * 
-     * Human readable string label.
-     *
- * - * optional string display_name = 3; - * @return The bytes for displayName. - */ - com.google.protobuf.ByteString - getDisplayNameBytes(); - } - /** - * Protobuf type {@code org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem} - */ - public static final class StringIntLabelMapItem extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem) - StringIntLabelMapItemOrBuilder { - private static final long serialVersionUID = 0L; - // Use StringIntLabelMapItem.newBuilder() to construct. - private StringIntLabelMapItem(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - private StringIntLabelMapItem() { - name_ = ""; - displayName_ = ""; - } - - @java.lang.Override - @SuppressWarnings({"unused"}) - protected java.lang.Object newInstance( - UnusedPrivateParameter unused) { - return new StringIntLabelMapItem(); - } - - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMapItem_descriptor; - } - - @java.lang.Override - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMapItem_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.class, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.Builder.class); - } - - private int bitField0_; - public static final int NAME_FIELD_NUMBER = 1; - private volatile java.lang.Object name_; - /** - * 
-     * String name. The most common practice is to set this to a MID or synsets
-     * id. Synset: a set of synonyms that share a common meaning.
-     * https://en.wikipedia.org/wiki/WordNet
-     *
- * - * optional string name = 1; - * @return Whether the name field is set. - */ - @java.lang.Override - public boolean hasName() { - return ((bitField0_ & 0x00000001) != 0); - } - /** - * 
-     * String name. The most common practice is to set this to a MID or synsets
-     * id. Synset: a set of synonyms that share a common meaning.
-     * https://en.wikipedia.org/wiki/WordNet
-     *
- * - * optional string name = 1; - * @return The name. - */ - @java.lang.Override - public java.lang.String getName() { - java.lang.Object ref = name_; - if (ref instanceof java.lang.String) { - return (java.lang.String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - if (bs.isValidUtf8()) { - name_ = s; - } - return s; - } - } - /** - * 
-     * String name. The most common practice is to set this to a MID or synsets
-     * id. Synset: a set of synonyms that share a common meaning.
-     * https://en.wikipedia.org/wiki/WordNet
-     *
- * - * optional string name = 1; - * @return The bytes for name. - */ - @java.lang.Override - public com.google.protobuf.ByteString - getNameBytes() { - java.lang.Object ref = name_; - if (ref instanceof java.lang.String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - name_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - public static final int ID_FIELD_NUMBER = 2; - private int id_; - /** - * 
-     * Integer id that maps to the string name above. Label ids should start
-     * from 1.
-     *
- * - * optional int32 id = 2; - * @return Whether the id field is set. - */ - @java.lang.Override - public boolean hasId() { - return ((bitField0_ & 0x00000002) != 0); - } - /** - * 
-     * Integer id that maps to the string name above. Label ids should start
-     * from 1.
-     *
- * - * optional int32 id = 2; - * @return The id. - */ - @java.lang.Override - public int getId() { - return id_; - } - - public static final int DISPLAY_NAME_FIELD_NUMBER = 3; - private volatile java.lang.Object displayName_; - /** - * 
-     * Human readable string label.
-     *
- * - * optional string display_name = 3; - * @return Whether the displayName field is set. - */ - @java.lang.Override - public boolean hasDisplayName() { - return ((bitField0_ & 0x00000004) != 0); - } - /** - * 
-     * Human readable string label.
-     *
- * - * optional string display_name = 3; - * @return The displayName. - */ - @java.lang.Override - public java.lang.String getDisplayName() { - java.lang.Object ref = displayName_; - if (ref instanceof java.lang.String) { - return (java.lang.String) ref; - } else { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - if (bs.isValidUtf8()) { - displayName_ = s; - } - return s; - } - } - /** - * 
-     * Human readable string label.
-     *
- * - * optional string display_name = 3; - * @return The bytes for displayName. - */ - @java.lang.Override - public com.google.protobuf.ByteString - getDisplayNameBytes() { - java.lang.Object ref = displayName_; - if (ref instanceof java.lang.String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - displayName_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - - private byte memoizedIsInitialized = -1; - @java.lang.Override - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized == 1) return true; - if (isInitialized == 0) return false; - - memoizedIsInitialized = 1; - return true; - } - - @java.lang.Override - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - if (((bitField0_ & 0x00000001) != 0)) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 1, name_); - } - if (((bitField0_ & 0x00000002) != 0)) { - output.writeInt32(2, id_); - } - if (((bitField0_ & 0x00000004) != 0)) { - com.google.protobuf.GeneratedMessageV3.writeString(output, 3, displayName_); - } - getUnknownFields().writeTo(output); - } - - @java.lang.Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) return size; - - size = 0; - if (((bitField0_ & 0x00000001) != 0)) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, name_); - } - if (((bitField0_ & 0x00000002) != 0)) { - size += com.google.protobuf.CodedOutputStream - .computeInt32Size(2, id_); - } - if (((bitField0_ & 0x00000004) != 0)) { - size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, displayName_); - } - size += getUnknownFields().getSerializedSize(); - memoizedSize = size; - return size; - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem)) { - return super.equals(obj); - } - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem other = (org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem) obj; - - if (hasName() != other.hasName()) return false; - if (hasName()) { - if (!getName() - .equals(other.getName())) return false; - } - if (hasId() != other.hasId()) return false; - if (hasId()) { - if (getId() - != other.getId()) return false; - } - if (hasDisplayName() != other.hasDisplayName()) return false; - if (hasDisplayName()) { - if (!getDisplayName() - .equals(other.getDisplayName())) return false; - } - if (!getUnknownFields().equals(other.getUnknownFields())) return false; - return true; - } - - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - if (hasName()) { - hash = (37 * hash) + NAME_FIELD_NUMBER; - hash = (53 * hash) + getName().hashCode(); - } - if (hasId()) { - hash = (37 * hash) + ID_FIELD_NUMBER; - hash = (53 * hash) + getId(); - } - if (hasDisplayName()) { - hash = (37 * hash) + DISPLAY_NAME_FIELD_NUMBER; - hash = (53 * hash) + getDisplayName().hashCode(); - } - hash = (29 * hash) + getUnknownFields().hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @java.lang.Override - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - public static Builder newBuilder(org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - @java.lang.Override - public Builder toBuilder() { - return this == DEFAULT_INSTANCE - ? new Builder() : new Builder().mergeFrom(this); - } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - // @@protoc_insertion_point(builder_implements:org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem) - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItemOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMapItem_descriptor; - } - - @java.lang.Override - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMapItem_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.class, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.Builder.class); - } - - // Construct using org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.newBuilder() - private Builder() { - - } - - private Builder( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - super(parent); - - } - @java.lang.Override - public Builder clear() { - super.clear(); - name_ = ""; - bitField0_ = (bitField0_ & ~0x00000001); - id_ = 0; - bitField0_ = (bitField0_ & ~0x00000002); - displayName_ = ""; - bitField0_ = (bitField0_ & ~0x00000004); - return this; - } - - @java.lang.Override - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMapItem_descriptor; - } - - @java.lang.Override - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem getDefaultInstanceForType() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.getDefaultInstance(); - } - - @java.lang.Override - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem build() { - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @java.lang.Override - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem buildPartial() { - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem result = new org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem(this); - int from_bitField0_ = bitField0_; - int to_bitField0_ = 0; - if (((from_bitField0_ & 0x00000001) != 0)) { - to_bitField0_ |= 0x00000001; - } - result.name_ = name_; - if (((from_bitField0_ & 0x00000002) != 0)) { - result.id_ = id_; - to_bitField0_ |= 0x00000002; - } - if (((from_bitField0_ & 0x00000004) != 0)) { - to_bitField0_ |= 0x00000004; - } - result.displayName_ = displayName_; - result.bitField0_ = to_bitField0_; - onBuilt(); - return result; - } - - @java.lang.Override - public Builder clone() { - return super.clone(); - } - @java.lang.Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return super.setField(field, value); - } - @java.lang.Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - @java.lang.Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - @java.lang.Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, java.lang.Object value) { - return super.setRepeatedField(field, index, value); - } - @java.lang.Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return super.addRepeatedField(field, value); - } - @java.lang.Override - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem) { - return mergeFrom((org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem other) { - if (other == org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.getDefaultInstance()) return this; - if (other.hasName()) { - bitField0_ |= 0x00000001; - name_ = other.name_; - onChanged(); - } - if (other.hasId()) { - setId(other.getId()); - } - if (other.hasDisplayName()) { - bitField0_ |= 0x00000004; - displayName_ = other.displayName_; - onChanged(); - } - this.mergeUnknownFields(other.getUnknownFields()); - onChanged(); - return this; - } - - @java.lang.Override - public final boolean isInitialized() { - return true; - } - - @java.lang.Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - if (extensionRegistry == null) { - throw new java.lang.NullPointerException(); - } - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - case 10: { - name_ = input.readBytes(); - bitField0_ |= 0x00000001; - break; - } // case 10 - case 16: { - id_ = input.readInt32(); - bitField0_ |= 0x00000002; - break; - } // case 16 - case 26: { - displayName_ = input.readBytes(); - bitField0_ |= 0x00000004; - break; - } // case 26 - default: { - if (!super.parseUnknownField(input, extensionRegistry, tag)) { - done = true; // was an endgroup tag - } - break; - } // default: - } // switch (tag) - } // while (!done) - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.unwrapIOException(); - } finally { - onChanged(); - } // finally - return this; - } - private int bitField0_; - - private java.lang.Object name_ = ""; - /** - * 
-       * String name. The most common practice is to set this to a MID or synsets
-       * id. Synset: a set of synonyms that share a common meaning.
-       * https://en.wikipedia.org/wiki/WordNet
-       *
- * - * optional string name = 1; - * @return Whether the name field is set. - */ - public boolean hasName() { - return ((bitField0_ & 0x00000001) != 0); - } - /** - * 
-       * String name. The most common practice is to set this to a MID or synsets
-       * id. Synset: a set of synonyms that share a common meaning.
-       * https://en.wikipedia.org/wiki/WordNet
-       *
- * - * optional string name = 1; - * @return The name. - */ - public java.lang.String getName() { - java.lang.Object ref = name_; - if (!(ref instanceof java.lang.String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - if (bs.isValidUtf8()) { - name_ = s; - } - return s; - } else { - return (java.lang.String) ref; - } - } - /** - * 
-       * String name. The most common practice is to set this to a MID or synsets
-       * id. Synset: a set of synonyms that share a common meaning.
-       * https://en.wikipedia.org/wiki/WordNet
-       *
- * - * optional string name = 1; - * @return The bytes for name. - */ - public com.google.protobuf.ByteString - getNameBytes() { - java.lang.Object ref = name_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - name_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - /** - * 
-       * String name. The most common practice is to set this to a MID or synsets
-       * id. Synset: a set of synonyms that share a common meaning.
-       * https://en.wikipedia.org/wiki/WordNet
-       *
- * - * optional string name = 1; - * @param value The name to set. - * @return This builder for chaining. - */ - public Builder setName( - java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000001; - name_ = value; - onChanged(); - return this; - } - /** - * 
-       * String name. The most common practice is to set this to a MID or synsets
-       * id. Synset: a set of synonyms that share a common meaning.
-       * https://en.wikipedia.org/wiki/WordNet
-       *
- * - * optional string name = 1; - * @return This builder for chaining. - */ - public Builder clearName() { - bitField0_ = (bitField0_ & ~0x00000001); - name_ = getDefaultInstance().getName(); - onChanged(); - return this; - } - /** - * 
-       * String name. The most common practice is to set this to a MID or synsets
-       * id. Synset: a set of synonyms that share a common meaning.
-       * https://en.wikipedia.org/wiki/WordNet
-       *
- * - * optional string name = 1; - * @param value The bytes for name to set. - * @return This builder for chaining. - */ - public Builder setNameBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000001; - name_ = value; - onChanged(); - return this; - } - - private int id_ ; - /** - * 
-       * Integer id that maps to the string name above. Label ids should start
-       * from 1.
-       *
- * - * optional int32 id = 2; - * @return Whether the id field is set. - */ - @java.lang.Override - public boolean hasId() { - return ((bitField0_ & 0x00000002) != 0); - } - /** - * 
-       * Integer id that maps to the string name above. Label ids should start
-       * from 1.
-       *
- * - * optional int32 id = 2; - * @return The id. - */ - @java.lang.Override - public int getId() { - return id_; - } - /** - * 
-       * Integer id that maps to the string name above. Label ids should start
-       * from 1.
-       *
- * - * optional int32 id = 2; - * @param value The id to set. - * @return This builder for chaining. - */ - public Builder setId(int value) { - bitField0_ |= 0x00000002; - id_ = value; - onChanged(); - return this; - } - /** - * 
-       * Integer id that maps to the string name above. Label ids should start
-       * from 1.
-       *
- * - * optional int32 id = 2; - * @return This builder for chaining. - */ - public Builder clearId() { - bitField0_ = (bitField0_ & ~0x00000002); - id_ = 0; - onChanged(); - return this; - } - - private java.lang.Object displayName_ = ""; - /** - * 
-       * Human readable string label.
-       *
- * - * optional string display_name = 3; - * @return Whether the displayName field is set. - */ - public boolean hasDisplayName() { - return ((bitField0_ & 0x00000004) != 0); - } - /** - * 
-       * Human readable string label.
-       *
- * - * optional string display_name = 3; - * @return The displayName. - */ - public java.lang.String getDisplayName() { - java.lang.Object ref = displayName_; - if (!(ref instanceof java.lang.String)) { - com.google.protobuf.ByteString bs = - (com.google.protobuf.ByteString) ref; - java.lang.String s = bs.toStringUtf8(); - if (bs.isValidUtf8()) { - displayName_ = s; - } - return s; - } else { - return (java.lang.String) ref; - } - } - /** - * 
-       * Human readable string label.
-       *
- * - * optional string display_name = 3; - * @return The bytes for displayName. - */ - public com.google.protobuf.ByteString - getDisplayNameBytes() { - java.lang.Object ref = displayName_; - if (ref instanceof String) { - com.google.protobuf.ByteString b = - com.google.protobuf.ByteString.copyFromUtf8( - (java.lang.String) ref); - displayName_ = b; - return b; - } else { - return (com.google.protobuf.ByteString) ref; - } - } - /** - * 
-       * Human readable string label.
-       *
- * - * optional string display_name = 3; - * @param value The displayName to set. - * @return This builder for chaining. - */ - public Builder setDisplayName( - java.lang.String value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000004; - displayName_ = value; - onChanged(); - return this; - } - /** - * 
-       * Human readable string label.
-       *
- * - * optional string display_name = 3; - * @return This builder for chaining. - */ - public Builder clearDisplayName() { - bitField0_ = (bitField0_ & ~0x00000004); - displayName_ = getDefaultInstance().getDisplayName(); - onChanged(); - return this; - } - /** - * 
-       * Human readable string label.
-       *
- * - * optional string display_name = 3; - * @param value The bytes for displayName to set. - * @return This builder for chaining. - */ - public Builder setDisplayNameBytes( - com.google.protobuf.ByteString value) { - if (value == null) { - throw new NullPointerException(); - } - bitField0_ |= 0x00000004; - displayName_ = value; - onChanged(); - return this; - } - @java.lang.Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @java.lang.Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem) - } - - // @@protoc_insertion_point(class_scope:org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem) - private static final org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem DEFAULT_INSTANCE; - static { - DEFAULT_INSTANCE = new org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem(); - } - - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - @java.lang.Deprecated public static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - @java.lang.Override - public StringIntLabelMapItem parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - Builder builder = newBuilder(); - try { - builder.mergeFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(builder.buildPartial()); - } catch (com.google.protobuf.UninitializedMessageException e) { - throw e.asInvalidProtocolBufferException().setUnfinishedMessage(builder.buildPartial()); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException(e) - .setUnfinishedMessage(builder.buildPartial()); - } - return builder.buildPartial(); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @java.lang.Override - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - - } - - public interface StringIntLabelMapOrBuilder extends - // @@protoc_insertion_point(interface_extends:org.springframework.cloud.fn.object.detection.protos.StringIntLabelMap) - com.google.protobuf.MessageOrBuilder { - - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - java.util.List - getItemList(); - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem getItem(int index); - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - int getItemCount(); - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - java.util.List - getItemOrBuilderList(); - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItemOrBuilder getItemOrBuilder( - int index); - } - /** - * Protobuf type {@code org.springframework.cloud.fn.object.detection.protos.StringIntLabelMap} - */ - public static final class StringIntLabelMap extends - com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:org.springframework.cloud.fn.object.detection.protos.StringIntLabelMap) - StringIntLabelMapOrBuilder { - private static final long serialVersionUID = 0L; - // Use StringIntLabelMap.newBuilder() to construct. - private StringIntLabelMap(com.google.protobuf.GeneratedMessageV3.Builder builder) { - super(builder); - } - private StringIntLabelMap() { - item_ = java.util.Collections.emptyList(); - } - - @java.lang.Override - @SuppressWarnings({"unused"}) - protected java.lang.Object newInstance( - UnusedPrivateParameter unused) { - return new StringIntLabelMap(); - } - - @java.lang.Override - public final com.google.protobuf.UnknownFieldSet - getUnknownFields() { - return this.unknownFields; - } - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMap_descriptor; - } - - @java.lang.Override - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMap_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap.class, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap.Builder.class); - } - - public static final int ITEM_FIELD_NUMBER = 1; - private java.util.List item_; - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - @java.lang.Override - public java.util.List getItemList() { - return item_; - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - @java.lang.Override - public java.util.List - getItemOrBuilderList() { - return item_; - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - @java.lang.Override - public int getItemCount() { - return item_.size(); - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - @java.lang.Override - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem getItem(int index) { - return item_.get(index); - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - @java.lang.Override - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItemOrBuilder getItemOrBuilder( - int index) { - return item_.get(index); - } - - private byte memoizedIsInitialized = -1; - @java.lang.Override - public final boolean isInitialized() { - byte isInitialized = memoizedIsInitialized; - if (isInitialized == 1) return true; - if (isInitialized == 0) return false; - - memoizedIsInitialized = 1; - return true; - } - - @java.lang.Override - public void writeTo(com.google.protobuf.CodedOutputStream output) - throws java.io.IOException { - for (int i = 0; i < item_.size(); i++) { - output.writeMessage(1, item_.get(i)); - } - getUnknownFields().writeTo(output); - } - - @java.lang.Override - public int getSerializedSize() { - int size = memoizedSize; - if (size != -1) return size; - - size = 0; - for (int i = 0; i < item_.size(); i++) { - size += com.google.protobuf.CodedOutputStream - .computeMessageSize(1, item_.get(i)); - } - size += getUnknownFields().getSerializedSize(); - memoizedSize = size; - return size; - } - - @java.lang.Override - public boolean equals(final java.lang.Object obj) { - if (obj == this) { - return true; - } - if (!(obj instanceof org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap)) { - return super.equals(obj); - } - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap other = (org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap) obj; - - if (!getItemList() - .equals(other.getItemList())) return false; - if (!getUnknownFields().equals(other.getUnknownFields())) return false; - return true; - } - - @java.lang.Override - public int hashCode() { - if (memoizedHashCode != 0) { - return memoizedHashCode; - } - int hash = 41; - hash = (19 * hash) + getDescriptor().hashCode(); - if (getItemCount() > 0) { - hash = (37 * hash) + ITEM_FIELD_NUMBER; - hash = (53 * hash) + getItemList().hashCode(); - } - hash = (29 * hash) + getUnknownFields().hashCode(); - memoizedHashCode = hash; - return hash; - } - - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseFrom( - java.nio.ByteBuffer data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseFrom( - java.nio.ByteBuffer data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseFrom( - com.google.protobuf.ByteString data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseFrom( - com.google.protobuf.ByteString data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseFrom(byte[] data) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseFrom( - byte[] data, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - return PARSER.parseFrom(data, extensionRegistry); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseDelimitedFrom(java.io.InputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseDelimitedFrom( - java.io.InputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseDelimitedWithIOException(PARSER, input, extensionRegistry); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseFrom( - com.google.protobuf.CodedInputStream input) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input); - } - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap parseFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - return com.google.protobuf.GeneratedMessageV3 - .parseWithIOException(PARSER, input, extensionRegistry); - } - - @java.lang.Override - public Builder newBuilderForType() { return newBuilder(); } - public static Builder newBuilder() { - return DEFAULT_INSTANCE.toBuilder(); - } - public static Builder newBuilder(org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap prototype) { - return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); - } - @java.lang.Override - public Builder toBuilder() { - return this == DEFAULT_INSTANCE - ? new Builder() : new Builder().mergeFrom(this); - } - - @java.lang.Override - protected Builder newBuilderForType( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - Builder builder = new Builder(parent); - return builder; - } - /** - * Protobuf type {@code org.springframework.cloud.fn.object.detection.protos.StringIntLabelMap} - */ - public static final class Builder extends - com.google.protobuf.GeneratedMessageV3.Builder implements - // @@protoc_insertion_point(builder_implements:org.springframework.cloud.fn.object.detection.protos.StringIntLabelMap) - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapOrBuilder { - public static final com.google.protobuf.Descriptors.Descriptor - getDescriptor() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMap_descriptor; - } - - @java.lang.Override - protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internalGetFieldAccessorTable() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMap_fieldAccessorTable - .ensureFieldAccessorsInitialized( - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap.class, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap.Builder.class); - } - - // Construct using org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap.newBuilder() - private Builder() { - - } - - private Builder( - com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { - super(parent); - - } - @java.lang.Override - public Builder clear() { - super.clear(); - if (itemBuilder_ == null) { - item_ = java.util.Collections.emptyList(); - } else { - item_ = null; - itemBuilder_.clear(); - } - bitField0_ = (bitField0_ & ~0x00000001); - return this; - } - - @java.lang.Override - public com.google.protobuf.Descriptors.Descriptor - getDescriptorForType() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMap_descriptor; - } - - @java.lang.Override - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap getDefaultInstanceForType() { - return org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap.getDefaultInstance(); - } - - @java.lang.Override - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap build() { - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap result = buildPartial(); - if (!result.isInitialized()) { - throw newUninitializedMessageException(result); - } - return result; - } - - @java.lang.Override - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap buildPartial() { - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap result = new org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap(this); - int from_bitField0_ = bitField0_; - if (itemBuilder_ == null) { - if (((bitField0_ & 0x00000001) != 0)) { - item_ = java.util.Collections.unmodifiableList(item_); - bitField0_ = (bitField0_ & ~0x00000001); - } - result.item_ = item_; - } else { - result.item_ = itemBuilder_.build(); - } - onBuilt(); - return result; - } - - @java.lang.Override - public Builder clone() { - return super.clone(); - } - @java.lang.Override - public Builder setField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return super.setField(field, value); - } - @java.lang.Override - public Builder clearField( - com.google.protobuf.Descriptors.FieldDescriptor field) { - return super.clearField(field); - } - @java.lang.Override - public Builder clearOneof( - com.google.protobuf.Descriptors.OneofDescriptor oneof) { - return super.clearOneof(oneof); - } - @java.lang.Override - public Builder setRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - int index, java.lang.Object value) { - return super.setRepeatedField(field, index, value); - } - @java.lang.Override - public Builder addRepeatedField( - com.google.protobuf.Descriptors.FieldDescriptor field, - java.lang.Object value) { - return super.addRepeatedField(field, value); - } - @java.lang.Override - public Builder mergeFrom(com.google.protobuf.Message other) { - if (other instanceof org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap) { - return mergeFrom((org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap)other); - } else { - super.mergeFrom(other); - return this; - } - } - - public Builder mergeFrom(org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap other) { - if (other == org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap.getDefaultInstance()) return this; - if (itemBuilder_ == null) { - if (!other.item_.isEmpty()) { - if (item_.isEmpty()) { - item_ = other.item_; - bitField0_ = (bitField0_ & ~0x00000001); - } else { - ensureItemIsMutable(); - item_.addAll(other.item_); - } - onChanged(); - } - } else { - if (!other.item_.isEmpty()) { - if (itemBuilder_.isEmpty()) { - itemBuilder_.dispose(); - itemBuilder_ = null; - item_ = other.item_; - bitField0_ = (bitField0_ & ~0x00000001); - itemBuilder_ = - com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ? - getItemFieldBuilder() : null; - } else { - itemBuilder_.addAllMessages(other.item_); - } - } - } - this.mergeUnknownFields(other.getUnknownFields()); - onChanged(); - return this; - } - - @java.lang.Override - public final boolean isInitialized() { - return true; - } - - @java.lang.Override - public Builder mergeFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws java.io.IOException { - if (extensionRegistry == null) { - throw new java.lang.NullPointerException(); - } - try { - boolean done = false; - while (!done) { - int tag = input.readTag(); - switch (tag) { - case 0: - done = true; - break; - case 10: { - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem m = - input.readMessage( - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.PARSER, - extensionRegistry); - if (itemBuilder_ == null) { - ensureItemIsMutable(); - item_.add(m); - } else { - itemBuilder_.addMessage(m); - } - break; - } // case 10 - default: { - if (!super.parseUnknownField(input, extensionRegistry, tag)) { - done = true; // was an endgroup tag - } - break; - } // default: - } // switch (tag) - } // while (!done) - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.unwrapIOException(); - } finally { - onChanged(); - } // finally - return this; - } - private int bitField0_; - - private java.util.List item_ = - java.util.Collections.emptyList(); - private void ensureItemIsMutable() { - if (!((bitField0_ & 0x00000001) != 0)) { - item_ = new java.util.ArrayList(item_); - bitField0_ |= 0x00000001; - } - } - - private com.google.protobuf.RepeatedFieldBuilderV3< - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.Builder, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItemOrBuilder> itemBuilder_; - - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public java.util.List getItemList() { - if (itemBuilder_ == null) { - return java.util.Collections.unmodifiableList(item_); - } else { - return itemBuilder_.getMessageList(); - } - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public int getItemCount() { - if (itemBuilder_ == null) { - return item_.size(); - } else { - return itemBuilder_.getCount(); - } - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem getItem(int index) { - if (itemBuilder_ == null) { - return item_.get(index); - } else { - return itemBuilder_.getMessage(index); - } - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public Builder setItem( - int index, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem value) { - if (itemBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); - } - ensureItemIsMutable(); - item_.set(index, value); - onChanged(); - } else { - itemBuilder_.setMessage(index, value); - } - return this; - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public Builder setItem( - int index, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.Builder builderForValue) { - if (itemBuilder_ == null) { - ensureItemIsMutable(); - item_.set(index, builderForValue.build()); - onChanged(); - } else { - itemBuilder_.setMessage(index, builderForValue.build()); - } - return this; - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public Builder addItem(org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem value) { - if (itemBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); - } - ensureItemIsMutable(); - item_.add(value); - onChanged(); - } else { - itemBuilder_.addMessage(value); - } - return this; - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public Builder addItem( - int index, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem value) { - if (itemBuilder_ == null) { - if (value == null) { - throw new NullPointerException(); - } - ensureItemIsMutable(); - item_.add(index, value); - onChanged(); - } else { - itemBuilder_.addMessage(index, value); - } - return this; - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public Builder addItem( - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.Builder builderForValue) { - if (itemBuilder_ == null) { - ensureItemIsMutable(); - item_.add(builderForValue.build()); - onChanged(); - } else { - itemBuilder_.addMessage(builderForValue.build()); - } - return this; - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public Builder addItem( - int index, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.Builder builderForValue) { - if (itemBuilder_ == null) { - ensureItemIsMutable(); - item_.add(index, builderForValue.build()); - onChanged(); - } else { - itemBuilder_.addMessage(index, builderForValue.build()); - } - return this; - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public Builder addAllItem( - java.lang.Iterable values) { - if (itemBuilder_ == null) { - ensureItemIsMutable(); - com.google.protobuf.AbstractMessageLite.Builder.addAll( - values, item_); - onChanged(); - } else { - itemBuilder_.addAllMessages(values); - } - return this; - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public Builder clearItem() { - if (itemBuilder_ == null) { - item_ = java.util.Collections.emptyList(); - bitField0_ = (bitField0_ & ~0x00000001); - onChanged(); - } else { - itemBuilder_.clear(); - } - return this; - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public Builder removeItem(int index) { - if (itemBuilder_ == null) { - ensureItemIsMutable(); - item_.remove(index); - onChanged(); - } else { - itemBuilder_.remove(index); - } - return this; - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.Builder getItemBuilder( - int index) { - return getItemFieldBuilder().getBuilder(index); - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItemOrBuilder getItemOrBuilder( - int index) { - if (itemBuilder_ == null) { - return item_.get(index); } else { - return itemBuilder_.getMessageOrBuilder(index); - } - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public java.util.List - getItemOrBuilderList() { - if (itemBuilder_ != null) { - return itemBuilder_.getMessageOrBuilderList(); - } else { - return java.util.Collections.unmodifiableList(item_); - } - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.Builder addItemBuilder() { - return getItemFieldBuilder().addBuilder( - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.getDefaultInstance()); - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.Builder addItemBuilder( - int index) { - return getItemFieldBuilder().addBuilder( - index, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.getDefaultInstance()); - } - /** - * repeated .org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapItem item = 1; - */ - public java.util.List - getItemBuilderList() { - return getItemFieldBuilder().getBuilderList(); - } - private com.google.protobuf.RepeatedFieldBuilderV3< - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.Builder, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItemOrBuilder> - getItemFieldBuilder() { - if (itemBuilder_ == null) { - itemBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3< - org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItem.Builder, org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMapItemOrBuilder>( - item_, - ((bitField0_ & 0x00000001) != 0), - getParentForChildren(), - isClean()); - item_ = null; - } - return itemBuilder_; - } - @java.lang.Override - public final Builder setUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.setUnknownFields(unknownFields); - } - - @java.lang.Override - public final Builder mergeUnknownFields( - final com.google.protobuf.UnknownFieldSet unknownFields) { - return super.mergeUnknownFields(unknownFields); - } - - - // @@protoc_insertion_point(builder_scope:org.springframework.cloud.fn.object.detection.protos.StringIntLabelMap) - } - - // @@protoc_insertion_point(class_scope:org.springframework.cloud.fn.object.detection.protos.StringIntLabelMap) - private static final org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap DEFAULT_INSTANCE; - static { - DEFAULT_INSTANCE = new org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap(); - } - - public static org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap getDefaultInstance() { - return DEFAULT_INSTANCE; - } - - @java.lang.Deprecated public static final com.google.protobuf.Parser - PARSER = new com.google.protobuf.AbstractParser() { - @java.lang.Override - public StringIntLabelMap parsePartialFrom( - com.google.protobuf.CodedInputStream input, - com.google.protobuf.ExtensionRegistryLite extensionRegistry) - throws com.google.protobuf.InvalidProtocolBufferException { - Builder builder = newBuilder(); - try { - builder.mergeFrom(input, extensionRegistry); - } catch (com.google.protobuf.InvalidProtocolBufferException e) { - throw e.setUnfinishedMessage(builder.buildPartial()); - } catch (com.google.protobuf.UninitializedMessageException e) { - throw e.asInvalidProtocolBufferException().setUnfinishedMessage(builder.buildPartial()); - } catch (java.io.IOException e) { - throw new com.google.protobuf.InvalidProtocolBufferException(e) - .setUnfinishedMessage(builder.buildPartial()); - } - return builder.buildPartial(); - } - }; - - public static com.google.protobuf.Parser parser() { - return PARSER; - } - - @java.lang.Override - public com.google.protobuf.Parser getParserForType() { - return PARSER; - } - - @java.lang.Override - public org.springframework.cloud.fn.object.detection.protos.StringIntLabelMapOuterClass.StringIntLabelMap getDefaultInstanceForType() { - return DEFAULT_INSTANCE; - } - - } - - private static final com.google.protobuf.Descriptors.Descriptor - internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMapItem_descriptor; - private static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMapItem_fieldAccessorTable; - private static final com.google.protobuf.Descriptors.Descriptor - internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMap_descriptor; - private static final - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable - internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMap_fieldAccessorTable; - - public static com.google.protobuf.Descriptors.FileDescriptor - getDescriptor() { - return descriptor; - } - private static com.google.protobuf.Descriptors.FileDescriptor - descriptor; - static { - java.lang.String[] descriptorData = { - "\n\032string_int_label_map.proto\0224org.spring" + - "framework.cloud.fn.object.detection.prot" + - "os\"G\n\025StringIntLabelMapItem\022\014\n\004name\030\001 \001(" + - "\t\022\n\n\002id\030\002 \001(\005\022\024\n\014display_name\030\003 \001(\t\"n\n\021S" + - "tringIntLabelMap\022Y\n\004item\030\001 \003(\0132K.org.spr" + - "ingframework.cloud.fn.object.detection.p" + - "rotos.StringIntLabelMapItem" - }; - descriptor = com.google.protobuf.Descriptors.FileDescriptor - .internalBuildGeneratedFileFrom(descriptorData, - new com.google.protobuf.Descriptors.FileDescriptor[] { - }); - internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMapItem_descriptor = - getDescriptor().getMessageTypes().get(0); - internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMapItem_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMapItem_descriptor, - new java.lang.String[] { "Name", "Id", "DisplayName", }); - internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMap_descriptor = - getDescriptor().getMessageTypes().get(1); - internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMap_fieldAccessorTable = new - com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( - internal_static_org_springframework_cloud_fn_object_detection_protos_StringIntLabelMap_descriptor, - new java.lang.String[] { "Item", }); - } - - // @@protoc_insertion_point(outer_class_scope) -} diff --git a/functions/function/object-detection-function/src/main/proto/example.proto__ b/functions/function/object-detection-function/src/main/proto/example.proto__ deleted file mode 100644 index fec7562aa..000000000 --- a/functions/function/object-detection-function/src/main/proto/example.proto__ +++ /dev/null @@ -1,301 +0,0 @@ -// Protocol messages for describing input data Examples for machine learning -// model training or inference. -syntax = "proto3"; - -import "feature.proto"; -option cc_enable_arenas = true; -option java_outer_classname = "ExampleProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.example"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/example"; -package tensorflow; - -// An Example is a mostly-normalized data format for storing data for -// training and inference. It contains a key-value store (features); where -// each key (string) maps to a Feature message (which is oneof packed BytesList, -// FloatList, or Int64List). This flexible and compact format allows the -// storage of large amounts of typed data, but requires that the data shape -// and use be determined by the configuration files and parsers that are used to -// read and write this format. That is, the Example is mostly *not* a -// self-describing format. In TensorFlow, Examples are read in row-major -// format, so any configuration that describes data with rank-2 or above -// should keep this in mind. For example, to store an M x N matrix of Bytes, -// the BytesList must contain M*N bytes, with M rows of N contiguous values -// each. That is, the BytesList value must store the matrix as: -// .... row 0 .... .... row 1 .... // ........... // ... row M-1 .... -// -// An Example for a movie recommendation application: -// features { -// feature { -// key: "age" -// value { float_list { -// value: 29.0 -// }} -// } -// feature { -// key: "movie" -// value { bytes_list { -// value: "The Shawshank Redemption" -// value: "Fight Club" -// }} -// } -// feature { -// key: "movie_ratings" -// value { float_list { -// value: 9.0 -// value: 9.7 -// }} -// } -// feature { -// key: "suggestion" -// value { bytes_list { -// value: "Inception" -// }} -// } -// # Note that this feature exists to be used as a label in training. -// # E.g., if training a logistic regression model to predict purchase -// # probability in our learning tool we would set the label feature to -// # "suggestion_purchased". -// feature { -// key: "suggestion_purchased" -// value { float_list { -// value: 1.0 -// }} -// } -// # Similar to "suggestion_purchased" above this feature exists to be used -// # as a label in training. -// # E.g., if training a linear regression model to predict purchase -// # price in our learning tool we would set the label feature to -// # "purchase_price". -// feature { -// key: "purchase_price" -// value { float_list { -// value: 9.99 -// }} -// } -// } -// -// A conformant Example data set obeys the following conventions: -// - If a Feature K exists in one example with data type T, it must be of -// type T in all other examples when present. It may be omitted. -// - The number of instances of Feature K list data may vary across examples, -// depending on the requirements of the model. -// - If a Feature K doesn't exist in an example, a K-specific default will be -// used, if configured. -// - If a Feature K exists in an example but contains no items, the intent -// is considered to be an empty tensor and no default will be used. - -message Example { - Features features = 1; -}; - -// A SequenceExample is an Example representing one or more sequences, and -// some context. The context contains features which apply to the entire -// example. The feature_lists contain a key, value map where each key is -// associated with a repeated set of Features (a FeatureList). -// A FeatureList thus represents the values of a feature identified by its key -// over time / frames. -// -// Below is a SequenceExample for a movie recommendation application recording a -// sequence of ratings by a user. The time-independent features ("locale", -// "age", "favorites") describing the user are part of the context. The sequence -// of movies the user rated are part of the feature_lists. For each movie in the -// sequence we have information on its name and actors and the user's rating. -// This information is recorded in three separate feature_list(s). -// In the example below there are only two movies. All three feature_list(s), -// namely "movie_ratings", "movie_names", and "actors" have a feature value for -// both movies. Note, that "actors" is itself a bytes_list with multiple -// strings per movie. -// -// context: { -// feature: { -// key : "locale" -// value: { -// bytes_list: { -// value: [ "pt_BR" ] -// } -// } -// } -// feature: { -// key : "age" -// value: { -// float_list: { -// value: [ 19.0 ] -// } -// } -// } -// feature: { -// key : "favorites" -// value: { -// bytes_list: { -// value: [ "Majesty Rose", "Savannah Outen", "One Direction" ] -// } -// } -// } -// } -// feature_lists: { -// feature_list: { -// key : "movie_ratings" -// value: { -// feature: { -// float_list: { -// value: [ 4.5 ] -// } -// } -// feature: { -// float_list: { -// value: [ 5.0 ] -// } -// } -// } -// } -// feature_list: { -// key : "movie_names" -// value: { -// feature: { -// bytes_list: { -// value: [ "The Shawshank Redemption" ] -// } -// } -// feature: { -// bytes_list: { -// value: [ "Fight Club" ] -// } -// } -// } -// } -// feature_list: { -// key : "actors" -// value: { -// feature: { -// bytes_list: { -// value: [ "Tim Robbins", "Morgan Freeman" ] -// } -// } -// feature: { -// bytes_list: { -// value: [ "Brad Pitt", "Edward Norton", "Helena Bonham Carter" ] -// } -// } -// } -// } -// } -// -// A conformant SequenceExample data set obeys the following conventions: -// -// Context: -// - All conformant context features K must obey the same conventions as -// a conformant Example's features (see above). -// Feature lists: -// - A FeatureList L may be missing in an example; it is up to the -// parser configuration to determine if this is allowed or considered -// an empty list (zero length). -// - If a FeatureList L exists, it may be empty (zero length). -// - If a FeatureList L is non-empty, all features within the FeatureList -// must have the same data type T. Even across SequenceExamples, the type T -// of the FeatureList identified by the same key must be the same. An entry -// without any values may serve as an empty feature. -// - If a FeatureList L is non-empty, it is up to the parser configuration -// to determine if all features within the FeatureList must -// have the same size. The same holds for this FeatureList across multiple -// examples. -// - For sequence modeling, e.g.: -// http://colah.github.io/posts/2015-08-Understanding-LSTMs/ -// https://github.com/tensorflow/nmt -// the feature lists represent a sequence of frames. -// In this scenario, all FeatureLists in a SequenceExample have the same -// number of Feature messages, so that the ith element in each FeatureList -// is part of the ith frame (or time step). -// Examples of conformant and non-conformant examples' FeatureLists: -// -// Conformant FeatureLists: -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { feature: { float_list: { value: [ 4.5 ] } } -// feature: { float_list: { value: [ 5.0 ] } } } -// } } -// -// Non-conformant FeatureLists (mismatched types): -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { feature: { float_list: { value: [ 4.5 ] } } -// feature: { int64_list: { value: [ 5 ] } } } -// } } -// -// Conditionally conformant FeatureLists, the parser configuration determines -// if the feature sizes must match: -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { feature: { float_list: { value: [ 4.5 ] } } -// feature: { float_list: { value: [ 5.0, 6.0 ] } } } -// } } -// -// Conformant pair of SequenceExample -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { feature: { float_list: { value: [ 4.5 ] } } -// feature: { float_list: { value: [ 5.0 ] } } } -// } } -// and: -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { feature: { float_list: { value: [ 4.5 ] } } -// feature: { float_list: { value: [ 5.0 ] } } -// feature: { float_list: { value: [ 2.0 ] } } } -// } } -// -// Conformant pair of SequenceExample -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { feature: { float_list: { value: [ 4.5 ] } } -// feature: { float_list: { value: [ 5.0 ] } } } -// } } -// and: -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { } -// } } -// -// Conditionally conformant pair of SequenceExample, the parser configuration -// determines if the second feature_lists is consistent (zero-length) or -// invalid (missing "movie_ratings"): -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { feature: { float_list: { value: [ 4.5 ] } } -// feature: { float_list: { value: [ 5.0 ] } } } -// } } -// and: -// feature_lists: { } -// -// Non-conformant pair of SequenceExample (mismatched types) -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { feature: { float_list: { value: [ 4.5 ] } } -// feature: { float_list: { value: [ 5.0 ] } } } -// } } -// and: -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { feature: { int64_list: { value: [ 4 ] } } -// feature: { int64_list: { value: [ 5 ] } } -// feature: { int64_list: { value: [ 2 ] } } } -// } } -// -// Conditionally conformant pair of SequenceExample; the parser configuration -// determines if the feature sizes must match: -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { feature: { float_list: { value: [ 4.5 ] } } -// feature: { float_list: { value: [ 5.0 ] } } } -// } } -// and: -// feature_lists: { feature_list: { -// key: "movie_ratings" -// value: { feature: { float_list: { value: [ 4.0 ] } } -// feature: { float_list: { value: [ 5.0, 3.0 ] } } -// } } - -message SequenceExample { - Features context = 1; - FeatureLists feature_lists = 2; -}; diff --git a/functions/function/object-detection-function/src/main/proto/feature.proto__ b/functions/function/object-detection-function/src/main/proto/feature.proto__ deleted file mode 100644 index 56f0f05e4..000000000 --- a/functions/function/object-detection-function/src/main/proto/feature.proto__ +++ /dev/null @@ -1,105 +0,0 @@ -// Protocol messages for describing features for machine learning model -// training or inference. -// -// There are three base Feature types: -// - bytes -// - float -// - int64 -// -// A Feature contains Lists which may hold zero or more values. These -// lists are the base values BytesList, FloatList, Int64List. -// -// Features are organized into categories by name. The Features message -// contains the mapping from name to Feature. -// -// Example Features for a movie recommendation application: -// feature { -// key: "age" -// value { float_list { -// value: 29.0 -// }} -// } -// feature { -// key: "movie" -// value { bytes_list { -// value: "The Shawshank Redemption" -// value: "Fight Club" -// }} -// } -// feature { -// key: "movie_ratings" -// value { float_list { -// value: 9.0 -// value: 9.7 -// }} -// } -// feature { -// key: "suggestion" -// value { bytes_list { -// value: "Inception" -// }} -// } -// feature { -// key: "suggestion_purchased" -// value { int64_list { -// value: 1 -// }} -// } -// feature { -// key: "purchase_price" -// value { float_list { -// value: 9.99 -// }} -// } -// - -syntax = "proto3"; -option cc_enable_arenas = true; -option java_outer_classname = "FeatureProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.example"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/example"; -package tensorflow; - -// Containers to hold repeated fundamental values. -message BytesList { - repeated bytes value = 1; -} -message FloatList { - repeated float value = 1 [packed = true]; -} -message Int64List { - repeated int64 value = 1 [packed = true]; -} - -// Containers for non-sequential data. -message Feature { - // Each feature can be exactly one kind. - oneof kind { - BytesList bytes_list = 1; - FloatList float_list = 2; - Int64List int64_list = 3; - } -}; - -message Features { - // Map from feature name to feature. - map feature = 1; -}; - -// Containers for sequential data. -// -// A FeatureList contains lists of Features. These may hold zero or more -// Feature values. -// -// FeatureLists are organized into categories by name. The FeatureLists message -// contains the mapping from name to FeatureList. -// -message FeatureList { - repeated Feature feature = 1; -}; - -message FeatureLists { - // Map from feature name to feature list. - map feature_list = 1; -}; diff --git a/functions/function/object-detection-function/src/main/proto/string_int_label_map.proto b/functions/function/object-detection-function/src/main/proto/string_int_label_map.proto deleted file mode 100644 index bc5ccf47c..000000000 --- a/functions/function/object-detection-function/src/main/proto/string_int_label_map.proto +++ /dev/null @@ -1,25 +0,0 @@ -// Message to store the mapping from class label strings to class id. Datasets -// use string labels to represent classes while the object detection framework -// works fromMemory class ids. This message maps them so they can be converted back -// and forth as needed. -syntax = "proto2"; - -package org.springframework.cloud.fn.object.detection.protos; - -message StringIntLabelMapItem { - // String name. The most common practice is to set this to a MID or synsets - // id. Synset: a set of synonyms that share a common meaning. - // https://en.wikipedia.org/wiki/WordNet - optional string name = 1; - - // Integer id that maps to the string name above. Label ids should start - // from 1. - optional int32 id = 2; - - // Human readable string label. - optional string display_name = 3; -}; - -message StringIntLabelMap { - repeated StringIntLabelMapItem item = 1; -}; diff --git a/functions/function/object-detection-function/src/main/resources/images/object-detection-augmented.jpg b/functions/function/object-detection-function/src/main/resources/images/object-detection-augmented.jpg deleted file mode 100644 index f17adfd83..000000000 Binary files a/functions/function/object-detection-function/src/main/resources/images/object-detection-augmented.jpg and /dev/null differ diff --git a/functions/function/object-detection-function/src/main/resources/images/object-detection-segmentation-augmented.jpg b/functions/function/object-detection-function/src/main/resources/images/object-detection-segmentation-augmented.jpg deleted file mode 100644 index e9466f1e6..000000000 Binary files a/functions/function/object-detection-function/src/main/resources/images/object-detection-segmentation-augmented.jpg and /dev/null differ diff --git a/functions/function/object-detection-function/src/main/resources/images/object-detection.jpg b/functions/function/object-detection-function/src/main/resources/images/object-detection.jpg deleted file mode 100644 index 9eb325ac5..000000000 Binary files a/functions/function/object-detection-function/src/main/resources/images/object-detection.jpg and /dev/null differ diff --git a/functions/function/object-detection-function/src/main/resources/images/object_detection_1.jpg b/functions/function/object-detection-function/src/main/resources/images/object_detection_1.jpg deleted file mode 100644 index 913b2e6db..000000000 Binary files a/functions/function/object-detection-function/src/main/resources/images/object_detection_1.jpg and /dev/null differ diff --git a/functions/function/object-detection-function/src/test/java/org/springframework/cloud/fn/object/detection/examples/ExampleInstanceSegmentation.java b/functions/function/object-detection-function/src/test/java/org/springframework/cloud/fn/object/detection/examples/ExampleInstanceSegmentation.java deleted file mode 100644 index c5185bd75..000000000 --- a/functions/function/object-detection-function/src/test/java/org/springframework/cloud/fn/object/detection/examples/ExampleInstanceSegmentation.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.object.detection.examples; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.List; - -import org.apache.commons.io.IOUtils; - -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; -import org.springframework.cloud.fn.common.tensorflow.deprecated.JsonMapperFunction; -import org.springframework.cloud.fn.object.detection.ObjectDetectionImageAugmenter; -import org.springframework.cloud.fn.object.detection.ObjectDetectionService; -import org.springframework.cloud.fn.object.detection.domain.ObjectDetection; -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.core.io.ResourceLoader; - -/** - * 4 of the pre-trained model in the model zoo (https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md) - * can also compute the masks of the detected objects, providing instance segmentation. - *

- * Here are the models that can be used for instance segmentation. - *

- * mask_rcnn_inception_resnet_v2_atrous_coco 771 36 Masks - * mask_rcnn_inception_v2_coco 79 25 Masks - * mask_rcnn_resnet101_atrous_coco 470 33 Masks - * mask_rcnn_resnet50_atrous_coco 343 29 Masks - * - * @author Christian Tzolov - */ -public class ExampleInstanceSegmentation { - - public static void main(String[] args) throws IOException { - - ResourceLoader resourceLoader = new DefaultResourceLoader(); - - // You can download pre-trained models directly from the zoo: https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md - // Just use the notation # - // For performance reasons you may consider downloading the model locally and use the file:/ URI instead! - String model = "https://download.tensorflow.org/models/object_detection/mask_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz#frozen_inference_graph.pb"; - - // All labels for the pre-trained models are available at: - // https://github.com/tensorflow/models/tree/master/research/object_detection/data - // Use the labels applicable for the model. - // Also, for performance reasons you may consider to download the labels and load them from file: instead. - String labels = "https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt"; - - // You can cache the TF model on the local file system to improve the bootstrap performance on consecutive runs! - boolean CACHE_TF_MODEL = true; - - // For the pre-trained models fromMemory mask you can set the INSTANCE_SEGMENTATION to enable object instance segmentation as well - boolean INSTANCE_SEGMENTATION = true; - - // Only object fromMemory confidence above the threshold are returned - float CONFIDENCE_THRESHOLD = 0.4f; - - ObjectDetectionService detectionService = - new ObjectDetectionService(model, labels, CONFIDENCE_THRESHOLD, INSTANCE_SEGMENTATION, CACHE_TF_MODEL); - - // You can use file:, http: or classpath: to provide the path to the input image. - byte[] image = GraphicsUtils.loadAsByteArray("classpath:/images/object-detection.jpg"); - - // Returns a list ObjectDetection domain classes to allow programmatic accesses to the detected objects's metadata - List detectedObjects = detectionService.detect(image); - - // Get JSON representation of the detected objects - String jsonObjectDetections = new JsonMapperFunction().apply(detectedObjects); - System.out.println(jsonObjectDetections); - - // Draw the detected object metadata on top of the original image and store the result - byte[] annotatedImage = new ObjectDetectionImageAugmenter(INSTANCE_SEGMENTATION).apply(image, detectedObjects); - File projectDir = new File("functions/function/object-detection-function"); - File output = new File(projectDir, "target/object-detection-segmentation-augmented.jpg"); - IOUtils.write(annotatedImage, new FileOutputStream(output)); - System.out.println("Created:" + output.getPath()); - } -} diff --git a/functions/function/object-detection-function/src/test/java/org/springframework/cloud/fn/object/detection/examples/ExampleObjectDetection.java b/functions/function/object-detection-function/src/test/java/org/springframework/cloud/fn/object/detection/examples/ExampleObjectDetection.java deleted file mode 100644 index 7a8899e1b..000000000 --- a/functions/function/object-detection-function/src/test/java/org/springframework/cloud/fn/object/detection/examples/ExampleObjectDetection.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.object.detection.examples; - -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; - -import org.apache.commons.io.IOUtils; - -import org.springframework.cloud.fn.common.tensorflow.deprecated.JsonMapperFunction; -import org.springframework.cloud.fn.object.detection.ObjectDetectionImageAugmenter; -import org.springframework.cloud.fn.object.detection.ObjectDetectionService; -import org.springframework.cloud.fn.object.detection.domain.ObjectDetection; -import org.springframework.core.io.DefaultResourceLoader; -import org.springframework.util.StreamUtils; - -/** - * @author Christian Tzolov - */ -public class ExampleObjectDetection { - - public static void main(String[] args) throws IOException { - - // You can download pre-trained models directly from the zoo: https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md - // Just use the notation # - // For performance reasons you may consider downloading the model locally and use the file:/ URI instead! - String model = "https://download.tensorflow.org/models/object_detection/faster_rcnn_nas_coco_2018_01_28.tar.gz#frozen_inference_graph.pb"; - //Resource model = resourceLoader.getResource("https://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_fgvc_2018_07_19.tar.gz#frozen_inference_graph.pb"); - //Resource model = resourceLoader.getResource("https://download.tensorflow.org/models/object_detection/faster_rcnn_resnet50_fgvc_2018_07_19.tar.gz#frozen_inference_graph.pb"); - - // All labels for the pre-trained models are available at: - // https://github.com/tensorflow/models/tree/master/research/object_detection/data - // Use the labels applicable for the model. - // Also, for performance reasons you may consider to download the labels and load them from file: instead. - String labels = "https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt"; - //Resource labels = resourceLoader.getResource("https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/fgvc_2854_classes_label_map.pbtxt"); - - // You can cache the TF model on the local file system to improve the bootstrap performance on consecutive runs! - boolean CACHE_TF_MODEL = true; - - // For the pre-trained models fromMemory mask you can set the INSTANCE_SEGMENTATION to enable object instance segmentation as well - boolean NO_INSTANCE_SEGMENTATION = false; - - // Only object fromMemory confidence above the threshold are returned - float CONFIDENCE_THRESHOLD = 0.4f; - - ObjectDetectionService detectionService = - new ObjectDetectionService(model, labels, CONFIDENCE_THRESHOLD, NO_INSTANCE_SEGMENTATION, CACHE_TF_MODEL); - - // You can use file:, http: or classpath: to provide the path to the input image. - String inputImageUri = "classpath:/images/object-detection.jpg"; - try (InputStream is = new DefaultResourceLoader().getResource(inputImageUri).getInputStream()) { - - byte[] image = StreamUtils.copyToByteArray(is); - - // Returns a list ObjectDetection domain classes to allow programmatic accesses to the detected objects's metadata - List detectedObjects = detectionService.detect(image); - - // Get JSON representation of the detected objects - String jsonObjectDetections = new JsonMapperFunction().apply(detectedObjects); - System.out.println(jsonObjectDetections); - - // Draw the detected object metadata on top of the original image and store the result - byte[] annotatedImage = new ObjectDetectionImageAugmenter(NO_INSTANCE_SEGMENTATION).apply(image, detectedObjects); - IOUtils.write(annotatedImage, new FileOutputStream("./object-detection-function/target/object-detection-augmented.jpg")); - } - } -} diff --git a/functions/function/object-detection-function/src/test/java/org/springframework/cloud/fn/object/detection/examples/SimpleExample.java b/functions/function/object-detection-function/src/test/java/org/springframework/cloud/fn/object/detection/examples/SimpleExample.java deleted file mode 100644 index 039cd8dcc..000000000 --- a/functions/function/object-detection-function/src/test/java/org/springframework/cloud/fn/object/detection/examples/SimpleExample.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.object.detection.examples; - - -import java.util.List; - -import org.springframework.cloud.fn.object.detection.ObjectDetectionService; -import org.springframework.cloud.fn.object.detection.domain.ObjectDetection; - -/** - * @author Christian Tzolov - */ -public class SimpleExample { - - public static void main(String[] args) { - // Select a pre-trained model from the model zoo: https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/detection_model_zoo.md - // Just use the notation # - String model = "https://download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_ppn_shared_box_predictor_300x300_coco14_sync_2018_07_03.tar.gz#frozen_inference_graph.pb"; - - // All labels for the pre-trained models are available at: https://github.com/tensorflow/models/tree/master/research/object_detection/data - String labels = "https://raw.githubusercontent.com/tensorflow/models/master/research/object_detection/data/mscoco_label_map.pbtxt"; - - ObjectDetectionService detectionService = new ObjectDetectionService(model, labels, - 0.4f, // Only object fromMemory confidence above the threshold are returned. Confidence range is [0, 1]. - false, // No instance segmentation - true); // cache the TF model locally - - // You can use file:, http: or classpath: to provide the path to the input image. - List detectedObjects = detectionService.detect("classpath:/images/object-detection.jpg"); - detectedObjects.stream().map(o -> o.toString()).forEach(System.out::println); - } -} diff --git a/functions/function/payload-converter-function/pom.xml b/functions/function/payload-converter-function/pom.xml deleted file mode 100644 index 8ca745b78..000000000 --- a/functions/function/payload-converter-function/pom.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - payload-converter-function - payload-converter-function - Utility message conversion functions - - diff --git a/functions/function/payload-converter-function/src/main/java/functions/ByteArrayTextToString.java b/functions/function/payload-converter-function/src/main/java/functions/ByteArrayTextToString.java deleted file mode 100644 index d087db26f..000000000 --- a/functions/function/payload-converter-function/src/main/java/functions/ByteArrayTextToString.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package functions; - -import java.util.function.Function; - -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.util.MimeTypeUtils; - -/** - * - * @author Christian Tzolov - */ -public class ByteArrayTextToString implements Function, Message> { - - @Override - public Message apply(Message message) { - - if (message.getPayload() instanceof byte[]) { - final MessageHeaders headers = message.getHeaders(); - String contentType = headers.containsKey(MessageHeaders.CONTENT_TYPE) - ? headers.get(MessageHeaders.CONTENT_TYPE).toString() - : MimeTypeUtils.APPLICATION_JSON_VALUE; - - if (contentType.contains("text") || contentType.contains("json") || contentType.contains("x-spring-tuple")) { - message = MessageBuilder.withPayload(new String(((byte[]) message.getPayload()))) - .copyHeaders(message.getHeaders()) - .build(); - } - } - - return message; - } -} - diff --git a/functions/function/payload-converter-function/src/main/java/functions/JsonBytesToMap.java b/functions/function/payload-converter-function/src/main/java/functions/JsonBytesToMap.java deleted file mode 100644 index a1bdc9df2..000000000 --- a/functions/function/payload-converter-function/src/main/java/functions/JsonBytesToMap.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package functions; - -import java.io.IOException; -import java.util.Map; -import java.util.function.Function; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import org.springframework.core.log.LogAccessor; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.util.MimeTypeUtils; - -/** - * The {@link Function} to deserialize {@code byte[]} payload into a Map - * if {@link MessageHeaders#CONTENT_TYPE} header is JSON. - * Otherwise, the message is returned as is. - * - * @author Artem Bilan - * - * @since 4.0 - */ -public class JsonBytesToMap implements Function, Message> { - - private static final LogAccessor logger = new LogAccessor(JsonBytesToMap.class); - - private final ObjectMapper objectMapper; - - public JsonBytesToMap(ObjectMapper objectMapper) { - this.objectMapper = objectMapper; - } - - @Override - public Message apply(Message message) { - if (message.getPayload() instanceof byte[] payload) { - MessageHeaders headers = message.getHeaders(); - String contentType = - headers.containsKey(MessageHeaders.CONTENT_TYPE) - ? headers.get(MessageHeaders.CONTENT_TYPE).toString() - : MimeTypeUtils.APPLICATION_JSON_VALUE; - - if (contentType.contains("json")) { - message = MessageBuilder.withPayload(payloadToMapIfCan(payload)) - .copyHeaders(message.getHeaders()) - .build(); - } - } - - return message; - } - - private Object payloadToMapIfCan(byte[] payload) { - try { - return this.objectMapper.readValue(payload, Map.class); - } - catch (IOException ex) { - logger.trace(ex, "Cannot deserialize to Map"); - // Was not able to construct the map from byte[] -- returning as is - return payload; - } - } - -} diff --git a/functions/function/payload-converter-function/src/test/java/functions/ByteArrayTextToStringTests.java b/functions/function/payload-converter-function/src/test/java/functions/ByteArrayTextToStringTests.java deleted file mode 100644 index f6492f0e0..000000000 --- a/functions/function/payload-converter-function/src/test/java/functions/ByteArrayTextToStringTests.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package functions; - -import java.util.Collections; -import java.util.function.Function; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.util.MimeTypeUtils; - -import static org.assertj.core.api.Assertions.assertThat; - -public class ByteArrayTextToStringTests { - - private static final String MESSAGE = "hello world"; - private static Function, Message> converter; - - @BeforeAll - static void before() { - converter = new ByteArrayTextToString(); - } - - @Test - public void testDefaultNoContentType() { - Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes())); - assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE); - - converted = converter.apply(new GenericMessage<>(MESSAGE)); // String - assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE); - } - - @Test - public void testApplicationJsonContentType() { - Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), - Collections.singletonMap(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON_VALUE))); - assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo("hello world"); - } - - @Test - public void testPlainTextContentType() { - Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), - Collections.singletonMap(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.TEXT_PLAIN_VALUE))); - assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE); - } - - @Test - public void testOctetContentType() { - Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), - Collections.singletonMap(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE))); - assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE.getBytes()); - } - - @Test - public void testRandomNonTextContentType() { - Message converted = converter.apply(new GenericMessage<>(MESSAGE.getBytes(), - Collections.singletonMap(MessageHeaders.CONTENT_TYPE, "Random Content Type"))); - assertThat(converted).isNotNull().extracting(Message::getPayload).isEqualTo(MESSAGE.getBytes()); - } - -} diff --git a/functions/function/pom.xml b/functions/function/pom.xml deleted file mode 100644 index fd8e67de2..000000000 --- a/functions/function/pom.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - 4.0.0 - - org.springframework.cloud.fn - java-functions-function - 5.0.0-SNAPSHOT - java-functions-function - Java Functions Function - pom - - - aggregator-function - filter-function - header-enricher-function - header-filter-function - http-request-function - spel-function - payload-converter-function - splitter-function - task-launch-request-function - twitter-function - image-recognition-function - object-detection-function - semantic-segmentation-function - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.0.0 - - true - - - - - diff --git a/functions/function/semantic-segmentation-function/README.adoc b/functions/function/semantic-segmentation-function/README.adoc deleted file mode 100644 index 6876b5a6d..000000000 --- a/functions/function/semantic-segmentation-function/README.adoc +++ /dev/null @@ -1,104 +0,0 @@ -:images-asciidoc: https://raw.githubusercontent.com/spring-cloud/stream-applications/master/functions/function/semantic-segmentation-function/src/main/resources/images/ -# Semantic Segmentation - -[.lead] -Image Semantic Segmentation based on the state-of-art https://github.com/tensorflow/models/tree/master/research/deeplab[DeepLab] Tensorflow model. - -[cols="1,2", frame=none, grid=none] -|=== -| image:{images-asciidoc}/VikiMaxiAdi-all.png[width=100%] -|Semantic Segmentation is the process of associating each pixel of an image with a class label, (such as flower, person, road, sky, ocean, or car). -Unlike the `Instance Segmentation`, which produces instance-aware region masks, the `Semantic Segmentation` produces class-aware masks. -For implementing `Instance Segmentation` consult the https://github.com/spring-cloud/stream-applications/tree/master/functions/function/object-detection-function[Object Detection Service] instead. -|=== - -The https://github.com/spring-cloud/stream-applications/blob/master/functions/common/tensorflow-common/src/main/java/org/springframework/cloud/fn/common/tensorflow/deprecated/JsonMapperFunction.java[JsonMapperFunction] permits -converting the `List` into JSON objects, and the -https://github.com/spring-cloud/stream-applications/blob/master/functions/function/object-detection-function/src/main/java/org/springframework/cloud/fn/object/detection/ObjectDetectionImageAugmenter.java[ObjectDetectionImageAugmenter] -allow to augment the input image with the detected bounding boxes and segmentation masks. - -## Usage - -Add the `semantic-segmentation` dependency to your pom (_use the latest version available_): - -[source,xml] ----- - - org.springframework.cloud.fn - semantic-segmentation-function - ${revision} - - - - org.springframework.cloud.fn - object-detection-function - ${revision} - ----- - -Following snippet demos how to use the PASCAL VOC model to apply mask to an input image - -[source,java,linenums] ----- - -SemanticSegmentation segmentationService = new SemanticSegmentation( - "https://download.tensorflow.org/models/deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz#frozen_inference_graph.pb", // <1> - true); // <2> - -byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/VikiMaxiAdi.jpg"); // <3> - -byte[] imageMask = segmentationService.masksAsImage(inputImage); // <4> -BufferedImage bi = ImageIO.read(new ByteArrayInputStream(imageMask)); -ImageIO.write(bi, "png", new FileOutputStream("./semantic-segmentation-function/target/VikiMaxiAdi_masks.png")); - -byte[] augmentedImage = segmentationService.augment(inputImage); // <5> -IOUtils.write(augmentedImage, new FileOutputStream("./semantic-segmentation-function/target/VikiMaxiAdi_augmented.jpg")); ----- -<1> Download the PASCAL 2012 trained model directly from the web. The `frozen_inference_graph.pb` is the name of the model -file inside the `tar.gz` archive. -<2> Cache the downloaded model locally -<3> Load the input image as byte array -<4> Read get the segmentation mask as separate image -<5> Blend the segmentation mask on top of the original image - -## Models - -Based on the training datasets, three groups of pre-trained models provided: - -[cols="1,2", frame=none, grid=none] -|=== -| image:{images-asciidoc}/VikiMaxiAdi-all.png[width=100%] -| https://github.com/tensorflow/models/blob/master/research/deeplab/g3doc/model_zoo.md#deeplab-models-trained-on-pascal-voc-2012[DeepLab models trained on PASCAL VOC 2012] - -| image:{images-asciidoc}/cityscape-all-small.png[width=100%] -| https://github.com/tensorflow/models/blob/master/research/deeplab/g3doc/model_zoo.md#deeplab-models-trained-on-cityscapes[DeepLab models trained on Cityscapes] - -| image:{images-asciidoc}/ADE20K-all-small.png[width=100%] -| https://github.com/tensorflow/models/blob/master/research/deeplab/g3doc/model_zoo.md#deeplab-models-trained-on-ade20k[DeepLab models trained on ADE20K] -|=== - -Select the model you want to use, copy its archive download Url and add a `#frozen_inference_graph.pb` fragment to it. -Later fragment is the frozen model's file name inside the archive - -TIP: Download the archive and uncompress the `frozen_inference_graph.pb` for required model. Then use the `file://` URI schema. - -Also, convenience there are a couple of models, extracted from the archive and uploaded to bintray: - -[cols=2*,, frame=none, grid=none] -|=== -|PASCAL VOC 2012 (default) -|https://dl.bintray.com/big-data/generic/deeplabv3_mnv2_pascal_train_aug_frozen_inference_graph.pb - -|CITYSCAPE -|https://dl.bintray.com/big-data/generic/deeplabv3_mnv2_cityscapes_train_2018_02_05_frozen_inference_graph.pb - -|ADE20K -|https://dl.bintray.com/big-data/generic/deeplabv3_xception_ade20k_train_2018_05_29_frozen_inference_graph.pb -|=== - -## References: -[.small] -* https://ai.googleblog.com/2018/03/semantic-image-segmentation-with.html[Semantic Image Segmentation with DeepLab in TensorFlow] -* https://github.com/tensorflow/models/tree/master/research/deeplab[DeepLab Project] -* https://medium.freecodecamp.org/how-to-use-deeplab-in-tensorflow-for-object-segmentation-using-deep-learning-a5777290ab6b[How to re-train DeepLab Segmentation models using Transfer Learning] - diff --git a/functions/function/semantic-segmentation-function/pom.xml b/functions/function/semantic-segmentation-function/pom.xml deleted file mode 100644 index 3cad6758f..000000000 --- a/functions/function/semantic-segmentation-function/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - semantic-segmentation-function - semantic-segmentation-function - Spring Native Function for Tensorflow semantic-segmentation integration - - - 1.3.2 - - - - - org.springframework.cloud.fn - tensorflow-common - ${project.version} - - - - - - tensorflow-snapshots - https://oss.sonatype.org/content/repositories/snapshots/ - - true - - - - - diff --git a/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/NativeImageUtils.java b/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/NativeImageUtils.java deleted file mode 100644 index b9ba1eeb9..000000000 --- a/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/NativeImageUtils.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.semantic.segmentation; - -import java.util.Arrays; - -import org.tensorflow.Operand; -import org.tensorflow.op.Ops; -import org.tensorflow.op.core.Concat; -import org.tensorflow.op.core.ExpandDims; -import org.tensorflow.op.core.Gather; -import org.tensorflow.op.core.Range; -import org.tensorflow.op.core.ReduceMax; -import org.tensorflow.op.core.Tile; -import org.tensorflow.op.math.Add; -import org.tensorflow.op.math.Mul; -import org.tensorflow.op.math.Sub; - -/** - * @author Christian Tzolov - */ -public final class NativeImageUtils { - - private NativeImageUtils() { - } - - /** - * grayscaleToRgb. - * https://github.com/tensorflow/tensorflow/blob/r1.13/tensorflow/python/ops/image_ops_impl.py#L1536 - */ - public static Operand grayscaleToRgb(Ops tf, Operand images) { - ExpandDims rank_1 = tf.expandDims( - tf.math.sub(tf.rank(images), tf.constant(1)), - tf.constant(0)); - // Create once 1D vector of the shape defined by the rank_1. - // E.g. for rank [2] will produce matrix [1, 1]. For [3] rank will produce a cube [1, 1, 1] - Add ones = tf.math.add(tf.zeros(rank_1, Integer.class), tf.constant(1)); - // Convert scalar 3 into 1D array [3] - ExpandDims channelsAs1D = tf.expandDims(tf.constant(3), tf.constant(0)); - Concat shapeList = tf.concat(Arrays.asList(ones, channelsAs1D), tf.constant(0)); - Tile tile = tf.withName("grayscaleToRgb").tile(images, shapeList); - return tile; - } - - public static Operand normalizeMask(Ops tf, Operand mask, float newValue) { - // generate array representing the axis indexes. - // For example of tensor of rank K the axisRange is {0, 1, 2 ...K} - Range axisRange = tf.range(tf.constant(0), // from - tf.dtypes.cast(tf.rank(mask), Integer.class), // to - tf.constant(1)); // step - - ReduceMax max = tf.reduceMax(mask, axisRange); - //Mul input2Float1 = tf.math.mul(tf.math.div(input2Float, max), tf.constant(1f)); - Mul normalizedMask = tf.math.mul(tf.math.div(mask, max), tf.constant(newValue)); - - return normalizedMask; - } - - /** - * Alpha Blending . - * https://en.wikipedia.org/wiki/Alpha_compositing#Alpha_blending - */ - public static Operand alphaBlending(Ops tf, Operand srcRgb, Operand dstRgb, Operand srcAlpha) { - Sub alpha = tf.math.sub(tf.onesLike(srcRgb), srcAlpha); - Mul src = tf.math.mul(srcRgb, alpha); - Mul dst = tf.math.mul(dstRgb, tf.math.sub(tf.constant(1.0f), alpha)); - Add out = tf.math.add(dst, src); - - //Mul out = tf.math.mul(srcRgbNormalized, dstRgb); - //Squeeze squeeze = tf.withName("squeeze").squeeze(out, Squeeze.axis(Arrays.asList(0L))); - - return out; - } - - /** - * The mask can contain label values larger than the list of colors provided in the color map. - * To avoid out-of-index errors we will "normalize" the label values in the mask to MOD max-color-table-value. - * @param tf - tensorflow - * @param colorTable Color map of shape [n, 3]. n is the count of label entries and 3 is the RGB color assigned - * to that label. - * @param mask Mask of shape [h, w] containing label vales. - * @return Mask of shape [h, w] fromMemory values normalized between [0, n] - */ - public static Operand normalizeMaskLabels(Ops tf, Operand colorTable, Operand mask) { - // The mask can contain label values larger than the list of colors provided in the color map. - // To avoid out-of-index errors we will "normalize" the label values in the mask to MOD max-color-table-value. - Sub colorTableShape = tf.math.sub(tf.shape(colorTable, Long.class), tf.constant(1L)); - // Color tables have shape [N, 3], where N is the count of label entries. Therefore the max label id is (N - 1). - Gather colorTableSize = tf.gather(colorTableShape, tf.constant(new int[] { 0 }), tf.constant(0)); - // Normalize the label values in the mask so they don't exceed the max value in the color map. - return tf.math.mod(mask, colorTableSize); - } -} diff --git a/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SegmentationColorMap.java b/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SegmentationColorMap.java deleted file mode 100644 index 7c42e1799..000000000 --- a/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SegmentationColorMap.java +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.semantic.segmentation; - -import java.io.InputStream; -import java.util.Arrays; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import org.springframework.core.io.DefaultResourceLoader; - -/** - * - * Visualizes the segmentation results via specified color map. - * Color maps helping to visualize the semantic segmentation results for the different datasets. - * - * Supported colormaps are: - * - ADE20K (http://groups.csail.mit.edu/vision/datasets/ADE20K/). - * - Cityscapes dataset (https://www.cityscapes-dataset.com). - * - Mapillary Vistas (https://research.mapillary.com). - * - PASCAL VOC 2012 (http://host.robots.ox.ac.uk/pascal/VOC/). - * - * Based on: https://github.com/tensorflow/models/blob/master/research/deeplab/utils/get_dataset_colormap.py - * - * @author Christian Tzolov - */ -public final class SegmentationColorMap { - - private SegmentationColorMap() { - - } - - /** MAPILLARY_COLORMAP . */ - public static final int[][] MAPILLARY_COLORMAP = new int[][] { - { 165, 42, 42 }, - { 0, 192, 0 }, - { 196, 196, 196 }, - { 190, 153, 153 }, - { 180, 165, 180 }, - { 102, 102, 156 }, - { 102, 102, 156 }, - { 128, 64, 255 }, - { 140, 140, 200 }, - { 170, 170, 170 }, - { 250, 170, 160 }, - { 96, 96, 96 }, - { 230, 150, 140 }, - { 128, 64, 128 }, - { 110, 110, 110 }, - { 244, 35, 232 }, - { 150, 100, 100 }, - { 70, 70, 70 }, - { 150, 120, 90 }, - { 220, 20, 60 }, - { 255, 0, 0 }, - { 255, 0, 0 }, - { 255, 0, 0 }, - { 200, 128, 128 }, - { 255, 255, 255 }, - { 64, 170, 64 }, - { 128, 64, 64 }, - { 70, 130, 180 }, - { 255, 255, 255 }, - { 152, 251, 152 }, - { 107, 142, 35 }, - { 0, 170, 30 }, - { 255, 255, 128 }, - { 250, 0, 30 }, - { 0, 0, 0 }, - { 220, 220, 220 }, - { 170, 170, 170 }, - { 222, 40, 40 }, - { 100, 170, 30 }, - { 40, 40, 40 }, - { 33, 33, 33 }, - { 170, 170, 170 }, - { 0, 0, 142 }, - { 170, 170, 170 }, - { 210, 170, 100 }, - { 153, 153, 153 }, - { 128, 128, 128 }, - { 0, 0, 142 }, - { 250, 170, 30 }, - { 192, 192, 192 }, - { 220, 220, 0 }, - { 180, 165, 180 }, - { 119, 11, 32 }, - { 0, 0, 142 }, - { 0, 60, 100 }, - { 0, 0, 142 }, - { 0, 0, 90 }, - { 0, 0, 230 }, - { 0, 80, 100 }, - { 128, 64, 64 }, - { 0, 0, 110 }, - { 0, 0, 70 }, - { 0, 0, 192 }, - { 32, 32, 32 }, - { 0, 0, 0 }, - { 0, 0, 0 }, - }; - - /** - * Label colormap used in ADE20K segmentation benchmark. - */ - public static final int[][] ADE20K_COLORMAP = new int[][] { - { 0, 0, 0 }, - { 120, 120, 120 }, - { 180, 120, 120 }, - { 6, 230, 230 }, - { 80, 50, 50 }, - { 4, 200, 3 }, - { 120, 120, 80 }, - { 140, 140, 140 }, - { 204, 5, 255 }, - { 230, 230, 230 }, - { 4, 250, 7 }, - { 224, 5, 255 }, - { 235, 255, 7 }, - { 150, 5, 61 }, - { 120, 120, 70 }, - { 8, 255, 51 }, - { 255, 6, 82 }, - { 143, 255, 140 }, - { 204, 255, 4 }, - { 255, 51, 7 }, - { 204, 70, 3 }, - { 0, 102, 200 }, - { 61, 230, 250 }, - { 255, 6, 51 }, - { 11, 102, 255 }, - { 255, 7, 71 }, - { 255, 9, 224 }, - { 9, 7, 230 }, - { 220, 220, 220 }, - { 255, 9, 92 }, - { 112, 9, 255 }, - { 8, 255, 214 }, - { 7, 255, 224 }, - { 255, 184, 6 }, - { 10, 255, 71 }, - { 255, 41, 10 }, - { 7, 255, 255 }, - { 224, 255, 8 }, - { 102, 8, 255 }, - { 255, 61, 6 }, - { 255, 194, 7 }, - { 255, 122, 8 }, - { 0, 255, 20 }, - { 255, 8, 41 }, - { 255, 5, 153 }, - { 6, 51, 255 }, - { 235, 12, 255 }, - { 160, 150, 20 }, - { 0, 163, 255 }, - { 140, 140, 140 }, - { 250, 10, 15 }, - { 20, 255, 0 }, - { 31, 255, 0 }, - { 255, 31, 0 }, - { 255, 224, 0 }, - { 153, 255, 0 }, - { 0, 0, 255 }, - { 255, 71, 0 }, - { 0, 235, 255 }, - { 0, 173, 255 }, - { 31, 0, 255 }, - { 11, 200, 200 }, - { 255, 82, 0 }, - { 0, 255, 245 }, - { 0, 61, 255 }, - { 0, 255, 112 }, - { 0, 255, 133 }, - { 255, 0, 0 }, - { 255, 163, 0 }, - { 255, 102, 0 }, - { 194, 255, 0 }, - { 0, 143, 255 }, - { 51, 255, 0 }, - { 0, 82, 255 }, - { 0, 255, 41 }, - { 0, 255, 173 }, - { 10, 0, 255 }, - { 173, 255, 0 }, - { 0, 255, 153 }, - { 255, 92, 0 }, - { 255, 0, 255 }, - { 255, 0, 245 }, - { 255, 0, 102 }, - { 255, 173, 0 }, - { 255, 0, 20 }, - { 255, 184, 184 }, - { 0, 31, 255 }, - { 0, 255, 61 }, - { 0, 71, 255 }, - { 255, 0, 204 }, - { 0, 255, 194 }, - { 0, 255, 82 }, - { 0, 10, 255 }, - { 0, 112, 255 }, - { 51, 0, 255 }, - { 0, 194, 255 }, - { 0, 122, 255 }, - { 0, 255, 163 }, - { 255, 153, 0 }, - { 0, 255, 10 }, - { 255, 112, 0 }, - { 143, 255, 0 }, - { 82, 0, 255 }, - { 163, 255, 0 }, - { 255, 235, 0 }, - { 8, 184, 170 }, - { 133, 0, 255 }, - { 0, 255, 92 }, - { 184, 0, 255 }, - { 255, 0, 31 }, - { 0, 184, 255 }, - { 0, 214, 255 }, - { 255, 0, 112 }, - { 92, 255, 0 }, - { 0, 224, 255 }, - { 112, 224, 255 }, - { 70, 184, 160 }, - { 163, 0, 255 }, - { 153, 0, 255 }, - { 71, 255, 0 }, - { 255, 0, 163 }, - { 255, 204, 0 }, - { 255, 0, 143 }, - { 0, 255, 235 }, - { 133, 255, 0 }, - { 255, 0, 235 }, - { 245, 0, 255 }, - { 255, 0, 122 }, - { 255, 245, 0 }, - { 10, 190, 212 }, - { 214, 255, 0 }, - { 0, 204, 255 }, - { 20, 0, 255 }, - { 255, 255, 0 }, - { 0, 153, 255 }, - { 0, 41, 255 }, - { 0, 255, 204 }, - { 41, 0, 255 }, - { 41, 255, 0 }, - { 173, 0, 255 }, - { 0, 245, 255 }, - { 71, 0, 255 }, - { 122, 0, 255 }, - { 0, 255, 184 }, - { 0, 92, 255 }, - { 184, 255, 0 }, - { 0, 133, 255 }, - { 255, 214, 0 }, - { 25, 194, 194 }, - { 102, 255, 0 }, - { 92, 0, 255 }, - }; - - /** BLACK_WHITE_COLORMAP . */ - public static int[][] BLACK_WHITE_COLORMAP = new int[][] { - { 0, 0, 0 }, - { 127, 127, 127 }, - { 255, 255, 255 }, - }; - - /** CITYMAP_COLORMAP . */ - public static final int[][] CITYMAP_COLORMAP = new int[255][3]; - - static { - - // Initialize citymap - int[][] _CITYMAP_COLORMAP = new int[][] { - { 128, 64, 128 }, - { 244, 35, 232 }, - { 70, 70, 70 }, - { 102, 102, 156 }, - { 190, 153, 153 }, - { 153, 153, 153 }, - { 250, 170, 30 }, - { 220, 220, 0 }, - { 107, 142, 35 }, - { 152, 251, 152 }, - { 70, 130, 180 }, - { 220, 20, 60 }, - { 255, 0, 0 }, - { 0, 0, 142 }, - { 0, 0, 70 }, - { 0, 60, 100 }, - { 0, 80, 100 }, - { 0, 0, 230 }, - { 119, 11, 32 } - }; - - for (int i = 0; i < _CITYMAP_COLORMAP.length; i++) { - System.arraycopy(_CITYMAP_COLORMAP[i], 0, CITYMAP_COLORMAP[i], 0, _CITYMAP_COLORMAP[i].length); - } - } - - public static int[][] loadColorMap(String resourceUri) { - try { - InputStream colorMapIs = new DefaultResourceLoader().getResource(resourceUri).getInputStream(); - ColorMap colorMap = new ObjectMapper().readValue(colorMapIs, ColorMap.class); - return colorMap.getColormap(); - } - catch (Exception exception) { - throw new RuntimeException(exception); - } - } - - public static class ColorMap { - private String name; - private String info; - private int[][] colormap; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getInfo() { - return info; - } - - public void setInfo(String info) { - this.info = info; - } - - public int[][] getColormap() { - return colormap; - } - - public void setColormap(int[][] colormap) { - this.colormap = colormap; - } - - @Override - public String toString() { - return "ColorMap{" + - "name='" + name + '\'' + - "info='" + info + '\'' + - ", colormap=" + Arrays.deepToString(colormap) + - '}'; - } - } -} diff --git a/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SemanticSegmentation.java b/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SemanticSegmentation.java deleted file mode 100644 index 351718893..000000000 --- a/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/SemanticSegmentation.java +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.semantic.segmentation; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Map; - -import javax.imageio.ImageIO; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.tensorflow.Operand; -import org.tensorflow.Tensor; -import org.tensorflow.op.Ops; -import org.tensorflow.op.core.Gather; -import org.tensorflow.op.core.Placeholder; -import org.tensorflow.op.core.Squeeze; -import org.tensorflow.op.core.ZerosLike; -import org.tensorflow.op.dtypes.Cast; -import org.tensorflow.op.image.DecodeJpeg; -import org.tensorflow.op.image.ExtractJpegShape; -import org.tensorflow.op.math.Add; -import org.tensorflow.op.math.Div; -import org.tensorflow.op.math.Equal; -import org.tensorflow.types.UInt8; - -import org.springframework.cloud.fn.common.tensorflow.Functions; -import org.springframework.cloud.fn.common.tensorflow.GraphRunner; -import org.springframework.cloud.fn.common.tensorflow.GraphRunnerMemory; -import org.springframework.cloud.fn.common.tensorflow.ProtoBufGraphDefinition; -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; -import org.springframework.core.io.DefaultResourceLoader; - -/** - * @author Christian Tzolov - */ -public class SemanticSegmentation implements AutoCloseable { - - private static final long CHANNELS = 3; - private static final float REQUIRED_INPUT_IMAGE_SIZE = 513f; - private final GraphRunner imageNormalization; - private final GraphRunner semanticSegmentation; - private final GraphRunner maskImageEncoding; - private final GraphRunner alphaBlending; - private final Tensor colorMapTensor; - private final Tensor maskTransparencyTensor; - - @Override - public void close() { - this.imageNormalization.close(); - this.semanticSegmentation.close(); - this.maskImageEncoding.close(); - this.alphaBlending.close(); - - this.colorMapTensor.close(); - this.maskTransparencyTensor.close(); - } - - public SemanticSegmentation(String modelUrl, int[][] colorMap, long[] labelFilter, float maskTransparency) { - - this.imageNormalization = new GraphRunner("input_image", "resized_image") - .withGraphDefinition(tf -> { - Placeholder input = tf.withName("input_image").placeholder(String.class); - ExtractJpegShape imageShapeAndChannel = tf.image.extractJpegShape(input); - Gather imageShape = tf.gather(imageShapeAndChannel, tf.constant(new int[] { 0, 1 }), tf.constant(0)); - - Cast maxSize = tf.dtypes.cast(tf.max(imageShape, tf.constant(0)), Float.class); - Div scale = tf.math.div(tf.constant(REQUIRED_INPUT_IMAGE_SIZE), maxSize); - Cast newSize = tf.dtypes.cast(tf.math.mul(scale, tf.dtypes.cast(imageShape, Float.class)), Integer.class); - - final Operand decodedImage = - tf.dtypes.cast(tf.image.decodeJpeg(input, DecodeJpeg.channels(CHANNELS)), Float.class); - - final Operand resizedImageFloat = - tf.image.resizeBilinear(tf.expandDims(decodedImage, tf.constant(0)), newSize); - - tf.withName("resized_image").dtypes.cast(resizedImageFloat, UInt8.class); - }); - - this.semanticSegmentation = new GraphRunner("ImageTensor:0", "SemanticPredictions:0") - .withGraphDefinition(new ProtoBufGraphDefinition(new DefaultResourceLoader().getResource(modelUrl), true)); - - this.colorMapTensor = Tensor.create(colorMap).expect(Integer.class); - - this.maskImageEncoding = new GraphRunner(Arrays.asList("color_map", "mask_pixels"), Arrays.asList("mask_png", "mask_rgb")) - .withGraphDefinition(tf -> { - Placeholder colorTable = tf.withName("color_map").placeholder(Integer.class); - - Placeholder batchedMask = tf.withName("mask_pixels").placeholder(Long.class); - // Remove batch dimension - Squeeze mask = tf.squeeze(batchedMask, Squeeze.axis(Arrays.asList(0L))); - - Operand filteredMask = labelFilter(tf, mask, labelFilter); - - // The mask can contain label values larger than the list of colors provided in the color map. - // To avoid out-of-index errors we will "normalize" the label values in the mask to MOD max-color-table-value. - Operand mask3 = NativeImageUtils.normalizeMaskLabels(tf, colorTable, filteredMask); - - Gather maskRgb = tf.withName("mask_rgb").gather(colorTable, mask3, tf.constant(0)); - - Operand png = tf.withName("mask_png").image.encodePng(tf.dtypes.cast(maskRgb, UInt8.class)); - - }); - - this.maskTransparencyTensor = Tensor.create(maskTransparency).expect(Float.class); - - this.alphaBlending = new GraphRunner( - Arrays.asList("input_image", "mask_image", "mask_transparency"), Arrays.asList("blended_png")) - .withGraphDefinition(tf -> { - // Input image [B, H, W, 3] - Cast inputImageRgb = tf.dtypes.cast(tf.withName("input_image").placeholder(UInt8.class), Float.class); - - Placeholder a = tf.withName("mask_image").placeholder(Integer.class); - Cast maskRgb = tf.dtypes.cast(a, Float.class); - - Squeeze inputImageRgb2 = tf.squeeze(inputImageRgb, Squeeze.axis(Arrays.asList(0L))); - - Placeholder maskTransparencyHolder = tf.withName("mask_transparency").placeholder(Float.class); - - // Blend the transparent maskImage on top of the input image. - Operand blended = NativeImageUtils.alphaBlending(tf, maskRgb, inputImageRgb2, maskTransparencyHolder); - - // Cut - //Operand condition = tf.math.equal(a, tf.zerosLike(a)); - //Operand blended = tf.where3(condition, tf.zerosLike(maskRgb), inputImageRgb2); - - // Encode PNG - tf.withName("blended_png").image.encodePng(tf.dtypes.cast(blended, UInt8.class)); - - }); - } - - public byte[] blendMask(byte[] image) { - try (Tensor inputTensor = Tensor.create(image); GraphRunnerMemory memory = new GraphRunnerMemory()) { - - Map> blendedTensors = - this.imageNormalization.andThen(memory) // (input_image) -> (resized_image) and memorize (resized_image) - .andThen(this.semanticSegmentation).andThen(memory) // (ImageTensor:0) -> (SemanticPredictions:0) and memorize (SemanticPredictions:0) - .andThen(Functions.rename("SemanticPredictions:0", "mask_pixels")) // (SemanticPredictions:0) -> (mask_pixels) - .andThen(Functions.enrichWith("color_map", this.colorMapTensor)) // (mask_pixels) -> (mask_pixels, color_map) - .andThen(this.maskImageEncoding).andThen(memory) // (color_map, mask_pixels) -> (mask_png, mask_rgb) and memorize (mask_png, mask_rgb) - .andThen(Functions.enrichFromMemory( - memory, "resized_image")) // (mask_png, mask_rgb) -> (mask_png, mask_rgb, resized_image), e.g. join the normalizedImageTensor - .andThen(Functions.rename( - "resized_image", "input_image", - "mask_rgb", "mask_image")) // (mask_png, mask_rgb, resized_image) -> (mask_image, input_image) - .andThen(Functions.enrichWith("mask_transparency", this.maskTransparencyTensor)) // (mask_image, input_image) -> (mask_image, input_image, mask_transparency) - .andThen(this.alphaBlending).andThen(memory) // (mask_image, input_image, mask_transparency) -> (blended_png) - .apply(Collections.singletonMap("input_image", inputTensor)); // () -> (input_image) - - byte[] blendedImage = blendedTensors.get("blended_png").bytesValue(); - - memory.getTensorMap().entrySet().stream().forEach(e -> System.out.println(e)); - - return blendedImage; - } - } - - public long[][] maskPixels(byte[] image) { - try (Tensor inputTensor = Tensor.create(image); GraphRunnerMemory memory = new GraphRunnerMemory()) { - - return this.imageNormalization.andThen(memory) // (input_image) -> (resized_image) and memorize (resized_image) - .andThen(this.semanticSegmentation).andThen(memory) // (ImageTensor:0) -> (SemanticPredictions:0) and memorize (SemanticPredictions:0) - .andThen(tensorMap -> { - Tensor maskTensor = tensorMap.get("SemanticPredictions:0"); - int width = (int) maskTensor.shape()[1]; - int height = (int) maskTensor.shape()[2]; - return maskTensor.copyTo(new long[1][width][height])[0]; // 1 == batch size - }) - .apply(Collections.singletonMap("input_image", inputTensor)); // () -> (input_image) - } - } - - public byte[] maskImage(byte[] image) { - - try (Tensor inputTensor = Tensor.create(image); GraphRunnerMemory memory = new GraphRunnerMemory()) { - - return this.imageNormalization.andThen(memory) // (input_image) -> (resized_image) and memorize (resized_image) - .andThen(this.semanticSegmentation).andThen(memory) // (ImageTensor:0) -> (SemanticPredictions:0) and memorize (SemanticPredictions:0) - .andThen(Functions.rename("SemanticPredictions:0", "mask_pixels")) // (SemanticPredictions:0) -> (mask_pixels) - .andThen(Functions.enrichWith("color_map", this.colorMapTensor)) // (mask_pixels) -> (mask_pixels, color_map) - .andThen(this.maskImageEncoding).andThen(memory) // (color_map, mask_pixels) -> (mask_png, mask_rgb) and memorize (mask_png, mask_rgb) - .andThen(tensorMap -> tensorMap.get("mask_png").bytesValue()) - .apply(Collections.singletonMap("input_image", inputTensor)); // () -> (input_image) - } - } - - private Operand labelFilter(Ops tf, Operand mask, long[] labels) { - - if (labels == null || labels.length == 0) { - return mask; - } - - ZerosLike zeroMask = tf.zerosLike(mask); - Operand result = zeroMask; - for (long label : labels) { - Add labelMask = tf.math.add(tf.zerosLike(mask), tf.constant(label)); - Equal condition = tf.math.equal(mask, labelMask); - result = tf.math.add(result, tf.where3(condition, labelMask, zeroMask)); - } - return result; - } - - public static void main(String[] args) throws IOException { - - try (SemanticSegmentation segmentationService = new SemanticSegmentation( - "https://download.tensorflow.org/models/deeplabv3_mnv2_cityscapes_train_2018_02_05.tar.gz#frozen_inference_graph.pb", - SegmentationColorMap.loadColorMap("classpath:/colormap/citymap_colormap.json"), null, 0.45f) - ) { - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/amsterdam-cityscape1.jpg"); - - // 1. Mask pixels - long[][] maskPixels = segmentationService.maskPixels(inputImage); - - String json = new ObjectMapper().writeValueAsString(maskPixels); - - // 2. Alpha Blending - byte[] blended = segmentationService.blendMask(inputImage); - ImageIO.write(ImageIO.read(new ByteArrayInputStream(blended)), "png", - new File("./functions/function/semantic-segmentation-function/target/blendedImage.png")); - - // 3. Mask Image - byte[] maskImage = segmentationService.maskImage(inputImage); - ImageIO.write(ImageIO.read(new ByteArrayInputStream(maskImage)), "png", - new File("./functions/function/semantic-segmentation-function/target/maskImage.png")); - } - - try (SemanticSegmentation segmentationService = new SemanticSegmentation( - "https://download.tensorflow.org/models/deeplabv3_xception_ade20k_train_2018_05_29.tar.gz#frozen_inference_graph.pb", - SegmentationColorMap.loadColorMap("classpath:/colormap/ade20k_colormap.json"), null, 0.45f) - ) { - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/interior.jpg"); - - // 1. Mask pixels - long[][] maskPixels = segmentationService.maskPixels(inputImage); - - // 2. Alpha Blending - byte[] blended = segmentationService.blendMask(inputImage); - ImageIO.write(ImageIO.read(new ByteArrayInputStream(blended)), "png", - new File("./functions/function/semantic-segmentation-function/target/inventory-blendedImage.png")); - - // 3. Mask Image - byte[] maskImage = segmentationService.maskImage(inputImage); - ImageIO.write(ImageIO.read(new ByteArrayInputStream(maskImage)), "png", - new File("./functions/function/semantic-segmentation-function/target/inventory-MaskImage.png")); - } - - try (SemanticSegmentation segmentationService = new SemanticSegmentation( - "https://download.tensorflow.org/models/deeplabv3_mnv2_pascal_trainval_2018_01_29.tar.gz#frozen_inference_graph.pb", - SegmentationColorMap.loadColorMap("classpath:/colormap/black_white_colormap.json"), null, 0.45f) - ) { - byte[] inputImage = GraphicsUtils.loadAsByteArray("classpath:/images/VikiMaxiAdi.jpg"); - - // 1. Mask pixels - long[][] maskPixels = segmentationService.maskPixels(inputImage); - - // 2. Alpha Blending - byte[] blended = segmentationService.blendMask(inputImage); - ImageIO.write(ImageIO.read(new ByteArrayInputStream(blended)), "png", - new File("./functions/function/semantic-segmentation-function/target/pascal-blendedImage.png")); - - // 3. Mask Image - byte[] maskImage = segmentationService.maskImage(inputImage); - ImageIO.write(ImageIO.read(new ByteArrayInputStream(maskImage)), "png", - new File("./functions/function/semantic-segmentation-function/target/pascal-MaskImage.png")); - } - - } -} diff --git a/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/attic/SemanticSegmentationUtils.java b/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/attic/SemanticSegmentationUtils.java deleted file mode 100644 index ee6c1ef6d..000000000 --- a/functions/function/semantic-segmentation-function/src/main/java/org/springframework/cloud/fn/semantic/segmentation/attic/SemanticSegmentationUtils.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.semantic.segmentation.attic; - -import java.awt.Color; -import java.awt.Graphics2D; -import java.awt.Image; -import java.awt.image.BufferedImage; -import java.awt.image.DataBufferByte; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Arrays; -import java.util.Base64; -import java.util.Collections; -import java.util.Map; - -import javax.imageio.ImageIO; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.tensorflow.Tensor; -import org.tensorflow.types.UInt8; - -import org.springframework.cloud.fn.common.tensorflow.deprecated.GraphicsUtils; -import org.springframework.cloud.fn.common.tensorflow.deprecated.TensorFlowService; -import org.springframework.core.io.DefaultResourceLoader; - -import static java.awt.image.BufferedImage.TYPE_3BYTE_BGR; - -/** - * - * Semantic image segmentation - the task of assigning a semantic label, such as “road”, “sky”, “person”, “dog”, to - * every pixel in an image. - * - * https://ai.googleblog.com/2018/03/semantic-image-segmentation-with.html - * https://github.com/tensorflow/models/blob/master/research/deeplab/g3doc/model_zoo.md - * https://github.com/tensorflow/models/tree/master/research/deeplab - * https://github.com/tensorflow/models/blob/master/research/deeplab/deeplab_demo.ipynb - * http://presentations.cocodataset.org/Places17-GMRI.pdf - * - * http://host.robots.ox.ac.uk/pascal/VOC/voc2012/index.html - * https://www.cityscapes-dataset.com/dataset-overview/#class-definitions - * http://groups.csail.mit.edu/vision/datasets/ADE20K/ - * - * https://github.com/mapillary/inplace_abn - * - * @author Christian Tzolov - */ -public class SemanticSegmentationUtils { - - /** INPUT_TENSOR_NAME . */ - public static final String INPUT_TENSOR_NAME = "ImageTensor:0"; - /** OUTPUT_TENSOR_NAME . */ - public static final String OUTPUT_TENSOR_NAME = "SemanticPredictions:0"; - - private static final int BATCH_SIZE = 1; - private static final long CHANNELS = 3; - private static final int REQUIRED_INPUT_IMAGE_SIZE = 513; - - public static BufferedImage scaledImage(String imagePath) { - try { - return scaledImage(ImageIO.read(new DefaultResourceLoader().getResource(imagePath).getInputStream())); - } - catch (IOException e) { - throw new IllegalStateException("Failed to load Image from: " + imagePath, e); - } - } - - public static BufferedImage scaledImage(byte[] image) { - try { - return scaledImage(ImageIO.read(new ByteArrayInputStream(image))); - } - catch (IOException e) { - throw new IllegalStateException("Failed to load Image from byte array", e); - } - } - - public static BufferedImage scaledImage(BufferedImage image) { - double scaleRatio = 1.0 * REQUIRED_INPUT_IMAGE_SIZE / Math.max(image.getWidth(), image.getHeight()); - return scale(image, scaleRatio); - } - - private static BufferedImage scale(BufferedImage originalImage, double scale) { - int newWidth = (int) (originalImage.getWidth() * scale); - int newHeight = (int) (originalImage.getHeight() * scale); - - Image tmpImage = originalImage.getScaledInstance(newWidth, newHeight, Image.SCALE_DEFAULT); - //BufferedImage resizedImage = new BufferedImage(newWidth, newHeight, TYPE_INT_BGR); - BufferedImage resizedImage = new BufferedImage(newWidth, newHeight, TYPE_3BYTE_BGR); - //BufferedImage resizedImage = new BufferedImage(newWidth, newHeight, originalImage.getType()); - - Graphics2D g2d = resizedImage.createGraphics(); - g2d.drawImage(tmpImage, 0, 0, null); - g2d.dispose(); - - return resizedImage; - } - - public static BufferedImage blendMask(BufferedImage mask, BufferedImage background) { - GraphicsUtils.overlayImages(background, mask, 0, 0); - return background; - } - - public static Tensor createInputTensor(BufferedImage scaledImage) { - if (scaledImage.getType() != TYPE_3BYTE_BGR) { - throw new IllegalArgumentException( - String.format("Expected 3-byte BGR encoding in BufferedImage, found %d", scaledImage.getType())); - } - - // ImageIO.read produces BGR-encoded images, while the model expects RGB. - byte[] data = bgrToRgb(toBytes(scaledImage)); - - // Expand dimensions since the model expects images to have shape: [1, None, None, 3] - long[] shape = new long[] { BATCH_SIZE, scaledImage.getHeight(), scaledImage.getWidth(), CHANNELS }; - - return Tensor.create(UInt8.class, shape, ByteBuffer.wrap(data)); - } - - private static byte[] bgrToRgb(byte[] brgImage) { - byte[] rgbImage = new byte[brgImage.length]; - for (int i = 0; i < brgImage.length; i += 3) { - rgbImage[i] = brgImage[i + 2]; - rgbImage[i + 1] = brgImage[i + 1]; - rgbImage[i + 2] = brgImage[i]; - } - return rgbImage; - } - - private static byte[] toBytes(BufferedImage bufferedImage) { - return ((DataBufferByte) bufferedImage.getRaster().getDataBuffer()).getData(); - } - - public static BufferedImage createMaskImage(int[][] maskPixels, int width, int height, double transparency) { - - maskPixels = rotate(maskPixels); - - int maskWidth = maskPixels.length; - int maskHeight = maskPixels[0].length; - - int[] maskArray = new int[maskWidth * maskHeight]; - int k = 0; - for (int i = 0; i < maskHeight; i++) { - for (int j = 0; j < maskWidth; j++) { - Color c = (maskPixels[j][i] == 0) ? Color.BLACK : GraphicsUtils.getClassColor(maskPixels[j][i]); - int t = (int) (255 * (1 - transparency)); - maskArray[k++] = new Color(c.getRed(), c.getGreen(), c.getBlue(), t).getRGB(); - } - } - - // Turn the pixel array into image; - BufferedImage maskImage = new BufferedImage(maskWidth, maskHeight, BufferedImage.TYPE_INT_ARGB); - maskImage.setRGB(0, 0, maskWidth, maskHeight, maskArray, 0, maskWidth); - - // Stretch the image to fit the target box width and height! - return GraphicsUtils.toBufferedImage(maskImage.getScaledInstance(width, height, Image.SCALE_SMOOTH)); - } - - /** - * rotate clockwise in 90 degree. - * @param input The 2D matrix to be rotated - * @return The input matrix rotated clockwise in 90 degrees - */ - private static int[][] rotate(int[][] input) { - - int w = input.length; - int h = input[0].length; - - int[][] output = new int[h][w]; - for (int y = 0; y < h; y++) { - for (int x = w - 1; x >= 0; x--) { - output[y][x] = input[x][y]; - } - } - return output; - } - - public static int[][] toIntArray(long[][] longArray) { - int[][] intArray = new int[longArray.length][longArray[0].length]; - for (int i = 0; i < longArray.length; i++) { - for (int j = 0; j < longArray[0].length; j++) { - intArray[i][j] = (int) longArray[i][j]; - } - } - return intArray; - } - - public String serializeToJson(int[][] pixels) { - String masksBase64 = Base64.getEncoder().encodeToString(toBytes(pixels)); - return String.format("{ \"columns\":%d, \"rows\":%d, \"masks\":\"%s\"}", pixels.length, pixels[0].length, masksBase64); - } - - public int[][] deserializeToMasks(String json) throws IOException { - Map map = new ObjectMapper().readValue(json, Map.class); - int cols = (int) map.get("columns"); - int rows = (int) map.get("rows"); - String masksBase64 = (String) map.get("masks"); - byte[] masks = Base64.getDecoder().decode(masksBase64); - return toInts(masks, cols, rows); - } - - private byte[] toBytes(int[][] pixels) { - byte[] b = new byte[pixels.length * pixels[0].length * 4]; - int bi = 0; - for (int i = 0; i < pixels.length; i++) { - for (int j = 0; j < pixels[0].length; j++) { - b[bi + 0] = (byte) (i >> 24); - b[bi + 1] = (byte) (i >> 16); - b[bi + 2] = (byte) (i >> 8); - b[bi + 3] = (byte) (i /*>> 0*/); - bi = bi + 4; - } - } - return b; - } - - private int[][] toInts(byte[] b, int ic, int jc) { - int[][] intResult = new int[ic][jc]; - int bi = 0; - for (int i = 0; i < ic; i++) { - for (int j = 0; j < jc; j++) { - intResult[i][j] = (b[bi] << 24) + - (b[bi + 1] << 16) + - (b[bi + 2] << 8) + - b[bi + 3]; - bi = bi + 4; - } - } - return intResult; - } - - public static void main(String[] args) throws IOException { - - // PASCAL VOC 2012 - //String tensorflowModelLocation = "file:/Users/ctzolov/Downloads/deeplabv3_mnv2_pascal_train_aug/frozen_inference_graph.pb"; - //String imagePath = "classpath:/images/VikiMaxiAdi.jpg"; - - // CITYSCAPE - //String tensorflowModelLocation = "file:/Users/ctzolov/Downloads/deeplabv3_mnv2_cityscapes_train/frozen_inference_graph.pb"; - //String imagePath = "classpath:/images/amsterdam-cityscape1.jpg"; - //String imagePath = "classpath:/images/amsterdam-channel.jpg"; - //String imagePath = "classpath:/images/landsmeer.png"; - - // ADE20K - String tensorflowModelLocation = "file:/Users/ctzolov/Downloads/deeplabv3_xception_ade20k_train/frozen_inference_graph.pb"; - String imagePath = "classpath:/images/interior.jpg"; - - BufferedImage inputImage = ImageIO.read(new DefaultResourceLoader().getResource(imagePath).getInputStream()); - - TensorFlowService tf = new TensorFlowService(new DefaultResourceLoader().getResource(tensorflowModelLocation), Arrays.asList(OUTPUT_TENSOR_NAME)); - - SemanticSegmentationUtils segmentationService = new SemanticSegmentationUtils(); - - BufferedImage scaledImage = segmentationService.scaledImage(inputImage); - - Tensor inTensor = segmentationService.createInputTensor(scaledImage); - - Map> output = tf.apply(Collections.singletonMap(INPUT_TENSOR_NAME, inTensor)); - - Tensor maskPixelsTensor = output.get(OUTPUT_TENSOR_NAME); - - int height = (int) maskPixelsTensor.shape()[1]; - int width = (int) maskPixelsTensor.shape()[2]; - long[][] maskPixels = maskPixelsTensor.copyTo(new long[BATCH_SIZE][height][width])[0]; // take 0 because the batch size is 1. - - int[][] maskPixelsInt = segmentationService.toIntArray(maskPixels); - - BufferedImage maskImage = segmentationService.createMaskImage(maskPixelsInt, scaledImage.getWidth(), scaledImage.getHeight(), 0.35); - - BufferedImage blended = segmentationService.blendMask(maskImage, scaledImage); - - ImageIO.write(maskImage, "png", new File("./semantic-segmentation/target/java2Dmask.jpg")); - ImageIO.write(blended, "png", new File("./semantic-segmentation/target/java2Dblended.jpg")); - } -} diff --git a/functions/function/semantic-segmentation-function/src/main/resources/colormap/ade20k_colormap.json b/functions/function/semantic-segmentation-function/src/main/resources/colormap/ade20k_colormap.json deleted file mode 100644 index 51648f4e2..000000000 --- a/functions/function/semantic-segmentation-function/src/main/resources/colormap/ade20k_colormap.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "name" : "ade20k", - "info" : "ADE20K (http://groups.csail.mit.edu/vision/datasets/ADE20K/)", - "colormap" :[ - [ 0, 0, 0 ], - [ 120, 120, 120 ], - [ 180, 120, 120 ], - [ 6, 230, 230 ], - [ 80, 50, 50 ], - [ 4, 200, 3 ], - [ 120, 120, 80 ], - [ 140, 140, 140 ], - [ 204, 5, 255 ], - [ 230, 230, 230 ], - [ 4, 250, 7 ], - [ 224, 5, 255 ], - [ 235, 255, 7 ], - [ 150, 5, 61 ], - [ 120, 120, 70 ], - [ 8, 255, 51 ], - [ 255, 6, 82 ], - [ 143, 255, 140 ], - [ 204, 255, 4 ], - [ 255, 51, 7 ], - [ 204, 70, 3 ], - [ 0, 102, 200 ], - [ 61, 230, 250 ], - [ 255, 6, 51 ], - [ 11, 102, 255 ], - [ 255, 7, 71 ], - [ 255, 9, 224 ], - [ 9, 7, 230 ], - [ 220, 220, 220 ], - [ 255, 9, 92 ], - [ 112, 9, 255 ], - [ 8, 255, 214 ], - [ 7, 255, 224 ], - [ 255, 184, 6 ], - [ 10, 255, 71 ], - [ 255, 41, 10 ], - [ 7, 255, 255 ], - [ 224, 255, 8 ], - [ 102, 8, 255 ], - [ 255, 61, 6 ], - [ 255, 194, 7 ], - [ 255, 122, 8 ], - [ 0, 255, 20 ], - [ 255, 8, 41 ], - [ 255, 5, 153 ], - [ 6, 51, 255 ], - [ 235, 12, 255 ], - [ 160, 150, 20 ], - [ 0, 163, 255 ], - [ 140, 140, 140 ], - [ 250, 10, 15 ], - [ 20, 255, 0 ], - [ 31, 255, 0 ], - [ 255, 31, 0 ], - [ 255, 224, 0 ], - [ 153, 255, 0 ], - [ 0, 0, 255 ], - [ 255, 71, 0 ], - [ 0, 235, 255 ], - [ 0, 173, 255 ], - [ 31, 0, 255 ], - [ 11, 200, 200 ], - [ 255, 82, 0 ], - [ 0, 255, 245 ], - [ 0, 61, 255 ], - [ 0, 255, 112 ], - [ 0, 255, 133 ], - [ 255, 0, 0 ], - [ 255, 163, 0 ], - [ 255, 102, 0 ], - [ 194, 255, 0 ], - [ 0, 143, 255 ], - [ 51, 255, 0 ], - [ 0, 82, 255 ], - [ 0, 255, 41 ], - [ 0, 255, 173 ], - [ 10, 0, 255 ], - [ 173, 255, 0 ], - [ 0, 255, 153 ], - [ 255, 92, 0 ], - [ 255, 0, 255 ], - [ 255, 0, 245 ], - [ 255, 0, 102 ], - [ 255, 173, 0 ], - [ 255, 0, 20 ], - [ 255, 184, 184 ], - [ 0, 31, 255 ], - [ 0, 255, 61 ], - [ 0, 71, 255 ], - [ 255, 0, 204 ], - [ 0, 255, 194 ], - [ 0, 255, 82 ], - [ 0, 10, 255 ], - [ 0, 112, 255 ], - [ 51, 0, 255 ], - [ 0, 194, 255 ], - [ 0, 122, 255 ], - [ 0, 255, 163 ], - [ 255, 153, 0 ], - [ 0, 255, 10 ], - [ 255, 112, 0 ], - [ 143, 255, 0 ], - [ 82, 0, 255 ], - [ 163, 255, 0 ], - [ 255, 235, 0 ], - [ 8, 184, 170 ], - [ 133, 0, 255 ], - [ 0, 255, 92 ], - [ 184, 0, 255 ], - [ 255, 0, 31 ], - [ 0, 184, 255 ], - [ 0, 214, 255 ], - [ 255, 0, 112 ], - [ 92, 255, 0 ], - [ 0, 224, 255 ], - [ 112, 224, 255 ], - [ 70, 184, 160 ], - [ 163, 0, 255 ], - [ 153, 0, 255 ], - [ 71, 255, 0 ], - [ 255, 0, 163 ], - [ 255, 204, 0 ], - [ 255, 0, 143 ], - [ 0, 255, 235 ], - [ 133, 255, 0 ], - [ 255, 0, 235 ], - [ 245, 0, 255 ], - [ 255, 0, 122 ], - [ 255, 245, 0 ], - [ 10, 190, 212 ], - [ 214, 255, 0 ], - [ 0, 204, 255 ], - [ 20, 0, 255 ], - [ 255, 255, 0 ], - [ 0, 153, 255 ], - [ 0, 41, 255 ], - [ 0, 255, 204 ], - [ 41, 0, 255 ], - [ 41, 255, 0 ], - [ 173, 0, 255 ], - [ 0, 245, 255 ], - [ 71, 0, 255 ], - [ 122, 0, 255 ], - [ 0, 255, 184 ], - [ 0, 92, 255 ], - [ 184, 255, 0 ], - [ 0, 133, 255 ], - [ 255, 214, 0 ], - [ 25, 194, 194 ], - [ 102, 255, 0 ], - [ 92, 0, 255 ]] -} diff --git a/functions/function/semantic-segmentation-function/src/main/resources/colormap/black_white_colormap.json b/functions/function/semantic-segmentation-function/src/main/resources/colormap/black_white_colormap.json deleted file mode 100644 index ca1c33eaa..000000000 --- a/functions/function/semantic-segmentation-function/src/main/resources/colormap/black_white_colormap.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name" : "black_white", - "info" : "Black and white color map", - "colormap" :[ - [ 0, 0, 0 ], - [ 127, 127, 127 ], - [ 255, 255, 255 ]] -} diff --git a/functions/function/semantic-segmentation-function/src/main/resources/colormap/citymap_colormap.json b/functions/function/semantic-segmentation-function/src/main/resources/colormap/citymap_colormap.json deleted file mode 100644 index dbd1ef2c3..000000000 --- a/functions/function/semantic-segmentation-function/src/main/resources/colormap/citymap_colormap.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name" : "citymap", - "info" : "Cityscapes dataset (https://www.cityscapes-dataset.com).", - "colormap" :[ - [ 128, 64, 128 ], - [ 244, 35, 232 ], - [ 70, 70, 70 ], - [ 102, 102, 156 ], - [ 190, 153, 153 ], - [ 153, 153, 153 ], - [ 250, 170, 30 ], - [ 220, 220, 0 ], - [ 107, 142, 35 ], - [ 152, 251, 152 ], - [ 70, 130, 180 ], - [ 220, 20, 60 ], - [ 255, 0, 0 ], - [ 0, 0, 142 ], - [ 0, 0, 70 ], - [ 0, 60, 100 ], - [ 0, 80, 100 ], - [ 0, 0, 230 ], - [ 119, 11, 32 ]] -} diff --git a/functions/function/semantic-segmentation-function/src/main/resources/colormap/mapillary_colormap.json b/functions/function/semantic-segmentation-function/src/main/resources/colormap/mapillary_colormap.json deleted file mode 100644 index f089268b2..000000000 --- a/functions/function/semantic-segmentation-function/src/main/resources/colormap/mapillary_colormap.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "name" : "mapillary", - "info" : "Mapillary Vistas (https://research.mapillary.com).", - "colormap" :[ - [ 165, 42, 42 ], - [ 0, 192, 0 ], - [ 196, 196, 196 ], - [ 190, 153, 153 ], - [ 180, 165, 180 ], - [ 102, 102, 156 ], - [ 102, 102, 156 ], - [ 128, 64, 255 ], - [ 140, 140, 200 ], - [ 170, 170, 170 ], - [ 250, 170, 160 ], - [ 96, 96, 96 ], - [ 230, 150, 140 ], - [ 128, 64, 128 ], - [ 110, 110, 110 ], - [ 244, 35, 232 ], - [ 150, 100, 100 ], - [ 70, 70, 70 ], - [ 150, 120, 90 ], - [ 220, 20, 60 ], - [ 255, 0, 0 ], - [ 255, 0, 0 ], - [ 255, 0, 0 ], - [ 200, 128, 128 ], - [ 255, 255, 255 ], - [ 64, 170, 64 ], - [ 128, 64, 64 ], - [ 70, 130, 180 ], - [ 255, 255, 255 ], - [ 152, 251, 152 ], - [ 107, 142, 35 ], - [ 0, 170, 30 ], - [ 255, 255, 128 ], - [ 250, 0, 30 ], - [ 0, 0, 0 ], - [ 220, 220, 220 ], - [ 170, 170, 170 ], - [ 222, 40, 40 ], - [ 100, 170, 30 ], - [ 40, 40, 40 ], - [ 33, 33, 33 ], - [ 170, 170, 170 ], - [ 0, 0, 142 ], - [ 170, 170, 170 ], - [ 210, 170, 100 ], - [ 153, 153, 153 ], - [ 128, 128, 128 ], - [ 0, 0, 142 ], - [ 250, 170, 30 ], - [ 192, 192, 192 ], - [ 220, 220, 0 ], - [ 180, 165, 180 ], - [ 119, 11, 32 ], - [ 0, 0, 142 ], - [ 0, 60, 100 ], - [ 0, 0, 142 ], - [ 0, 0, 90 ], - [ 0, 0, 230 ], - [ 0, 80, 100 ], - [ 128, 64, 64 ], - [ 0, 0, 110 ], - [ 0, 0, 70 ], - [ 0, 0, 192 ], - [ 32, 32, 32 ], - [ 0, 0, 0 ], - [ 0, 0, 0 ]] -} diff --git a/functions/function/semantic-segmentation-function/src/main/resources/images/ADE20K-all-small.png b/functions/function/semantic-segmentation-function/src/main/resources/images/ADE20K-all-small.png deleted file mode 100644 index bede9db42..000000000 Binary files a/functions/function/semantic-segmentation-function/src/main/resources/images/ADE20K-all-small.png and /dev/null differ diff --git a/functions/function/semantic-segmentation-function/src/main/resources/images/VikiMaxiAdi-all.png b/functions/function/semantic-segmentation-function/src/main/resources/images/VikiMaxiAdi-all.png deleted file mode 100644 index 719814e1f..000000000 Binary files a/functions/function/semantic-segmentation-function/src/main/resources/images/VikiMaxiAdi-all.png and /dev/null differ diff --git a/functions/function/semantic-segmentation-function/src/main/resources/images/VikiMaxiAdi.jpg b/functions/function/semantic-segmentation-function/src/main/resources/images/VikiMaxiAdi.jpg deleted file mode 100644 index 5d901702f..000000000 Binary files a/functions/function/semantic-segmentation-function/src/main/resources/images/VikiMaxiAdi.jpg and /dev/null differ diff --git a/functions/function/semantic-segmentation-function/src/main/resources/images/amsterdam-cityscape1.jpg b/functions/function/semantic-segmentation-function/src/main/resources/images/amsterdam-cityscape1.jpg deleted file mode 100644 index d77cae406..000000000 Binary files a/functions/function/semantic-segmentation-function/src/main/resources/images/amsterdam-cityscape1.jpg and /dev/null differ diff --git a/functions/function/semantic-segmentation-function/src/main/resources/images/cityscape-all-small.png b/functions/function/semantic-segmentation-function/src/main/resources/images/cityscape-all-small.png deleted file mode 100644 index 050a9cbe6..000000000 Binary files a/functions/function/semantic-segmentation-function/src/main/resources/images/cityscape-all-small.png and /dev/null differ diff --git a/functions/function/semantic-segmentation-function/src/main/resources/images/giant_panda_in_beijing_zoo_1.jpg b/functions/function/semantic-segmentation-function/src/main/resources/images/giant_panda_in_beijing_zoo_1.jpg deleted file mode 100644 index e7a44cae2..000000000 Binary files a/functions/function/semantic-segmentation-function/src/main/resources/images/giant_panda_in_beijing_zoo_1.jpg and /dev/null differ diff --git a/functions/function/semantic-segmentation-function/src/main/resources/images/interior.jpg b/functions/function/semantic-segmentation-function/src/main/resources/images/interior.jpg deleted file mode 100644 index f55160155..000000000 Binary files a/functions/function/semantic-segmentation-function/src/main/resources/images/interior.jpg and /dev/null differ diff --git a/functions/function/spel-function/README.adoc b/functions/function/spel-function/README.adoc deleted file mode 100644 index 0545d1d5a..000000000 --- a/functions/function/spel-function/README.adoc +++ /dev/null @@ -1,28 +0,0 @@ -# SpEL Function - -This module provides a SpEL function that can be reused and composed in other applications. -The function can be used to apply SpEL transformations on data based on a SpEL expression. - -## Beans for injection - -You can import the `SpelFunctionConfiguration` in a Spring Boot application and then inject the following bean. - -`spelFunction` - -You can use `spelFunction` as a qualifier when injecting. - -Once injected, you can use the `apply` method of the `Function` to invoke it and get the result. - -## Configuration Options - -All configuration properties are prefixed with `spel`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/spel/SpelFunctionProperties.java[SpelFunctionProperties.java] - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/spel/SpelFunctionApplicationTests.java[test suite] for examples of how this function is used. - -## Other usage - -See this link:../../../applications/processor/transform-processor/README.adoc[README] where this function is used to create a Spring Cloud Stream transformer application. \ No newline at end of file diff --git a/functions/function/spel-function/pom.xml b/functions/function/spel-function/pom.xml deleted file mode 100644 index 8bba8c28b..000000000 --- a/functions/function/spel-function/pom.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - spel-function - spel-function - Spring Native Function for applying SpEL expressions - - - - org.springframework.cloud.fn - payload-converter-function - ${project.version} - - - - diff --git a/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionConfiguration.java b/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionConfiguration.java deleted file mode 100644 index c3279d134..000000000 --- a/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionConfiguration.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.spel; - -import java.util.function.Function; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.integration.transformer.ExpressionEvaluatingTransformer; -import org.springframework.messaging.Message; - -@Configuration -@EnableConfigurationProperties(SpelFunctionProperties.class) -public class SpelFunctionConfiguration { - - @Bean - public Function, Message> spelFunction( - ExpressionEvaluatingTransformer expressionEvaluatingTransformer) { - - return message -> expressionEvaluatingTransformer.transform(message); - } - - @Bean - public ExpressionEvaluatingTransformer expressionEvaluatingTransformer( - SpelFunctionProperties spelFunctionProperties) { - - return new ExpressionEvaluatingTransformer(new SpelExpressionParser() - .parseExpression(spelFunctionProperties.getExpression())); - } - -} diff --git a/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionProperties.java b/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionProperties.java deleted file mode 100644 index 36ba3da9c..000000000 --- a/functions/function/spel-function/src/main/java/org/springframework/cloud/fn/spel/SpelFunctionProperties.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.spel; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.standard.SpelExpressionParser; - -/** - * Configuration properties for the SpEL function. - * - * @author Gary Russell - * @author Artem Bilan - */ -@ConfigurationProperties("spel.function") -public class SpelFunctionProperties { - - private static final Expression DEFAULT_EXPRESSION = new SpelExpressionParser().parseExpression("payload"); - - /** - * A SpEL expression to apply. - */ - private String expression = DEFAULT_EXPRESSION.getExpressionString(); - - public String getExpression() { - return this.expression; - } - - public void setExpression(String expression) { - this.expression = expression; - } - -} diff --git a/functions/function/spel-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/function/spel-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index 6cd2fda37..000000000 --- a/functions/function/spel-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.spel.SpelFunctionConfiguration diff --git a/functions/function/spel-function/src/test/java/org/springframework/cloud/fn/spel/SpelFunctionApplicationTests.java b/functions/function/spel-function/src/test/java/org/springframework/cloud/fn/spel/SpelFunctionApplicationTests.java deleted file mode 100644 index e3e00851d..000000000 --- a/functions/function/spel-function/src/test/java/org/springframework/cloud/fn/spel/SpelFunctionApplicationTests.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.spel; - -import java.util.function.Function; - -import org.junit.jupiter.api.Test; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.util.MimeTypeUtils; - -import static org.assertj.core.api.Assertions.assertThat; - -@SpringBootTest(properties = "spel.function.expression=payload.toUpperCase()") -@DirtiesContext -public class SpelFunctionApplicationTests { - - @Autowired - Function, Message> transformer; - - @Test - public void testTransform() { - final Message transformed = this.transformer.apply(new GenericMessage<>("hello,world")); - assertThat(transformed.getPayload()).isEqualTo("HELLO,WORLD"); - } - - @Test - public void testJson() { - Message message = MessageBuilder.withPayload("{\"foo\":\"bar\"}") - .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON).build(); - final Message transformed = this.transformer.apply(message); - assertThat(transformed.getPayload()).isEqualTo("{\"FOO\":\"BAR\"}"); - } - - @SpringBootApplication - static class SpelFunctionTestApplication { - - } -} diff --git a/functions/function/splitter-function/README.adoc b/functions/function/splitter-function/README.adoc deleted file mode 100644 index 7c28989d6..000000000 --- a/functions/function/splitter-function/README.adoc +++ /dev/null @@ -1,25 +0,0 @@ -# Splitter Function - -This module provides a header enricher function that can be reused and composed in other applications. - -## Beans for injection - -You can import the `SpliiterFunctionConfiguration` in a Spring Boot application and then inject the following bean. - -`splitterFunction` - -You can use `splitterFunction` as a qualifier when injecting. - -Once injected, you can use the `apply` method of the `Function` to invoke it and get the result. - -## Configuration Options - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionProperties.java[SplitterFunctionProperties.java] - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/splitter/SplitterFunctionApplicationTests.java[test suite] for examples of how this function is used. - -## Other usage - -See this link:../../../applications/processor/splitter-processor/README.adoc[README] where this function is used to create a Spring Cloud Stream application. \ No newline at end of file diff --git a/functions/function/splitter-function/pom.xml b/functions/function/splitter-function/pom.xml deleted file mode 100644 index e752c3f7e..000000000 --- a/functions/function/splitter-function/pom.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - splitter-function - splitter-function - Spring Native Function for Splitter - - - - org.springframework.integration - spring-integration-file - - - - diff --git a/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionConfiguration.java b/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionConfiguration.java deleted file mode 100644 index f8adff49d..000000000 --- a/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionConfiguration.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2011-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.splitter; - -import java.nio.charset.Charset; -import java.util.List; -import java.util.function.Function; - -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; - -import org.springframework.boot.autoconfigure.condition.AnyNestedCondition; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Conditional; -import org.springframework.context.annotation.Configuration; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.integration.channel.ReactiveStreamsSubscribableChannel; -import org.springframework.integration.file.splitter.FileSplitter; -import org.springframework.integration.splitter.AbstractMessageSplitter; -import org.springframework.integration.splitter.DefaultMessageSplitter; -import org.springframework.integration.splitter.ExpressionEvaluatingSplitter; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageChannel; - -@Configuration -@EnableConfigurationProperties(SplitterFunctionProperties.class) -public class SplitterFunctionConfiguration { - - @Bean - public Function, List>> splitterFunction(AbstractMessageSplitter messageSplitter, - SplitterFunctionProperties splitterFunctionProperties) { - - messageSplitter.setApplySequence(splitterFunctionProperties.isApplySequence()); - ThreadLocalFluxSinkMessageChannel outputChannel = new ThreadLocalFluxSinkMessageChannel(); - messageSplitter.setOutputChannel(outputChannel); - return message -> { - messageSplitter.handleMessage(message); - return outputChannel.publisherThreadLocal.get(); - }; - } - - @Bean - @ConditionalOnProperty(prefix = "splitter", name = "expression") - public AbstractMessageSplitter expressionSplitter(SplitterFunctionProperties splitterFunctionProperties) { - return new ExpressionEvaluatingSplitter( - new SpelExpressionParser() - .parseExpression(splitterFunctionProperties.getExpression())); - } - - @Bean - @ConditionalOnMissingBean - @Conditional(FileSplitterCondition.class) - public AbstractMessageSplitter fileSplitter(SplitterFunctionProperties splitterFunctionProperties) { - Boolean markers = splitterFunctionProperties.getFileMarkers(); - String charset = splitterFunctionProperties.getCharset(); - if (markers == null) { - markers = false; - } - FileSplitter fileSplitter = new FileSplitter(true, markers, splitterFunctionProperties.getMarkersJson()); - if (charset != null) { - fileSplitter.setCharset(Charset.forName(charset)); - } - return fileSplitter; - } - - @Bean - @ConditionalOnMissingBean - public AbstractMessageSplitter defaultSplitter(SplitterFunctionProperties splitterFunctionProperties) { - DefaultMessageSplitter defaultMessageSplitter = new DefaultMessageSplitter(); - defaultMessageSplitter.setDelimiters(splitterFunctionProperties.getDelimiters()); - return defaultMessageSplitter; - } - - static class FileSplitterCondition extends AnyNestedCondition { - - FileSplitterCondition() { - super(ConfigurationPhase.REGISTER_BEAN); - } - - @ConditionalOnProperty(prefix = "splitter", name = "charset") - static class Charset { - } - - @ConditionalOnProperty(prefix = "splitter", name = "fileMarkers") - static class FileMarkers { - } - - } - - private static final class ThreadLocalFluxSinkMessageChannel - implements MessageChannel, ReactiveStreamsSubscribableChannel { - - private final ThreadLocal>> publisherThreadLocal = new ThreadLocal<>(); - - @Override - @SuppressWarnings("unchecked") - public void subscribeTo(Publisher> publisher) { - this.publisherThreadLocal.set(Flux.from(publisher).collectList().cast(List.class).block()); - } - - @Override - public boolean send(Message message, long l) { - throw new UnsupportedOperationException("This channel only supports a reactive 'subscribeTo()' "); - } - - } - -} diff --git a/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionProperties.java b/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionProperties.java deleted file mode 100644 index eef233b0b..000000000 --- a/functions/function/splitter-function/src/main/java/org/springframework/cloud/fn/splitter/SplitterFunctionProperties.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright 2019-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.splitter; - -import jakarta.validation.constraints.AssertTrue; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * Configuration properties for the Splitter Processor app. - * - * @author Gary Russell - * @author Artem Bilan - */ -@ConfigurationProperties("splitter") -@Validated -public class SplitterFunctionProperties { - - /** - * A SpEL expression for splitting payloads. - */ - private String expression; - - /** - * When expression is null, delimiters to use when tokenizing - * {@link String} payloads. - */ - private String delimiters; - - /** - * Set to true or false to use a {@code FileSplitter} (to split - * text-based files by line) that includes - * (or not) beginning/end of file markers. - */ - private Boolean fileMarkers; - - /** - * When 'fileMarkers == true', specify if they should be produced - * as FileSplitter.FileMarker objects or JSON. - */ - private boolean markersJson = true; - - /** - * The charset to use when converting bytes in text-based files - * to String. - */ - private String charset; - - /** - * Add correlation/sequence information in headers to facilitate later - * aggregation. - */ - private boolean applySequence = true; - - public String getExpression() { - return this.expression; - } - - public void setExpression(String expression) { - this.expression = expression; - } - - public String getDelimiters() { - return this.delimiters; - } - - public void setDelimiters(String delimiters) { - this.delimiters = delimiters; - } - - public Boolean getFileMarkers() { - return this.fileMarkers; - } - - public void setFileMarkers(Boolean fileMarkers) { - this.fileMarkers = fileMarkers; - } - - public boolean getMarkersJson() { - return this.markersJson; - } - - public void setMarkersJson(boolean markersJson) { - this.markersJson = markersJson; - } - - public String getCharset() { - return this.charset; - } - - public void setCharset(String charset) { - this.charset = charset; - } - - public boolean isApplySequence() { - return this.applySequence; - } - - public void setApplySequence(boolean applySequence) { - this.applySequence = applySequence; - } - - @AssertTrue(message = "'delimiters' is not allowed when an 'expression' is provided") - public boolean isDelimitersAllowed() { - return this.expression == null || this.delimiters == null; - } - - @AssertTrue(message = "File properties are not allowed when an 'expression' or 'delimiters' property is provided") - public boolean isFilePropsAllowed() { - return !(this.expression != null || this.delimiters != null) || this.fileMarkers == null && this.charset == null; - } - -} diff --git a/functions/function/splitter-function/src/test/java/org/springframework/cloud/fn/splitter/SplitterFunctionApplicationTests.java b/functions/function/splitter-function/src/test/java/org/springframework/cloud/fn/splitter/SplitterFunctionApplicationTests.java deleted file mode 100644 index 5c3a1baf1..000000000 --- a/functions/function/splitter-function/src/test/java/org/springframework/cloud/fn/splitter/SplitterFunctionApplicationTests.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2011-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.splitter; - -import java.util.List; -import java.util.function.Function; - -import org.junit.jupiter.api.Test; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - -@SpringBootTest(properties = "splitter.expression=payload.split(',')") -@DirtiesContext -public class SplitterFunctionApplicationTests { - - @Autowired - Function, List>> splitter; - - @Test - public void testExpressionSplitter() { - List> messageList = this.splitter.apply(new GenericMessage<>("hello,world")); - assertThat(messageList).extracting(m -> m.getPayload().toString()).contains("hello", "world"); - } - - @SpringBootApplication - static class SplitterFunctionTestApplication { - } -} diff --git a/functions/function/task-launch-request-function/README.adoc b/functions/function/task-launch-request-function/README.adoc deleted file mode 100644 index c3e8b65e0..000000000 --- a/functions/function/task-launch-request-function/README.adoc +++ /dev/null @@ -1,22 +0,0 @@ -# Task Launch Request Function - -This module provides a function that can be reused and composed in other applications to transform the output to a link:src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequest.java[TaskLaunchRequest] -that can be used as input to the Tasklauncher function to launch a task. - -## Beans for injection - -You can import the `TaskLaunchRequestFunctionConfiguration` in a Spring Boot application and then inject the following bean. - -`taskLaunchRequestFunction` as a link:src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunction.java[TaskLaunchRequestFunction]. - -You can use `taskLaunchRequestFunction` as a qualifier when injecting. - -Once injected, you can use the `apply` method of the `Function` to invoke it and get the result. - -## Configuration Options - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionProperties.java[TaskLaunchRequestFunctionProperties.java] - -## Examples - -See this link:src/test/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionApplicationTests.java[test suite] for examples of how this function is used. diff --git a/functions/function/task-launch-request-function/pom.xml b/functions/function/task-launch-request-function/pom.xml deleted file mode 100644 index fef080f3e..000000000 --- a/functions/function/task-launch-request-function/pom.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - task-launch-request-function - task-launch-request-function - - diff --git a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/CommandLineArgumentsMessageMapper.java b/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/CommandLineArgumentsMessageMapper.java deleted file mode 100644 index 8cac1b0f4..000000000 --- a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/CommandLineArgumentsMessageMapper.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import java.util.Collection; - -import org.springframework.integration.handler.MessageProcessor; - -public interface CommandLineArgumentsMessageMapper extends MessageProcessor> { -} diff --git a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/KeyValueListParser.java b/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/KeyValueListParser.java deleted file mode 100644 index 48f01f0e5..000000000 --- a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/KeyValueListParser.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; - -import org.springframework.util.StringUtils; - -/** - * Parses a comma delimited list of key value pairs in which the values can contain commas as well. - * - * @author Chris Schaeffer - * @author David Turanski - **/ -abstract class KeyValueListParser { - - static Map parseCommaDelimitedKeyValuePairs(String value) { - Map keyValuePairs = new HashMap<>(); - - if (StringUtils.isEmpty(value)) { - return keyValuePairs; - } - - ArrayList pairs = new ArrayList<>(); - - String[] candidates = StringUtils.commaDelimitedListToStringArray(value); - - for (int i = 0; i < candidates.length; i++) { - if (i > 0 && !candidates[i].contains("=")) { - pairs.add(pairs.get(pairs.size() - 1) + "," + candidates[i]); - } - else { - pairs.add(candidates[i]); - } - } - - for (String pair : pairs) { - addKeyValuePair(pair, keyValuePairs); - } - - return keyValuePairs; - } - - private static void addKeyValuePair(String pair, Map properties) { - int firstEquals = pair.indexOf('='); - if (firstEquals != -1) { - properties.put(pair.substring(0, firstEquals).trim(), pair.substring(firstEquals + 1).trim()); - } - } -} diff --git a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequest.java b/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequest.java deleted file mode 100644 index 3644499b1..000000000 --- a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class TaskLaunchRequest { - @JsonProperty("args") - private List commandlineArguments = new ArrayList<>(); - - @JsonProperty("deploymentProps") - private Map deploymentProperties = new HashMap<>(); - - @JsonProperty("name") - private String taskName; - - public void setCommandlineArguments(List commandlineArguments) { - this.commandlineArguments = new ArrayList<>(commandlineArguments); - } - - public List getCommandlineArguments() { - return this.commandlineArguments; - } - - public void setDeploymentProperties(Map deploymentProperties) { - this.deploymentProperties = deploymentProperties; - } - - public Map getDeploymentProperties() { - return this.deploymentProperties; - } - - public void setTaskName(String taskName) { - this.taskName = taskName; - } - - public String getTaskName() { - return this.taskName; - } - - public TaskLaunchRequest addCommmandLineArguments(Collection args) { - this.commandlineArguments.addAll(args); - return this; - } -} diff --git a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunction.java b/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunction.java deleted file mode 100644 index 8497b3958..000000000 --- a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunction.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import java.util.function.Function; - -import org.springframework.messaging.Message; - -/** - * A marker interface useful for unambiguous dependency injection of this Function. - * - * @author David Turanski - **/ -@FunctionalInterface -public interface TaskLaunchRequestFunction extends Function, Message> { - -} diff --git a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionConfiguration.java b/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionConfiguration.java deleted file mode 100644 index 475c14022..000000000 --- a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionConfiguration.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.expression.EvaluationContext; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.integration.expression.ExpressionUtils; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.util.StringUtils; - -/** - * Configuration for a {@link TaskLaunchRequestFunction}, provided as a common function that can be composed with other Suppliers or - * Functions to transform any {@link Message} to a {@link TaskLaunchRequest} which may be used as input to the {@code TaskLauncherFunction} to launch a task. - * - * Command line arguments used by the task, as well as the task name itself may be statically configured or extracted from - * the message contents, using SpEL. See {@link TaskLaunchRequestFunctionProperties} for details. - * - * It is also possible to provide your own implementations of {@link CommandLineArgumentsMessageMapper} and {@link TaskNameMessageMapper}. - * - * @author David Turanski - **/ -@AutoConfiguration -@EnableConfigurationProperties(TaskLaunchRequestFunctionProperties.class) -public class TaskLaunchRequestFunctionConfiguration { - - /** - * The function name. - */ - public final static String TASK_LAUNCH_REQUEST_FUNCTION_NAME = "taskLaunchRequestFunction"; - - /** - * A {@link java.util.function.Function} to transform a {@link Message} payload to a - * {@link TaskLaunchRequest}. - * - * @param taskLaunchRequestMessageProcessor a {@link TaskLaunchRequestMessageProcessor}. - * - * @return a {@code TaskLaunchRequest} Message. - */ - @Bean(name = TASK_LAUNCH_REQUEST_FUNCTION_NAME) - public TaskLaunchRequestFunction taskLaunchRequest( - TaskLaunchRequestMessageProcessor taskLaunchRequestMessageProcessor) { - return message -> taskLaunchRequestMessageProcessor.postProcessMessage(message); - } - - @Bean - public TaskLaunchRequestSupplier taskLaunchRequestInitializer( - TaskLaunchRequestFunctionProperties taskLaunchRequestProperties) { - return new TaskLaunchRequestPropertiesInitializer(taskLaunchRequestProperties); - } - - @SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") - @Bean - public TaskLaunchRequestMessageProcessor taskLaunchRequestMessageProcessor( - TaskLaunchRequestSupplier taskLaunchRequestInitializer, - TaskLaunchRequestFunctionProperties properties, - EvaluationContext evaluationContext, - @Nullable TaskNameMessageMapper taskNameMessageMapper, - @Nullable CommandLineArgumentsMessageMapper commandLineArgumentsMessageMapper) { - - if (taskNameMessageMapper == null) { - taskNameMessageMapper = taskNameMessageMapper(properties, evaluationContext); - } - - if (commandLineArgumentsMessageMapper == null) { - commandLineArgumentsMessageMapper = commandLineArgumentsMessageMapper(properties, evaluationContext); - } - - return new TaskLaunchRequestMessageProcessor(taskLaunchRequestInitializer, - taskNameMessageMapper, - commandLineArgumentsMessageMapper); - } - - @Bean - public EvaluationContext evaluationContext(BeanFactory beanFactory) { - return ExpressionUtils.createStandardEvaluationContext(beanFactory); - } - - private TaskNameMessageMapper taskNameMessageMapper(TaskLaunchRequestFunctionProperties taskLaunchRequestProperties, - EvaluationContext evaluationContext) { - if (StringUtils.hasText(taskLaunchRequestProperties.getTaskNameExpression())) { - SpelExpressionParser expressionParser = new SpelExpressionParser(); - Expression taskNameExpression = expressionParser - .parseExpression(taskLaunchRequestProperties.getTaskNameExpression()); - return new ExpressionEvaluatingTaskNameMessageMapper(taskNameExpression, evaluationContext); - } - - return message -> taskLaunchRequestProperties.getTaskName(); - } - - private CommandLineArgumentsMessageMapper commandLineArgumentsMessageMapper( - TaskLaunchRequestFunctionProperties taskLaunchRequestFunctionProperties, - EvaluationContext evaluationContext) { - return new ExpressionEvaluatingCommandLineArgsMapper(taskLaunchRequestFunctionProperties.getArgExpressions(), - evaluationContext); - } - - private static class TaskLaunchRequestPropertiesInitializer extends TaskLaunchRequestSupplier { - TaskLaunchRequestPropertiesInitializer( - TaskLaunchRequestFunctionProperties taskLaunchRequestProperties) { - - this.commandLineArgumentSupplier( - () -> new ArrayList<>(taskLaunchRequestProperties.getArgs())); - - this.deploymentPropertiesSupplier( - () -> KeyValueListParser.parseCommaDelimitedKeyValuePairs( - taskLaunchRequestProperties.getDeploymentProperties())); - - this.taskNameSupplier(() -> taskLaunchRequestProperties.getTaskName()); - } - } - - private static class ExpressionEvaluatingTaskNameMessageMapper implements TaskNameMessageMapper { - - private final Expression expression; - private final EvaluationContext evaluationContext; - - ExpressionEvaluatingTaskNameMessageMapper(Expression expression, EvaluationContext evaluationContext) { - this.evaluationContext = evaluationContext; - this.expression = expression; - } - - @Override - public String processMessage(Message message) { - return expression.getValue(evaluationContext, message).toString(); - } - } - - private static class ExpressionEvaluatingCommandLineArgsMapper implements CommandLineArgumentsMessageMapper { - private final Map argExpressionsMap; - - private final EvaluationContext evaluationContext; - - ExpressionEvaluatingCommandLineArgsMapper(String argExpressions, EvaluationContext evaluationContext) { - this.evaluationContext = evaluationContext; - this.argExpressionsMap = new HashMap<>(); - if (StringUtils.hasText(argExpressions)) { - SpelExpressionParser expressionParser = new SpelExpressionParser(); - - KeyValueListParser.parseCommaDelimitedKeyValuePairs(argExpressions).forEach( - (k, v) -> argExpressionsMap.put(k, expressionParser.parseExpression(v))); - } - } - - @Override - public Collection processMessage(Message message) { - return evaluateArgExpressions(message); - } - - private Collection evaluateArgExpressions(Message message) { - List results = new LinkedList<>(); - this.argExpressionsMap.forEach((k, expression) -> results - .add(String.format("%s=%s", k, expression.getValue(this.evaluationContext, message)))); - return results; - } - } - -} diff --git a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionProperties.java b/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionProperties.java deleted file mode 100644 index 8ac70a270..000000000 --- a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionProperties.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import java.util.ArrayList; -import java.util.List; - -import jakarta.validation.constraints.AssertFalse; -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.util.StringUtils; -import org.springframework.validation.annotation.Validated; - -/** - * Base Properties to create a {@link TaskLaunchRequest}. - * - * @author Chris Schaefer - * @author David Turanski - */ -@Validated -@ConfigurationProperties("task.launch.request") -public class TaskLaunchRequestFunctionProperties { - - /** - * Comma separated list of optional args in key=value format. - */ - private List args = new ArrayList<>(); - - /** - * Comma separated list of option args as SpEL expressions in key=value format. - */ - private String argExpressions = ""; - - /** - * Comma delimited list of deployment properties to be applied to the - * TaskLaunchRequest. - */ - private String deploymentProperties = ""; - - /** - * The Data Flow task name. - */ - private String taskName; - - - /** - * A SpEL expression to extract the task name from each Message, using the Message as the evaluation context. - */ - private String taskNameExpression; - - @NotNull - public List getArgs() { - return this.args; - } - - public void setArgs(List args) { - this.args = new ArrayList<>(args); - } - - @NotNull - public String getDeploymentProperties() { - return this.deploymentProperties; - } - - public void setDeploymentProperties(String deploymentProperties) { - this.deploymentProperties = deploymentProperties; - } - - public String getTaskName() { - return taskName; - } - - public void setTaskName(String taskName) { - this.taskName = taskName; - } - - public String getTaskNameExpression() { - return taskNameExpression; - } - - public void setTaskNameExpression(String taskNameExpression) { - this.taskNameExpression = taskNameExpression; - } - - public String getArgExpressions() { - return argExpressions; - } - - public void setArgExpressions(String argExpressions) { - this.argExpressions = argExpressions; - } - - @AssertFalse(message = "Cannot specify both 'taskName' and 'taskNameExpression'.") - public boolean isTaskNameAndTaskNameExpressionSet() { - return StringUtils.hasText(this.taskName) && StringUtils.hasText(this.taskNameExpression); - } - -} diff --git a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestMessageProcessor.java b/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestMessageProcessor.java deleted file mode 100644 index 1bbea1902..000000000 --- a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestMessageProcessor.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.core.MessagePostProcessor; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.util.Assert; -import org.springframework.util.MimeTypeUtils; -import org.springframework.util.StringUtils; - -class TaskLaunchRequestMessageProcessor implements MessagePostProcessor { - - private final TaskNameMessageMapper taskNameMessageMapper; - - private final CommandLineArgumentsMessageMapper commandLineArgumentsMessageMapper; - - private final TaskLaunchRequestSupplier taskLaunchRequestInitializer; - - TaskLaunchRequestMessageProcessor(TaskLaunchRequestSupplier taskLaunchRequestInitializer, - TaskNameMessageMapper taskNameMessageMapper, - CommandLineArgumentsMessageMapper commandLIneArgumentsMessageMapper) { - - this.taskLaunchRequestInitializer = taskLaunchRequestInitializer; - - this.taskNameMessageMapper = taskNameMessageMapper; - - this.commandLineArgumentsMessageMapper = commandLIneArgumentsMessageMapper; - - } - - @Override - public Message postProcessMessage(Message message) { - TaskLaunchRequest taskLaunchRequest = taskLaunchRequestInitializer.get(); - - if (!StringUtils.hasText(taskLaunchRequest.getTaskName())) { - taskLaunchRequest.setTaskName(taskNameMessageMapper.processMessage(message)); - Assert.hasText(taskLaunchRequest.getTaskName(), - () -> "'taskName' is required in " + TaskLaunchRequest.class.getName()); - } - - taskLaunchRequest.addCommmandLineArguments(commandLineArgumentsMessageMapper.processMessage(message)); - - MessageBuilder builder = MessageBuilder.withPayload(taskLaunchRequest) - .copyHeaders(message.getHeaders()); - return adjustHeaders(builder).build(); - } - - private MessageBuilder adjustHeaders(MessageBuilder builder) { - builder.setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON); - return builder; - } -} diff --git a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestSupplier.java b/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestSupplier.java deleted file mode 100644 index 1d6bb373e..000000000 --- a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestSupplier.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; - -import org.springframework.util.Assert; - -class TaskLaunchRequestSupplier implements Supplier { - - private Supplier taskNameSupplier; - - private Supplier> commandLineArgumentsSupplier; - - private Supplier> deploymentPropertiesSupplier; - - public TaskLaunchRequestSupplier taskNameSupplier(Supplier taskNameSupplier) { - this.taskNameSupplier = taskNameSupplier; - return this; - } - - public TaskLaunchRequestSupplier commandLineArgumentSupplier(Supplier> commandLineArgumentsSupplier) { - this.commandLineArgumentsSupplier = commandLineArgumentsSupplier; - return this; - } - - public TaskLaunchRequestSupplier deploymentPropertiesSupplier( - Supplier> deploymentPropertiesSupplier) { - this.deploymentPropertiesSupplier = deploymentPropertiesSupplier; - return this; - } - - @Override - public TaskLaunchRequest get() { - - Assert.notNull(this.taskNameSupplier, "'taskNameSupplier' is required."); - - TaskLaunchRequest taskLaunchRequest = new TaskLaunchRequest(); - taskLaunchRequest.setTaskName(this.taskNameSupplier.get()); - - if (this.commandLineArgumentsSupplier != null) { - taskLaunchRequest.setCommandlineArguments(this.commandLineArgumentsSupplier.get()); - } - - if (this.deploymentPropertiesSupplier != null) { - taskLaunchRequest.setDeploymentProperties(this.deploymentPropertiesSupplier.get()); - } - - return taskLaunchRequest; - } -} diff --git a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskNameMessageMapper.java b/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskNameMessageMapper.java deleted file mode 100644 index 09eeb9383..000000000 --- a/functions/function/task-launch-request-function/src/main/java/org/springframework/cloud/fn/task/launch/request/TaskNameMessageMapper.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import org.springframework.integration.handler.MessageProcessor; - -@FunctionalInterface -public interface TaskNameMessageMapper extends MessageProcessor { -} diff --git a/functions/function/task-launch-request-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/function/task-launch-request-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index a8ac8f00b..000000000 --- a/functions/function/task-launch-request-function/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.task.launch.request.TaskLaunchRequestFunctionConfiguration diff --git a/functions/function/task-launch-request-function/src/test/java/org/springframework/cloud/fn/task/launch/request/KeyValueListParserTests.java b/functions/function/task-launch-request-function/src/test/java/org/springframework/cloud/fn/task/launch/request/KeyValueListParserTests.java deleted file mode 100644 index 28d353615..000000000 --- a/functions/function/task-launch-request-function/src/test/java/org/springframework/cloud/fn/task/launch/request/KeyValueListParserTests.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import java.util.Map; - -import org.junit.jupiter.api.Test; - -import static junit.framework.TestCase.assertEquals; -import static junit.framework.TestCase.assertTrue; - -/** - * @author Chris Schaefer - * @author David Turanski - */ -public class KeyValueListParserTests { - - @Test - public void testParseSimpleDeploymentProperty() { - Map deploymentProperties = KeyValueListParser.parseCommaDelimitedKeyValuePairs( - "app.sftp.param=value"); - assertTrue("Invalid number of deployment properties: " + deploymentProperties.size(), - deploymentProperties.size() == 1); - assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.param")); - assertEquals("Invalid deployment value", "value", deploymentProperties.get("app.sftp.param")); - } - - @Test - public void testParseSimpleDeploymentPropertyMultipleValues() { - Map deploymentProperties = KeyValueListParser.parseCommaDelimitedKeyValuePairs( - "app.sftp.param=value1,value2,value3"); - - assertTrue("Invalid number of deployment properties: " + deploymentProperties.size(), - deploymentProperties.size() == 1); - assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.param")); - assertEquals("Invalid deployment value", "value1,value2,value3", deploymentProperties.get("app.sftp.param")); - } - - @Test - public void testParseSpelExpressionMultipleValues() { - Map argExpressions = KeyValueListParser.parseCommaDelimitedKeyValuePairs( - "arg1=payload.substr(0,2),arg2=headers['foo'],arg3=headers['bar']==false"); - - assertTrue("Invalid number of deployment properties: " + argExpressions.size(), - argExpressions.size() == 3); - assertTrue("Expected deployment key not found", argExpressions.containsKey("arg1")); - assertEquals("Invalid deployment value", "payload.substr(0,2)", argExpressions.get("arg1")); - - assertTrue("Expected deployment key not found", argExpressions.containsKey("arg2")); - assertEquals("Invalid deployment value", "headers['foo']", argExpressions.get("arg2")); - - assertTrue("Expected deployment key not found", argExpressions.containsKey("arg3")); - assertEquals("Invalid deployment value", "headers['bar']==false", argExpressions.get("arg3")); - } - - @Test - public void testParseMultipleDeploymentPropertiesSingleValue() { - Map deploymentProperties = KeyValueListParser.parseCommaDelimitedKeyValuePairs( - "app.sftp.param=value1,app.sftp.other.param=value2"); - - assertTrue("Invalid number of deployment properties: " + deploymentProperties.size(), - deploymentProperties.size() == 2); - assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.param")); - assertEquals("Invalid deployment value", "value1", deploymentProperties.get("app.sftp.param")); - assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.other.param")); - assertEquals("Invalid deployment value", "value2", deploymentProperties.get("app.sftp.other.param")); - } - - @Test - public void testParseMultipleDeploymentPropertiesMultipleValues() { - TaskLaunchRequestFunctionProperties taskLaunchRequestProperties = new TaskLaunchRequestFunctionProperties(); - - Map deploymentProperties = KeyValueListParser.parseCommaDelimitedKeyValuePairs( - "app.sftp.param=value1,value2,app.sftp.other.param=other1,other2"); - - assertTrue("Invalid number of deployment properties: " + deploymentProperties.size(), - deploymentProperties.size() == 2); - assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.param")); - assertEquals("Invalid deployment value", "value1,value2", deploymentProperties.get("app.sftp.param")); - assertTrue("Expected deployment key not found", deploymentProperties.containsKey("app.sftp.other.param")); - assertEquals("Invalid deployment value", "other1,other2", deploymentProperties.get("app.sftp.other.param")); - } -} diff --git a/functions/function/task-launch-request-function/src/test/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionApplicationTests.java b/functions/function/task-launch-request-function/src/test/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionApplicationTests.java deleted file mode 100644 index f23399aef..000000000 --- a/functions/function/task-launch-request-function/src/test/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionApplicationTests.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import java.io.IOException; -import java.util.Collections; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import org.springframework.boot.WebApplicationType; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; - -/** - * @author David Turanski - **/ -public class TaskLaunchRequestFunctionApplicationTests { - - private SpringApplicationBuilder springApplicationBuilder; - - @BeforeEach - public void setUp() { - springApplicationBuilder = new SpringApplicationBuilder(TaskLaunchRequestFunctionTestApplication.class) - .web(WebApplicationType.NONE); - } - - @Test - @DirtiesContext - public void simpleDataflowTaskLaunchRequest() throws IOException { - - ApplicationContext context = springApplicationBuilder.properties( - "spring.jmx.enabled=false", - "spring.cloud.function.definition=taskLaunchRequestFunction", - "task.launch.request.task-name=foo") - .run(); - - TaskLaunchRequest taskLaunchRequest = verifyAndreceiveTaskLaunchRequest(context); - - assertThat(taskLaunchRequest.getTaskName()).isEqualTo("foo"); - assertThat(taskLaunchRequest.getCommandlineArguments()).hasSize(0); - assertThat(taskLaunchRequest.getDeploymentProperties()).hasSize(0); - } - - @Test - @DirtiesContext - public void dataflowTaskLaunchRequestWithArgsAndDeploymentProperties() throws IOException { - - ApplicationContext context = springApplicationBuilder.properties( - "spring.jmx.enabled=false", "spring.cloud.function.definition=taskLaunchRequestFunction", - "task.launch.request.task-name=foo", "task.launch.request.args=foo=bar,baz=boo", - "task.launch.request.deploymentProperties=count=3") - .run(); - TaskLaunchRequest taskLaunchRequest = verifyAndreceiveTaskLaunchRequest(context); - - assertThat(taskLaunchRequest.getTaskName()).isEqualTo("foo"); - assertThat(taskLaunchRequest.getCommandlineArguments()).containsExactlyInAnyOrder("foo=bar", - "baz=boo"); - assertThat(taskLaunchRequest.getDeploymentProperties()).containsOnly(entry("count", "3")); - } - - @Test - @DirtiesContext - public void taskLaunchRequestWithCommandLineArgsMessageMapper() throws IOException { - - ApplicationContext context = springApplicationBuilder.properties( - "spring.jmx.enabled=false", "spring.cloud.function.definition=taskLaunchRequestFunction", - "task.launch.request.task-name=foo", "enhanceTLRArgs=true") - .run(); - - TaskLaunchRequest taskLaunchRequest = verifyAndreceiveTaskLaunchRequest(context); - - assertThat(taskLaunchRequest.getTaskName()).isEqualTo("foo"); - assertThat(taskLaunchRequest.getCommandlineArguments()).hasSize(1); - assertThat(taskLaunchRequest.getCommandlineArguments()).containsExactly("runtimeArg"); - - } - - @Test - @DirtiesContext - public void taskLaunchRequestWithArgExpressions() throws IOException { - ApplicationContext context = springApplicationBuilder.properties( - "spring.jmx.enabled=false", - "spring.cloud.function.definition=taskLaunchRequestFunction", - "task.launch.request.task-name=foo", - "task.launch.request.arg-expressions=foo=payload.toUpperCase(),bar=payload.substring(0,2)") - .run(); - - Message message = MessageBuilder.withPayload("hello").build(); - - TaskLaunchRequestFunction taskLaunchRequestFunction = context.getBean(TaskLaunchRequestFunction.class); - - Message response = taskLaunchRequestFunction.apply(message); - - assertThat(response).isNotNull(); - TaskLaunchRequest request = response.getPayload(); - assertThat(request.getCommandlineArguments()).containsExactlyInAnyOrder("foo=HELLO", "bar=he"); - } - - @Test - @DirtiesContext - public void taskLaunchRequestWithIntPayload() throws IOException { - ApplicationContext context = springApplicationBuilder.properties( - "spring.jmx.enabled=false", "spring.cloud.function.definition=taskLaunchRequestFunction", - "task.launch.request.task-name=foo", - "task.launch.request.arg-expressions=i=payload") - .run(); - - TaskLaunchRequestFunction taskLaunchRequestFunction = context.getBean(TaskLaunchRequestFunction.class); - - Message message = MessageBuilder.withPayload(123).build(); - - Message response = taskLaunchRequestFunction.apply(message); - - assertThat(response).isNotNull(); - - TaskLaunchRequest request = response.getPayload(); - assertThat(request.getCommandlineArguments()).containsExactly("i=123"); - } - - @Test - @DirtiesContext - public void taskNameExpression() throws IOException { - ApplicationContext context = springApplicationBuilder.properties( - "spring.jmx.enabled=false", "spring.cloud.function.definition=taskLaunchRequestFunction", - "task.launch.request.task-name-expression=payload+'_task'") - .run(); - - TaskLaunchRequestFunction taskLaunchRequestFunction = context.getBean(TaskLaunchRequestFunction.class); - - Message message = MessageBuilder.withPayload("foo").build(); - - Message response = taskLaunchRequestFunction.apply(message); - assertThat(response).isNotNull(); - - TaskLaunchRequest request = response.getPayload(); - assertThat(request.getTaskName()).isEqualTo("foo_task"); - } - - @Test - @DirtiesContext - public void customTaskNameExtractor() throws IOException { - ApplicationContext context = springApplicationBuilder.properties( - "spring.jmx.enabled=false", "spring.cloud.function.definition=taskLaunchRequestFunction", - "customTaskNameExtractor=true") - .run(); - TaskLaunchRequestFunction taskLaunchRequestFunction = context.getBean(TaskLaunchRequestFunction.class); - - Message message = MessageBuilder.withPayload("foo").build(); - - Message response = taskLaunchRequestFunction.apply(message); - assertThat(response).isNotNull(); - - TaskLaunchRequest request = response.getPayload(); - assertThat(request.getTaskName()).isEqualTo("fooTask"); - - message = MessageBuilder.withPayload("bar").build(); - response = taskLaunchRequestFunction.apply(message); - request = response.getPayload(); - - assertThat(request.getTaskName()).isEqualTo("defaultTask"); - } - - private TaskLaunchRequest verifyAndreceiveTaskLaunchRequest(ApplicationContext context) - throws IOException { - TaskLaunchRequestFunction taskLaunchRequestFunction = context.getBean(TaskLaunchRequestFunction.class); - Message message = taskLaunchRequestFunction - .apply(MessageBuilder.withPayload(new byte[] {}).build()); - assertThat(message).isNotNull(); - return message.getPayload(); - } - - @SpringBootApplication - protected static class TaskLaunchRequestFunctionTestApplication { - - @Bean - @ConditionalOnProperty("customTaskNameExtractor") - TaskNameMessageMapper taskNameExtractor() { - return message -> ((String) (message.getPayload())).equalsIgnoreCase("foo") ? "fooTask" : "defaultTask"; - } - - @Bean - @ConditionalOnProperty("enhanceTLRArgs") - CommandLineArgumentsMessageMapper commandLineArgumentsProvider() { - return message -> Collections.singletonList("runtimeArg"); - } - } -} diff --git a/functions/function/task-launch-request-function/src/test/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionPropertiesTests.java b/functions/function/task-launch-request-function/src/test/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionPropertiesTests.java deleted file mode 100644 index a452a0e9b..000000000 --- a/functions/function/task-launch-request-function/src/test/java/org/springframework/cloud/fn/task/launch/request/TaskLaunchRequestFunctionPropertiesTests.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.task.launch.request; - -import java.util.List; - -import org.junit.jupiter.api.Test; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.boot.test.util.TestPropertyValues; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.integration.config.EnableIntegration; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author David Turanski - **/ -public class TaskLaunchRequestFunctionPropertiesTests { - - @Test - public void deploymentPropertiesCanBeCustomized() { - TaskLaunchRequestFunctionProperties properties = getBatchProperties( - "task.launch.request.deploymentProperties:prop1=val1,prop2=val2"); - assertThat(properties.getDeploymentProperties()).isEqualTo("prop1=val1,prop2=val2"); - } - - @Test - public void parametersCanBeCustomized() { - TaskLaunchRequestFunctionProperties properties = getBatchProperties( - "task.launch.request.args:jp1=jpv1,jp2=jpv2"); - List args = properties.getArgs(); - - assertThat(args).isNotNull(); - assertThat(args).hasSize(2); - assertThat(args.get(0)).isEqualTo("jp1=jpv1"); - assertThat(args.get(1)).isEqualTo("jp2=jpv2"); - } - - private TaskLaunchRequestFunctionProperties getBatchProperties(String... var) { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - - if (var != null) { - TestPropertyValues.of(var).applyTo(context); - } - - context.register(Conf.class); - context.refresh(); - - return context.getBean(TaskLaunchRequestFunctionProperties.class); - } - - @Configuration - @EnableIntegration - @EnableConfigurationProperties(TaskLaunchRequestFunctionProperties.class) - @Import(TaskLaunchRequestFunctionConfiguration.class) - static class Conf { - - } -} diff --git a/functions/function/twitter-function/README.adoc b/functions/function/twitter-function/README.adoc deleted file mode 100644 index 1427a796c..000000000 --- a/functions/function/twitter-function/README.adoc +++ /dev/null @@ -1,149 +0,0 @@ -# Twitter Functions - -This module provides couple of twitter functions that can be reused and composed in other applications. - -To use those functions add the following dependency to your POM: - -[source,XML] ----- - - org.springframework.cloud.fn - twitter-function - ${java-functions.version} - ----- - -## 1. Twitter Trend Function - -Functions can return either Trends topics or the Locations of the trending topics. The `twitter.trend.trend-query-type` property allows choosing between both types. - -* Trends - `twitter.trend.trend-query-type` is set to `trend`. Leverages the https://developer.twitter.com/en/docs/trends/trends-for-location/api-reference/get-trends-place[Trends API] to return the https://help.twitter.com/en/using-twitter/twitter-trending-faqs[trending topics] near a specific latitude, longitude location. - -* Trend Locations - the `twitter.trend.trend-query-type` is set `trendLocation`. Retrieve a full or nearby locations list of trending topics by location. If the `latitude`, `longitude` parameters are NOT provided the processor performs the https://developer.twitter.com/en/docs/trends/locations-with-trending-topics/api-reference/get-trends-available[Trends Available API] and returns the locations that Twitter has trending topic information for. -If the `latitude`, `longitude` parameters are provided the processor performs the https://developer.twitter.com/en/docs/trends/locations-with-trending-topics/api-reference/get-trends-closest[Trends Closest API] and returns the locations that Twitter has trending topic information for, closest to a specified location. - -Response is an array of `locations` that encode the location's WOEID and some other human-readable information such as a canonical name and country the location belongs in. - -### 1.1 Beans for injection - -You can import the `TwitterTrendFunctionConfiguration` in a Spring Boot application and then inject the following bean. - -`filterFunction` - -You can use `Function, Message> twitterTrendFunction` as a qualifier when injecting. - -Once injected, you can use the `apply` method of the `Function` to invoke it and get the result. - -### 1.2 Configuration Options - -TIP: For `SpEL` expression properties wrap the literal values in single quotes (`'`). - -For more information on the various options available, please see link:../twitter-function/src/main/java/org/springframework/cloud/fn/twitter/trend/TwitterTrendFunctionProperties.java[TwitterTrendFunctionProperties.java] - -### 1.3 Tests -See this link:src/test/java/org/springframework/cloud/fn/twitter/trend/TwitterTrendFunctionTests.java[test suite] for examples of how this function is used. - -### 1.4 Other usage - -Leveraging the Spring Cloud Function composability, you can compose the Trend function in your boot app like this: - -[source,Java] ----- -@SpringBootApplication -@Import(TwitterTrendFunctionConfiguration.class ) -public class MyTwitterTrendBootApp { - - public static void main(String[] args) { - SpringApplication.run(MyTwitterTrendBootApp.class, - "--spring.cloud.function.definition=trend|twitterTrendFunction|managedJson"); - } -} ----- - -## 2. Twitter Geo Function. - -Function based on the https://developer.twitter.com/en/docs/geo/places-near-location/overview[Geo API] that retrieves Twitter place information based on query parameters such as (`latitude`, `longitude`) pair, an `IP` address, or a place `name`. - -There are two types for geo search queries `search` and `reverse` controlled by the `twitter.geo.search.type` property. - -* reverse - Given a latitude and a longitude, searches for up to 20 places that can be used as a `placeId` when updating a status. -This request is an informative call and will deliver generalized results about geography. - -* search - Search for places that can be attached to statuses/update. Given a latitude and a longitude pair, an IP address, or a name, this request will return a list of all the valid places that can be used as the place_id when updating a status. - -Conceptually, a query can be made from the user’s location, retrieve a list of places, have the user validate the location he or she is at, and then send the ID of this location with a call to POST statuses/update. - -This is the recommended method to use find places that can be attached to statuses/update. Unlike GET geo/reverse_geocode which provides raw data access, this endpoint can potentially re-order places with regard to the user who is authenticated. Use this approach for interactive place matching with the user. - -Some parameters in this method are only required based on the existence of other parameters. For instance, “lat” is required if “long” is provided, and vice-versa. Authentication is recommended, but not required with this method. - -NOTE: Limits: 15 requests / 15-min window (user auth). - -### 2.1 Beans for injection - -You can import the `TwitterGeoFunctionConfiguration` in a Spring Boot application and then inject the following bean. - -`filterFunction` - -You can use `Function, Message> twitterGeoFunction` as a qualifier when injecting. -Once injected, you can use the `apply` method of the `Function` to invoke it and get the result. - -### 2.2 Configuration Options - -TIP: For `SpEL` expression properties wrap the literal values in single quotes (`'`). - -For more information on the various options available, please see link:../twitter-function/src/main/java/org/springframework/cloud/fn/twitter/geo/TwitterGeoFunctionProperties.java[TwitterGeoFunctionProperties.java] - -### 2.3 Tests - -See this link:src/test/java/org/springframework/cloud/fn/twitter/geo/TwitterGeoFunctionTest.java[test suite] for examples of how this function is used. - -### 2.4 Other usage - -Leveraging the Spring Cloud Function composability, you can compose the Geo function in your boot app like this: - -[source,Java] ----- -@SpringBootApplication -@Import(TwitterGeoFunctionConfiguration.class ) -public class MyTwitterGeoProcessorBootApp { - - public static void main(String[] args) { - SpringApplication.run(MyTwitterGeoProcessorBootApp.class, - "--spring.cloud.function.definition=messageToGeoQueryFunction|twitterSearchPlacesFunction|managedJson"); - } -} ----- - -## 3. Twitter Users Function - -Retrieves users either by list of use ids and/or screen-names (https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-users-lookup[Users Lookup API]) or by text search query (https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-users-search[Users Search API]). -Uses SpEL expressions to compute the query parameters from the input message. -Use the single quoted literals to set static values (e.g. user-id: '6666, 9999, 10000'). - -Use `twitter.users.type` property allow to select the query approaches. - -* https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-users-lookup[Users Lookup API] - Returns fully-hydrated user objects for up to 100 users per request, as specified by comma-separated values passed to the `userId` and/or `screenName` parameters. Rate limits: (300 requests / 15-min window) -* https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-users-search[Users Search API] - Relevance-based search interface to public user accounts on Twitter. -Querying by topical interest, full name, company name, location, or other criteria. Exact match searches are not supported. Only the first 1,000 matching results are available. Rate limits:(900 requests / 15-min window) - -### 3.1 Beans for injection - -You can import the `TwitterUsersFunctionConfiguration` in a Spring Boot application and then inject the following bean. - -`filterFunction` - -You can use `Function, Message> twitterUsersFunction` as a qualifier when injecting. -Once injected, you can use the `apply` method of the `Function` to invoke it and get the result. - - -### 3.2 Configuration Options - -TIP: For `SpEL` expression properties wrap the literal values in single quotes (`'`). - -For more information on the various options available, please see link:../twitter-function/src/main/java/org/springframework/cloud/fn/twitter/users/TwitterUsersFunctionProperties.java[TwitterUsersFunctionProperties.java] - -### 3.3 Tests -See this link:src/test/java/org/springframework/cloud/fn/twitter/users/TwitterUsersFunctionTests.java[test suite] for examples of how this function is used. - -### 3.4 Other usage diff --git a/functions/function/twitter-function/pom.xml b/functions/function/twitter-function/pom.xml deleted file mode 100644 index 17bf724a3..000000000 --- a/functions/function/twitter-function/pom.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - twitter-function - twitter-function - twitter functions - - - - org.springframework.cloud.fn - twitter-common - ${project.version} - - - org.mock-server - mockserver-netty - ${mockserver.version} - test - - - org.mock-server - mockserver-client-java - ${mockserver.version} - test - - - - diff --git a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/geo/TwitterGeoFunctionConfiguration.java b/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/geo/TwitterGeoFunctionConfiguration.java deleted file mode 100644 index e58443ced..000000000 --- a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/geo/TwitterGeoFunctionConfiguration.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.twitter.geo; - -import java.util.List; -import java.util.function.Function; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import twitter4j.GeoLocation; -import twitter4j.GeoQuery; -import twitter4j.Place; -import twitter4j.Twitter; -import twitter4j.TwitterException; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; - -/** - * - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties(TwitterGeoFunctionProperties.class) -@Import(TwitterConnectionConfiguration.class) -public class TwitterGeoFunctionConfiguration { - - private static final Log logger = LogFactory.getLog(TwitterGeoFunctionConfiguration.class); - - @Bean - public Function, GeoQuery> messageToGeoQueryFunction(TwitterGeoFunctionProperties geoProperties) { - return message -> { - String ip = null; - if (geoProperties.getSearch().getIp() != null) { - ip = geoProperties.getSearch().getIp().getValue(message, String.class); - } - GeoLocation geoLocation = null; - if (geoProperties.getLocation().getLat() != null && geoProperties.getLocation().getLon() != null) { - Double lat = geoProperties.getLocation().getLat().getValue(message, Double.class); - Double lon = geoProperties.getLocation().getLon().getValue(message, Double.class); - geoLocation = new GeoLocation(lat, lon); - } - - String query = null; - if (geoProperties.getSearch().getQuery() != null) { - query = geoProperties.getSearch().getQuery().getValue(message, String.class); - } - GeoQuery geoQuery = new GeoQuery(query, ip, geoLocation); - - geoQuery.setMaxResults(geoProperties.getMaxResults()); - geoQuery.setAccuracy(geoProperties.getAccuracy()); - geoQuery.setGranularity(geoProperties.getGranularity()); - - return geoQuery; - }; - } - - @Bean - @ConditionalOnProperty(name = "twitter.geo.search.type", havingValue = "search", matchIfMissing = true) - public Function> twitterSearchPlacesFunction(Twitter twitter) { - return geoQuery -> { - try { - return twitter.searchPlaces(geoQuery); - } - catch (TwitterException e) { - logger.error("Places Search failed!", e); - } - return null; - }; - } - - @Bean - @ConditionalOnProperty(name = "twitter.geo.search.type", havingValue = "reverse") - public Function> twitterReverseGeocodeFunction(Twitter twitter) { - return geoQuery -> { - try { - return twitter.reverseGeoCode(geoQuery); - } - catch (TwitterException e) { - logger.error("Reverse Geocode failed!", e); - } - return null; - }; - } - - @Bean - public Function, Message> twitterGeoFunction( - Function, GeoQuery> toGeoQuery, - Function> places, - Function> managedJson) { - - return toGeoQuery.andThen(places).andThen(managedJson)::apply; - } -} diff --git a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/geo/TwitterGeoFunctionProperties.java b/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/geo/TwitterGeoFunctionProperties.java deleted file mode 100644 index 9e308342d..000000000 --- a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/geo/TwitterGeoFunctionProperties.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.twitter.geo; - -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.validation.annotation.Validated; - - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("twitter.geo") -@Validated -public class TwitterGeoFunctionProperties { - - private static final Expression DEFAULT_EXPRESSION = new SpelExpressionParser().parseExpression("payload"); - - public enum GeoType { - /** Geo retrieval type. */ - reverse, search - } - - /** - * Geo search API type: reverse or search. - */ - @NotNull - private GeoType type = GeoType.search; - - /** - * Search geo type filter parameters. - */ - private Search search = new Search(); - - /** - * - */ - private Location location = new Location(); - - /** - * Hints for the number of results to return. This does not guarantee that the number of results - * returned will equal max_results, but instead informs how many 'nearby' results to return. - */ - private int maxResults = -1; - - /** - * Sets a hint on the 'region' in which to search. If a number, then this is a radius in meters, but it - * can also take a string that is suffixed with ft to specify feet. If this is not passed in, then it is - * assumed to be 0m. If coming from a device, in practice, this value is whatever accuracy the device - * has measuring its location (whether it be coming from a GPS, WiFi triangulation, etc.). - */ - private String accuracy = null; - - /** - * Minimal granularity of data to return. If this is not passed in, then neighborhood is assumed. - * City can also be passed. - */ - private String granularity = null; - - public Search getSearch() { - return search; - } - - public void setSearch(Search search) { - this.search = search; - } - - public GeoType getType() { - return type; - } - - public void setType(GeoType type) { - this.type = type; - } - - public Location getLocation() { - return location; - } - - public void setLocation(Location location) { - this.location = location; - } - - public int getMaxResults() { - return maxResults; - } - - public void setMaxResults(int maxResults) { - this.maxResults = maxResults; - } - - public String getAccuracy() { - return accuracy; - } - - public void setAccuracy(String accuracy) { - this.accuracy = accuracy; - } - - public String getGranularity() { - return granularity; - } - - public void setGranularity(String granularity) { - this.granularity = granularity; - } - - @AssertTrue(message = "Either the IP or the Location must be set") - public boolean isAtLeastOne() { - return this.getSearch().getIp() == null ^ (this.getLocation().getLat() == null && this.getLocation().getLon() == null); - } - - @AssertTrue(message = "The IP parameter is applicable only for 'Search' GeoType") - public boolean isIpUsedWithSearchGeoType() { - if (this.getSearch().getIp() != null) { - return this.type == GeoType.search; - } - return true; - } - - public static class Search { - /** - * An IP address. Used when attempting to fix geolocation based off of the user's IP address. - * Applicable only for 'search' geo type. - */ - private Expression ip = null; - - /** - * Query expression to filter Places in search results. - */ - private Expression query = null; - - public Expression getIp() { - return ip; - } - - public void setIp(Expression ip) { - this.ip = ip; - } - - public Expression getQuery() { - return query; - } - - public void setQuery(Expression query) { - this.query = query; - } - } - - public static class Location { - - /** - * User's lat. - */ - private Expression lat; - - /** - * User's lon. - */ - private Expression lon; - - - public Expression getLat() { - return lat; - } - - public void setLat(Expression lat) { - this.lat = lat; - } - - public Expression getLon() { - return lon; - } - - public void setLon(Expression lon) { - this.lon = lon; - } - } -} diff --git a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/trend/TwitterTrendFunctionConfiguration.java b/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/trend/TwitterTrendFunctionConfiguration.java deleted file mode 100644 index c82f023cd..000000000 --- a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/trend/TwitterTrendFunctionConfiguration.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.twitter.trend; - -import java.util.List; -import java.util.function.Function; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import twitter4j.GeoLocation; -import twitter4j.Location; -import twitter4j.Trends; -import twitter4j.Twitter; -import twitter4j.TwitterException; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; - -/** - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties(TwitterTrendFunctionProperties.class) -@Import(TwitterConnectionConfiguration.class) -public class TwitterTrendFunctionConfiguration { - - private static final Log logger = LogFactory.getLog(TwitterTrendFunctionConfiguration.class); - - @Bean - public Function, Trends> trend(TwitterTrendFunctionProperties properties, Twitter twitter) { - return message -> { - try { - int woeid = properties.getLocationId().getValue(message, int.class); - return twitter.getPlaceTrends(woeid); - } - catch (TwitterException e) { - logger.error("Twitter API error!", e); - } - return null; - }; - } - - @Bean - public Function, List> closestOrAvailableTrends( - TwitterTrendFunctionProperties properties, Twitter twitter) { - return message -> { - try { - if (properties.getClosest().getLat() != null && properties.getClosest().getLon() != null) { - double lat = properties.getClosest().getLat().getValue(message, double.class); - double lon = properties.getClosest().getLon().getValue(message, double.class); - return twitter.getClosestTrends(new GeoLocation(lat, lon)); - } - else { - return twitter.getAvailableTrends(); - } - } - catch (TwitterException e) { - logger.error("Twitter API error!", e); - } - return null; - }; - } - - @Bean - public Function, Message> twitterTrendFunction( - Function> managedJson, Function, Trends> trend, - TwitterTrendFunctionProperties properties, Function, - List> closestOrAvailableTrends) { - - return (properties.getTrendQueryType() == TwitterTrendFunctionProperties.TrendQueryType.trend) ? - trend.andThen(managedJson) : closestOrAvailableTrends.andThen(managedJson); - } -} diff --git a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/trend/TwitterTrendFunctionProperties.java b/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/trend/TwitterTrendFunctionProperties.java deleted file mode 100644 index e3fe33ba9..000000000 --- a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/trend/TwitterTrendFunctionProperties.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.twitter.trend; - -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.validation.annotation.Validated; - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("twitter.trend") -@Validated -public class TwitterTrendFunctionProperties { - - private static final Expression DEFAULT_EXPRESSION = new SpelExpressionParser().parseExpression("payload"); - - enum TrendQueryType { - /** Retrieve trending places. */ - trend, - /** Retrieve the Locations of trending places. */ - trendLocation - } - - private TrendQueryType trendQueryType = TrendQueryType.trend; - - public TrendQueryType getTrendQueryType() { - return trendQueryType; - } - - public void setTrendQueryType(TrendQueryType trendQueryType) { - this.trendQueryType = trendQueryType; - } - - /** - * The Yahoo! Where On Earth ID of the location to return trending information for. - * Global information is available by using 1 as the WOEID. - */ - @NotNull - private Expression locationId = DEFAULT_EXPRESSION; - - public Expression getLocationId() { - return locationId; - } - - public void setLocationId(Expression locationId) { - this.locationId = locationId; - } - - /** - * - */ - private Closest closest = new Closest(); - - public Closest getClosest() { - return closest; - } - - public static class Closest { - /** - * If provided with a long parameter the available trend locations will be sorted by distance, nearest - * to furthest, to the co-ordinate pair. - * The valid ranges for longitude is -180.0 to +180.0 (West is negative, East is positive) inclusive. - */ - private Expression lat; - - /** - * If provided with a lat parameter the available trend locations will be sorted by distance, nearest to - * furthest, to the co-ordinate pair. The valid ranges for longitude is -180.0 to +180.0 (West is negative, - * East is positive) inclusive. - */ - private Expression lon; - - public Expression getLat() { - return lat; - } - - public void setLat(Expression lat) { - this.lat = lat; - } - - public Expression getLon() { - return lon; - } - - public void setLon(Expression lon) { - this.lon = lon; - } - } -} diff --git a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/users/TwitterUsersFunctionConfiguration.java b/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/users/TwitterUsersFunctionConfiguration.java deleted file mode 100644 index 17dfafe3c..000000000 --- a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/users/TwitterUsersFunctionConfiguration.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.twitter.users; - -import java.util.List; -import java.util.function.Function; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import twitter4j.ResponseList; -import twitter4j.Twitter; -import twitter4j.TwitterException; -import twitter4j.User; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; - -/** - * - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties(TwitterUsersFunctionProperties.class) -@Import(TwitterConnectionConfiguration.class) -public class TwitterUsersFunctionConfiguration { - - private static final Log logger = LogFactory.getLog(TwitterUsersFunctionConfiguration.class); - - @Bean - @ConditionalOnProperty(name = "twitter.users.type", havingValue = "search") - public Function, List> userSearch(Twitter twitter, TwitterUsersFunctionProperties properties) { - - return message -> { - String query = properties.getSearch().getQuery().getValue(message, String.class); - try { - ResponseList users = twitter.searchUsers(query, properties.getSearch().getPage()); - return users; - } - catch (TwitterException e) { - logger.error("Twitter API error!", e); - } - return null; - }; - } - - @Bean - @ConditionalOnProperty(name = "twitter.users.type", havingValue = "lookup") - public Function, List> userLookup(Twitter twitter, TwitterUsersFunctionProperties properties) { - - return message -> { - - try { - TwitterUsersFunctionProperties.Lookup lookup = properties.getLookup(); - if (lookup.getScreenName() != null) { - String[] screenNames = lookup.getScreenName().getValue(message, String[].class); - return twitter.lookupUsers(screenNames); - } - else if (lookup.getUserId() != null) { - long[] ids = lookup.getUserId().getValue(message, long[].class); - return twitter.lookupUsers(ids); - } - } - catch (TwitterException e) { - logger.error("Twitter API error!", e); - } - return null; - }; - } - - @Bean - /** - * queryUsers - depends on the `twitter.users.type` property is either userSearch or userLookup. - * managedJson - converts Users into JSON message payload. - */ - public Function, Message> twitterUsersFunction(Function, List> queryUsers, - Function> managedJson) { - return queryUsers.andThen(managedJson); - } -} diff --git a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/users/TwitterUsersFunctionProperties.java b/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/users/TwitterUsersFunctionProperties.java deleted file mode 100644 index 8f1d5359b..000000000 --- a/functions/function/twitter-function/src/main/java/org/springframework/cloud/fn/twitter/users/TwitterUsersFunctionProperties.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.twitter.users; - -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.validation.annotation.Validated; - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("twitter.users") -@Validated -public class TwitterUsersFunctionProperties { - - private static final Expression DEFAULT_EXPRESSION = new SpelExpressionParser().parseExpression("payload"); - - public enum UserQueryType { - /** User retrieval types. */ - search, lookup - } - - /** - * Perform search or lookup type of search. - */ - @NotNull - private UserQueryType type = UserQueryType.search; - - /** - * Returns fully-hydrated user objects for specified by comma-separated values passed to the user_id and/or - * screen_name parameters. - */ - private final Lookup lookup = new Lookup(); - - /** - * relevance-based search interface for querying by topical interest, full name, company name, location, - * or other criteria. - */ - private final Search search = new Search(); - - public UserQueryType getType() { - return type; - } - - public void setType(UserQueryType type) { - this.type = type; - } - - public Lookup getLookup() { - return lookup; - } - - public Search getSearch() { - return search; - } - - @AssertTrue(message = "Per query type validate the required parameters") - public boolean checkParametersPerType() { - if (this.getType() == UserQueryType.lookup) { - return (this.getLookup().getScreenName() != null) || (this.getLookup().getUserId() != null); - } - else if (this.getType() == UserQueryType.search) { - return (this.getSearch().getQuery() != null); - } - - return false; - } - - public static class Lookup { - /** - * A comma separated list of user IDs, up to 100 are allowed in a single request. - * You are strongly encouraged to use a POST for larger requests. - */ - private Expression userId; - - /** - * A comma separated list of screen names, up to 100 are allowed in a single request. - * You are strongly encouraged to use a POST for larger (up to 100 screen names) requests. - */ - private Expression screenName; - - public Expression getUserId() { - return userId; - } - - public void setUserId(Expression userId) { - this.userId = userId; - } - - public Expression getScreenName() { - return screenName; - } - - public void setScreenName(Expression screenName) { - this.screenName = screenName; - } - } - - public static class Search { - /** - * The search query to run against people search. - */ - private Expression query = DEFAULT_EXPRESSION; - - /** - * Specifies the page of results to retrieve. - */ - private int page = 3; - - public Expression getQuery() { - return query; - } - - public void setQuery(Expression query) { - this.query = query; - } - - public int getPage() { - return page; - } - - public void setPage(int page) { - this.page = page; - } - } -} diff --git a/functions/function/twitter-function/src/test/java/org/springframework/cloud/fn/twitter/geo/TwitterGeoFunctionTest.java b/functions/function/twitter-function/src/test/java/org/springframework/cloud/fn/twitter/geo/TwitterGeoFunctionTest.java deleted file mode 100644 index e3b50e79c..000000000 --- a/functions/function/twitter-function/src/test/java/org/springframework/cloud/fn/twitter/geo/TwitterGeoFunctionTest.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.twitter.geo; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.mockserver.client.MockServerClient; -import org.mockserver.integration.ClientAndServer; -import twitter4j.conf.ConfigurationBuilder; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties; -import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.util.TestSocketUtils; -import org.springframework.util.MimeTypeUtils; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockserver.matchers.Times.unlimited; -import static org.mockserver.model.HttpRequest.request; -import static org.mockserver.model.HttpResponse.response; -import static org.mockserver.verify.VerificationTimes.once; - -/** - * @author Christian Tzolov - */ -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { - "twitter.connection.consumerKey=consumerKey666", - "twitter.connection.consumerSecret=consumerSecret666", - "twitter.connection.accessToken=accessToken666", - "twitter.connection.accessTokenSecret=accessTokenSecret666" - }) -@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) -public abstract class TwitterGeoFunctionTest { - - private static final String MOCK_SERVER_IP = "127.0.0.1"; - - private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort(); - - private static ClientAndServer mockServer; - - private static MockServerClient mockClient; - - @Autowired - protected Function, Message> twitterUsersFunction; - - public static void recordRequestExpectation(Map> parameters) { - - mockClient - .when( - request() - .withMethod("GET") - .withPath("/geo/search.json") - .withQueryStringParameters(parameters), - unlimited()) - .respond( - response() - .withStatusCode(200) - .withHeader("Content-Type", "application/json; charset=utf-8") - .withBody(TwitterTestUtils.asString("classpath:/response/search_places_amsterdam.json")) - .withDelay(TimeUnit.SECONDS, 1)); - - } - - @BeforeAll - public static void startMockServer() { - mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT); - mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT); - } - - @AfterAll - public static void stopMockServer() { - mockServer.stop(); - } - - @TestPropertySource(properties = { - "twitter.geo.search.ip='127.0.0.1'", - "twitter.geo.search.query=payload.toUpperCase()" - }) - public static class TwitterGeoSearchByIPAndQueryTests extends TwitterGeoFunctionTest { - - @Test - public void testOne() throws IOException { - - Map> queryParameters = new HashMap<>(); - queryParameters.put("ip", Collections.singletonList("127.0.0.1")); - queryParameters.put("query", Collections.singletonList("Amsterdam")); - - recordRequestExpectation(queryParameters); - - String inPayload = "Amsterdam"; - - Message received = twitterUsersFunction.apply(MessageBuilder.withPayload(inPayload).build()); - - mockClient.verify(request() - .withMethod("GET") - .withPath("/geo/search.json") - .withQueryStringParameter("ip", "127.0.0.1") - .withQueryStringParameter("query", "AMSTERDAM"), - once()); - - String outPayload = new String((byte[]) received.getPayload()); - - assertThat(outPayload).isNotNull(); - - List places = new ObjectMapper().readValue(outPayload, List.class); - assertThat(places).hasSize(12); - } - } - - @TestPropertySource(properties = { - "twitter.geo.location.lat='52.378'", - "twitter.geo.location.lon='4.9'", - "twitter.geo.search.query=payload.toUpperCase()" - }) - public static class TwitterGeoSearchByLocationTests extends TwitterGeoFunctionTest { - - @Test - public void testOne() throws IOException { - - Map> queryParameters = new HashMap<>(); - queryParameters.put("lat", Collections.singletonList("52.378")); - queryParameters.put("long", Collections.singletonList("4.9")); - queryParameters.put("query", Collections.singletonList("Amsterdam")); - - recordRequestExpectation(queryParameters); - - String inPayload = "Amsterdam"; - - Message received = twitterUsersFunction.apply(MessageBuilder.withPayload(inPayload).build()); - - mockClient.verify(request() - .withMethod("GET") - .withPath("/geo/search.json") - .withQueryStringParameter("lat", "52.378") - .withQueryStringParameter("long", "4.9") - .withQueryStringParameter("query", "Amsterdam"), - once()); - - String outPayload = new String((byte[]) received.getPayload()); - - assertThat(outPayload).isNotNull(); - - List places = new ObjectMapper().readValue(outPayload, List.class); - assertThat(places).hasSize(12); - } - } - - @TestPropertySource(properties = { - "twitter.geo.type=reverse", - "twitter.geo.location.lat='52.378'", - "twitter.geo.location.lon='4.9'" - }) - public static class TwitterGeoSearchByLocation2Tests extends TwitterGeoFunctionTest { - - @Test - public void testOne() throws IOException { - - Map> queryParameters = new HashMap<>(); - queryParameters.put("lat", Arrays.asList("52.378")); - queryParameters.put("long", Arrays.asList("4.9")); - - recordRequestExpectation(queryParameters); - - String inPayload = "Amsterdam"; - - Message received = twitterUsersFunction.apply(MessageBuilder.withPayload(inPayload).build()); - - mockClient.verify(request() - .withMethod("GET") - .withPath("/geo/search.json") - .withQueryStringParameter("lat", "52.378") - .withQueryStringParameter("long", "4.9"), - once()); - - String outPayload = new String((byte[]) received.getPayload()); - - assertThat(outPayload).isNotNull(); - - List places = new ObjectMapper().readValue(outPayload, List.class); - assertThat(places).hasSize(12); - } - } - - @TestPropertySource(properties = { - "twitter.geo.location.lat=#jsonPath(new String(payload),'$.location.lat')", - "twitter.geo.location.lon=#jsonPath(new String(payload),'$.location.lon')", - "twitter.geo.search.query=#jsonPath(new String(payload),'$.country')" - }) - public static class TwitterGeoSearchJsonPathTests extends TwitterGeoFunctionTest { - - @Test - public void testOne() throws IOException { - - Map> queryParameters = new HashMap<>(); - queryParameters.put("lat", Collections.singletonList("52.0")); - queryParameters.put("long", Collections.singletonList("5.0")); - queryParameters.put("query", Collections.singletonList("Netherlands")); - - recordRequestExpectation(queryParameters); - - String inPayload = "{ \"country\" : \"Netherlands\", \"location\" : { \"lat\" : 52.00 , \"lon\" : 5.0 } }"; - - Message received = twitterUsersFunction.apply(MessageBuilder - .withPayload(inPayload) - .setHeader("contentType", MimeTypeUtils.APPLICATION_JSON_VALUE) - .build()); - - mockClient.verify(request() - .withMethod("GET") - .withPath("/geo/search.json") - .withQueryStringParameter("lat", "52.0") - .withQueryStringParameter("long", "5.0") - .withQueryStringParameter("query", "Netherlands"), - once()); - - String outPayload = new String((byte[]) received.getPayload()); - - assertThat(outPayload).isNotNull(); - - List places = new ObjectMapper().readValue(outPayload, List.class); - assertThat(places).hasSize(12); - } - } - - @SpringBootConfiguration - @EnableAutoConfiguration - @Import(TwitterGeoFunctionConfiguration.class) - public static class TwitterGeoFunctionTestApplication { - @Bean - @Primary - public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionProperties properties, - Function toConfigurationBuilder) { - - Function mockedConfiguration = - toConfigurationBuilder.andThen( - new TwitterTestUtils().mockTwitterUrls( - String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT))); - - return mockedConfiguration.apply(properties).build(); - } - } -} diff --git a/functions/function/twitter-function/src/test/java/org/springframework/cloud/fn/twitter/trend/TwitterTrendFunctionTests.java b/functions/function/twitter-function/src/test/java/org/springframework/cloud/fn/twitter/trend/TwitterTrendFunctionTests.java deleted file mode 100644 index bcd49d9da..000000000 --- a/functions/function/twitter-function/src/test/java/org/springframework/cloud/fn/twitter/trend/TwitterTrendFunctionTests.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.twitter.trend; - -import java.util.concurrent.TimeUnit; -import java.util.function.Function; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.mockserver.client.MockServerClient; -import org.mockserver.integration.ClientAndServer; -import org.mockserver.model.Header; -import org.mockserver.model.HttpRequest; -import twitter4j.conf.ConfigurationBuilder; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties; -import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.util.TestSocketUtils; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockserver.matchers.Times.exactly; -import static org.mockserver.model.HttpRequest.request; -import static org.mockserver.model.HttpResponse.response; -import static org.mockserver.verify.VerificationTimes.once; - -/** - * @author Christian Tzolov - */ -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { - "twitter.connection.consumerKey=consumerKey666", - "twitter.connection.consumerSecret=consumerSecret666", - "twitter.connection.accessToken=accessToken666", - "twitter.connection.accessTokenSecret=accessTokenSecret666" - }) -@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) -public abstract class TwitterTrendFunctionTests { - - private static final String MOCK_SERVER_IP = "127.0.0.1"; - - private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort(); - - private static ClientAndServer mockServer; - - private static MockServerClient mockClient; - private static HttpRequest trendsRequest; - - @Autowired - protected Function, Message> twitterTrendFunction; - - @BeforeAll - public static void startServer() { - mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT); - mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT); - - trendsRequest = setExpectation(request() - .withMethod("GET") - .withPath("/trends/place.json") - .withQueryStringParameter("id", "2972")); - } - - @AfterAll - public static void stopServer() { - mockServer.stop(); - } - - public static HttpRequest setExpectation(HttpRequest request) { - mockClient - .when(request, exactly(1)) - .respond(response() - .withStatusCode(200) - .withHeaders( - new Header("Content-Type", "application/json; charset=utf-8"), - new Header("Cache-Control", "public, max-age=86400")) - .withBody(TwitterTestUtils.asString("classpath:/response/trends.json")) - .withDelay(TimeUnit.SECONDS, 1) - ); - return request; - } - - @TestPropertySource(properties = { - "twitter.trend.locationId='2972'", - "twitter.connection.rawJson=true" - }) - public static class TwitterTrendPayloadTests extends TwitterTrendFunctionTests { - - @Test - public void testOne() { - Message received = twitterTrendFunction.apply(MessageBuilder.withPayload("Hello").build()); - mockClient.verify(trendsRequest, once()); - assertThat(received).isNotNull(); - } - } - - @SpringBootConfiguration - @EnableAutoConfiguration - @Import(TwitterTrendFunctionConfiguration.class) - public static class TwitterTrendFunctionTestApplication { - @Bean - @Primary - public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionProperties properties, - Function toConfigurationBuilder) { - - Function mockedConfiguration = - toConfigurationBuilder.andThen( - new TwitterTestUtils().mockTwitterUrls( - String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT))); - - return mockedConfiguration.apply(properties).build(); - } - } - -} diff --git a/functions/function/twitter-function/src/test/java/org/springframework/cloud/fn/twitter/users/TwitterUsersFunctionTests.java b/functions/function/twitter-function/src/test/java/org/springframework/cloud/fn/twitter/users/TwitterUsersFunctionTests.java deleted file mode 100644 index 8417d8e46..000000000 --- a/functions/function/twitter-function/src/test/java/org/springframework/cloud/fn/twitter/users/TwitterUsersFunctionTests.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.twitter.users; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; - -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.mockserver.client.MockServerClient; -import org.mockserver.integration.ClientAndServer; -import org.mockserver.model.Header; -import org.mockserver.model.HttpRequest; -import twitter4j.conf.ConfigurationBuilder; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties; -import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.util.TestSocketUtils; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockserver.matchers.Times.exactly; -import static org.mockserver.model.HttpRequest.request; -import static org.mockserver.model.HttpResponse.response; -import static org.mockserver.verify.VerificationTimes.once; - -/** - * @author Christian Tzolov - */ -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { - "twitter.connection.consumerKey=consumerKey666", - "twitter.connection.consumerSecret=consumerSecret666", - "twitter.connection.accessToken=accessToken666", - "twitter.connection.accessTokenSecret=accessTokenSecret666" - }) -@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) -public abstract class TwitterUsersFunctionTests { - - private static final String MOCK_SERVER_IP = "127.0.0.1"; - - private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort(); - - private static ClientAndServer mockServer; - - private static MockServerClient mockClient; - private static HttpRequest searchUsersRequest; - private static HttpRequest lookupUsersRequest; - private static HttpRequest lookupUsersRequest2; - - @Autowired - protected ObjectMapper mapper; - - @Autowired - Function, Message> twitterUsersFunction; - - public static HttpRequest setExpectation(HttpRequest request, String responseUri) { - mockClient - .when(request, exactly(1)) - .respond(response() - .withStatusCode(200) - .withHeaders( - new Header("Content-Type", "application/json; charset=utf-8"), - new Header("Cache-Control", "public, max-age=86400")) - .withBody(TwitterTestUtils.asString(responseUri)) - .withDelay(TimeUnit.SECONDS, 1) - ); - return request; - } - - @BeforeAll - public static void startServer() { - mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT); - mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT); - - searchUsersRequest = setExpectation(request() - .withMethod("GET") - .withPath("/users/search.json") - .withQueryStringParameter("q", "tzolov") - .withQueryStringParameter("page", "3"), - "classpath:/response/search_users.json"); - - lookupUsersRequest = setExpectation(request() - .withMethod("GET") - .withPath("/users/lookup.json") - .withQueryStringParameter("user_id", - "710705860343963648,326896547,267603736,781497571629989888,838754923"), - "classpath:/response/lookup_users_id.json"); - - lookupUsersRequest2 = setExpectation(request() - .withMethod("GET") - .withPath("/users/lookup.json") - .withQueryStringParameter("screen_name", - "TzolovMarto,Rabotnik57,antzolov,peyo_tzolov,ivantzolov"), - "classpath:/response/lookup_users_id.json"); - } - - @AfterAll - public static void stopServer() { - mockServer.stop(); - } - - @TestPropertySource(properties = { - "twitter.users.type=search", - "twitter.users.search.query=payload" - }) - public static class TwitterSearchUsersTests extends TwitterUsersFunctionTests { - - @Test - public void testOne() throws IOException { - Message received = twitterUsersFunction.apply(MessageBuilder.withPayload("tzolov").build()); - mockClient.verify(searchUsersRequest, once()); - assertThat(received).isNotNull(); - List list = mapper.readValue(new String((byte[]) received.getPayload()), List.class); - assertThat(list).hasSize(20); - } - } - - @TestPropertySource(properties = { - "twitter.users.type=lookup", - "twitter.users.lookup.userId='710705860343963648,326896547,267603736,781497571629989888,838754923'" - }) - public static class TwitterLookupUserIdLiteralTests extends TwitterUsersFunctionTests { - - @Test - public void testOne() throws IOException { - - Message received = twitterUsersFunction.apply(MessageBuilder.withPayload("tzolov").build()); - - mockClient.verify(lookupUsersRequest, once()); - assertThat(received).isNotNull(); - List list = mapper.readValue(new String((byte[]) received.getPayload()), List.class); - assertThat(list).hasSize(5); - } - } - - @TestPropertySource(properties = { - "twitter.users.type=lookup", - "twitter.users.lookup.screenName=#jsonPath(payload,'$[*].code')" - }) - public static class TwitterLookupScreenNamePayloadTests extends TwitterUsersFunctionTests { - - @Test - public void testOne() throws IOException { - - Object payload = "[{\"code\":\"TzolovMarto\"},{\"code\":\"Rabotnik57\"},{\"code\":\"antzolov\"},{\"code\":\"peyo_tzolov\"},{\"code\":\"ivantzolov\"}]"; - - Message received = twitterUsersFunction.apply(MessageBuilder.withPayload(payload).build()); - assertThat(received).isNotNull(); - - mockClient.verify(lookupUsersRequest2, once()); - assertThat(received).isNotNull(); - List list = mapper.readValue(new String((byte[]) received.getPayload()), List.class); - assertThat(list).hasSize(5); - } - } - - @SpringBootConfiguration - @EnableAutoConfiguration - @Import(TwitterUsersFunctionConfiguration.class) - static class TwitterUsersFunctionTestApplication { - @Bean - @Primary - public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionProperties properties, - Function toConfigurationBuilder) { - - Function mockedConfiguration = - toConfigurationBuilder.andThen( - new TwitterTestUtils().mockTwitterUrls( - String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT))); - - return mockedConfiguration.apply(properties).build(); - } - } -} diff --git a/functions/function/twitter-function/src/test/resources/logback.xml b/functions/function/twitter-function/src/test/resources/logback.xml deleted file mode 100644 index f7e48fcff..000000000 --- a/functions/function/twitter-function/src/test/resources/logback.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - diff --git a/functions/function/twitter-function/src/test/resources/response/lookup_users_id.json b/functions/function/twitter-function/src/test/resources/response/lookup_users_id.json deleted file mode 100644 index afbae7826..000000000 --- a/functions/function/twitter-function/src/test/resources/response/lookup_users_id.json +++ /dev/null @@ -1 +0,0 @@ -[{"id":710705860343963648,"id_str":"710705860343963648","name":"Marto Tzolov","screen_name":"TzolovMarto","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":10,"friends_count":34,"listed_count":0,"created_at":"Fri Mar 18 05:54:16 +0000 2016","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"bg","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":326896547,"id_str":"326896547","name":"Mladen Tzolov","screen_name":"Rabotnik57","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":0,"friends_count":1,"listed_count":0,"created_at":"Thu Jun 30 17:25:54 +0000 2011","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":267603736,"id_str":"267603736","name":"Anatoli Tzolov","screen_name":"antzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":4,"friends_count":0,"listed_count":0,"created_at":"Thu Mar 17 06:40:31 +0000 2011","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":781497571629989888,"id_str":"781497571629989888","name":"Peyo Tzolov","screen_name":"peyo_tzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":1,"friends_count":34,"listed_count":0,"created_at":"Thu Sep 29 14:15:15 +0000 2016","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":838754923,"id_str":"838754923","name":"Ivan Stoyanov Tzolov","screen_name":"ivantzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":2,"friends_count":1,"listed_count":0,"created_at":"Fri Sep 21 23:43:31 +0000 2012","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":17,"lang":"en","status":{"created_at":"Fri Sep 12 08:47:45 +0000 2014","id":510349029784694784,"id_str":"510349029784694784","full_text":"http:\/\/t.co\/DAjlG8Nb2H","truncated":false,"display_text_range":[0,22],"entities":{"hashtags":[],"symbols":[],"user_mentions":[],"urls":[{"url":"http:\/\/t.co\/DAjlG8Nb2H","expanded_url":"http:\/\/startupmedia.net\/goodenough.html","display_url":"startupmedia.net\/goodenough.html","indices":[0,22]}]},"source":"\u003ca href=\"http:\/\/twitter.com\" rel=\"nofollow\"\u003eTwitter Web Client\u003c\/a\u003e","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":0,"favorite_count":0,"favorited":false,"retweeted":false,"possibly_sensitive":false,"lang":"und"},"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"}] diff --git a/functions/function/twitter-function/src/test/resources/response/search_places_amsterdam.json b/functions/function/twitter-function/src/test/resources/response/search_places_amsterdam.json deleted file mode 100644 index 6c0a1af0b..000000000 --- a/functions/function/twitter-function/src/test/resources/response/search_places_amsterdam.json +++ /dev/null @@ -1 +0,0 @@ -{"result":{"places":[{"id":"99cdab25eddd6bce","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/99cdab25eddd6bce.json","place_type":"city","name":"Amsterdam","full_name":"Amsterdam, The Netherlands","country_code":"NL","country":"The Netherlands","contained_within":[{"id":"00201664628f798b","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/00201664628f798b.json","place_type":"admin","name":"De Randstad","full_name":"De Randstad","country_code":"NL","country":"The Netherlands","centroid":[4.316128202618518,52.068423499999994],"bounding_box":{"type":"Polygon","coordinates":[[[4.112784,51.761243],[4.112784,52.52808],[5.468162,52.52808],[5.468162,51.761243],[4.112784,51.761243]]]},"attributes":{}}],"centroid":[4.8980371531271425,52.37482405],"bounding_box":{"type":"Polygon","coordinates":[[[4.7288999,52.2782266],[4.7288999,52.4312289],[5.0792072,52.4312289],[5.0792072,52.2782266],[4.7288999,52.2782266]]]},"attributes":{}},{"id":"4e542833600bca97","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/4e542833600bca97.json","place_type":"city","name":"Amsterdam","full_name":"Amsterdam, NY","country_code":"US","country":"United States","contained_within":[{"id":"4ffa03b662822a0e","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/4ffa03b662822a0e.json","place_type":"admin","name":"ALBANY-SCHENECTADY-TROY","full_name":"ALBANY-SCHENECTADY-TROY","country_code":"","country":"","centroid":[-73.88149182600827,43.049265000000005],"bounding_box":{"type":"Polygon","coordinates":[[[-74.867704,41.977978],[-74.867704,44.120552],[-72.819773,44.120552],[-72.819773,41.977978],[-74.867704,41.977978]]]},"attributes":{}}],"centroid":[-74.18907623554325,42.948039],"bounding_box":{"type":"Polygon","coordinates":[[[-74.222596,42.918987],[-74.222596,42.977091],[-74.158646,42.977091],[-74.158646,42.918987],[-74.222596,42.918987]]]},"attributes":{}},{"id":"014354e25fd7e37d","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/014354e25fd7e37d.json","place_type":"city","name":"New Amsterdam","full_name":"New Amsterdam, WI","country_code":"US","country":"United States","contained_within":[{"id":"f265fa733db1efa2","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/f265fa733db1efa2.json","place_type":"admin","name":"LA CROSSE-EAU CLAIRE","full_name":"LA CROSSE-EAU CLAIRE","country_code":"","country":"","centroid":[-91.11594598161932,44.313717999999994],"bounding_box":{"type":"Polygon","coordinates":[[[-92.31617,42.98817],[-92.31617,45.639266],[-90.311017,45.639266],[-90.311017,42.98817],[-92.31617,42.98817]]]},"attributes":{}}],"centroid":[-91.29723353577143,43.977616999999995],"bounding_box":{"type":"Polygon","coordinates":[[[-91.3194822,43.969888],[-91.3194822,43.985346],[-91.286008,43.985346],[-91.286008,43.969888],[-91.3194822,43.969888]]]},"attributes":{}},{"id":"6b1c779ec0b0bf96","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/6b1c779ec0b0bf96.json","place_type":"neighborhood","name":"Ámsterdam","full_name":"Ámsterdam, Corregidora","country_code":"MX","country":"Mexico","contained_within":[{"id":"ff1c0dd545aca772","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/ff1c0dd545aca772.json","place_type":"city","name":"Corregidora","full_name":"Corregidora, Querétaro Arteaga","country_code":"MX","country":"Mexico","centroid":[-100.44113916818549,20.4787551],"bounding_box":{"type":"Polygon","coordinates":[[[-100.508629,20.3617911],[-100.508629,20.5957191],[-100.370833,20.5957191],[-100.370833,20.3617911],[-100.508629,20.3617911]]]},"attributes":{}}],"centroid":[-100.41340904153098,20.5456092],"bounding_box":{"type":"Polygon","coordinates":[[[-100.414615,20.5443904],[-100.414615,20.546828],[-100.412591,20.546828],[-100.412591,20.5443904],[-100.414615,20.5443904]]]},"attributes":{}},{"id":"40277f690ea6c867","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/40277f690ea6c867.json","place_type":"neighborhood","name":"Oude Amsterdamsebuurt","full_name":"Oude Amsterdamsebuurt, Haarlem","country_code":"NL","country":"The Netherlands","contained_within":[{"id":"280b2451d4f8f73f","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/280b2451d4f8f73f.json","place_type":"city","name":"Haarlem","full_name":"Haarlem, Nederland","country_code":"NL","country":"The Netherlands","centroid":[4.643405250172844,52.3837897],"bounding_box":{"type":"Polygon","coordinates":[[[4.5993661,52.3388496],[4.5993661,52.4287298],[4.6866562,52.4287298],[4.6866562,52.3388496],[4.5993661,52.3388496]]]},"attributes":{}}],"centroid":[4.647010110383041,52.378719146890205],"bounding_box":{"type":"Polygon","coordinates":[[[4.64155453233553,52.3762651137569],[4.64155453233553,52.3811731800235],[4.65223371374939,52.3811731800235],[4.65223371374939,52.3762651137569],[4.64155453233553,52.3762651137569]]]},"attributes":{}},{"id":"60c449308fe5ce7b","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/60c449308fe5ce7b.json","place_type":"neighborhood","name":"Maarn waaronder Klein Amsterdam","full_name":"Maarn waaronder Klein Amsterdam, Utrechtse Heuvelrug","country_code":"NL","country":"The Netherlands","contained_within":[{"id":"9f3d46f4d4289d3d","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/9f3d46f4d4289d3d.json","place_type":"city","name":"Utrechtse Heuvelrug","full_name":"Utrechtse Heuvelrug, Nederland","country_code":"NL","country":"The Netherlands","centroid":[5.394745705687344,52.02860325],"bounding_box":{"type":"Polygon","coordinates":[[[5.2472216,51.9707539],[5.2472216,52.0864526],[5.5293757,52.0864526],[5.5293757,51.9707539],[5.2472216,51.9707539]]]},"attributes":{}}],"centroid":[5.369017195148887,52.0642160845185],"bounding_box":{"type":"Polygon","coordinates":[[[5.35037061367215,52.0576013365081],[5.35037061367215,52.0708308325289],[5.38919817898849,52.0708308325289],[5.38919817898849,52.0576013365081],[5.35037061367215,52.0576013365081]]]},"attributes":{}},{"id":"1e1900a11be73ee5","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/1e1900a11be73ee5.json","place_type":"neighborhood","name":"Gebied ten zuiden van Amsterdam-Rijnkanaal","full_name":"Gebied ten zuiden van Amsterdam-Rijnkanaal, Wijk bij Duurstede","country_code":"NL","country":"The Netherlands","contained_within":[{"id":"55e84f33f17942fa","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/55e84f33f17942fa.json","place_type":"city","name":"Wijk bij Duurstede","full_name":"Wijk bij Duurstede, Nederland","country_code":"NL","country":"The Netherlands","centroid":[5.329360942494974,51.990985699999996],"bounding_box":{"type":"Polygon","coordinates":[[[5.2426404,51.9549448],[5.2426404,52.0270266],[5.415259,52.0270266],[5.415259,51.9549448],[5.2426404,51.9549448]]]},"attributes":{}}],"centroid":[5.292265575333346,51.9678009541292],"bounding_box":{"type":"Polygon","coordinates":[[[5.25623018002343,51.9547209695773],[5.25623018002343,51.9808809386811],[5.34094161624555,51.9808809386811],[5.34094161624555,51.9547209695773],[5.25623018002343,51.9547209695773]]]},"attributes":{}},{"id":"208b7abc84e1d0bb","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/208b7abc84e1d0bb.json","place_type":"neighborhood","name":"Verspreide huizen Nieuw-Amsterdam","full_name":"Verspreide huizen Nieuw-Amsterdam, Emmen","country_code":"NL","country":"The Netherlands","contained_within":[{"id":"bd593b98d3277413","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/bd593b98d3277413.json","place_type":"city","name":"Emmen","full_name":"Emmen, Nederland","country_code":"NL","country":"The Netherlands","centroid":[6.952461959050538,52.75285365],"bounding_box":{"type":"Polygon","coordinates":[[[6.8263158,52.6327357],[6.8263158,52.8729716],[7.0926136,52.8729716],[7.0926136,52.6327357],[6.8263158,52.6327357]]]},"attributes":{}}],"centroid":[6.880029781032325,52.72497950062615],"bounding_box":{"type":"Polygon","coordinates":[[[6.84381703095638,52.6832729158951],[6.84381703095638,52.7368586649565],[6.90332685388762,52.7368586649565],[6.90332685388762,52.6832729158951],[6.84381703095638,52.6832729158951]]]},"attributes":{}},{"id":"34033fdb84866032","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/34033fdb84866032.json","place_type":"neighborhood","name":"Amsterdamscheveld","full_name":"Amsterdamscheveld, Emmen","country_code":"NL","country":"The Netherlands","contained_within":[{"id":"bd593b98d3277413","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/bd593b98d3277413.json","place_type":"city","name":"Emmen","full_name":"Emmen, Nederland","country_code":"NL","country":"The Netherlands","centroid":[6.952461959050538,52.75285365],"bounding_box":{"type":"Polygon","coordinates":[[[6.8263158,52.6327357],[6.8263158,52.8729716],[7.0926136,52.8729716],[7.0926136,52.6327357],[6.8263158,52.6327357]]]},"attributes":{}}],"centroid":[6.9184542815347765,52.6909569408471],"bounding_box":{"type":"Polygon","coordinates":[[[6.90917004254957,52.6839229502653],[6.90917004254957,52.6979909314289],[6.92513464497432,52.6979909314289],[6.92513464497432,52.6839229502653],[6.90917004254957,52.6839229502653]]]},"attributes":{}},{"id":"36b507ef2eebeced","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/36b507ef2eebeced.json","place_type":"neighborhood","name":"Nieuw-Amsterdam-Centrum","full_name":"Nieuw-Amsterdam-Centrum, Emmen","country_code":"NL","country":"The Netherlands","contained_within":[{"id":"bd593b98d3277413","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/bd593b98d3277413.json","place_type":"city","name":"Emmen","full_name":"Emmen, Nederland","country_code":"NL","country":"The Netherlands","centroid":[6.952461959050538,52.75285365],"bounding_box":{"type":"Polygon","coordinates":[[[6.8263158,52.6327357],[6.8263158,52.8729716],[7.0926136,52.8729716],[7.0926136,52.6327357],[6.8263158,52.6327357]]]},"attributes":{}}],"centroid":[6.859243233543406,52.70883893584595],"bounding_box":{"type":"Polygon","coordinates":[[[6.84660242442842,52.6924882873464],[6.84660242442842,52.7251895843455],[6.87877464927096,52.7251895843455],[6.87877464927096,52.6924882873464],[6.84660242442842,52.6924882873464]]]},"attributes":{}},{"id":"42d34e9faa2b4dc6","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/42d34e9faa2b4dc6.json","place_type":"neighborhood","name":"Amsterdamse Bos","full_name":"Amsterdamse Bos, Amstelveen","country_code":"NL","country":"The Netherlands","contained_within":[{"id":"be6da6fc44d6fd09","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/be6da6fc44d6fd09.json","place_type":"city","name":"Amstelveen","full_name":"Amstelveen, Nederland","country_code":"NL","country":"The Netherlands","centroid":[4.852805257748059,52.2862412],"bounding_box":{"type":"Polygon","coordinates":[[[4.7951199,52.2418745],[4.7951199,52.3306079],[4.9092753,52.3306079],[4.9092753,52.2418745],[4.7951199,52.2418745]]]},"attributes":{}}],"centroid":[4.828504535673222,52.310381530918306],"bounding_box":{"type":"Polygon","coordinates":[[[4.80951516342238,52.2906215346603],[4.80951516342238,52.3301415271763],[4.85648163248058,52.3301415271763],[4.85648163248058,52.2906215346603],[4.80951516342238,52.2906215346603]]]},"attributes":{}},{"id":"3c0675b1ab813077","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/3c0675b1ab813077.json","place_type":"city","name":"Amsterdam","full_name":"Amsterdam, South Africa","country_code":"ZA","country":"South Africa","contained_within":[{"id":"5679c21e1181c3a3","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/5679c21e1181c3a3.json","place_type":"admin","name":"Mpumalanga","full_name":"Mpumalanga","country_code":"ZA","country":"South Africa","centroid":[30.158989601575485,-25.731830000000002],"bounding_box":{"type":"Polygon","coordinates":[[[28.2434629,-27.5062538],[28.2434629,-23.9574062],[32.0339072,-23.9574062],[32.0339072,-27.5062538],[28.2434629,-27.5062538]]]},"attributes":{}}],"centroid":[30.665700090075823,-26.638494],"bounding_box":{"type":"Polygon","coordinates":[[[30.6473754,-26.6670719],[30.6473754,-26.6099161],[30.6860752,-26.6099161],[30.6860752,-26.6670719],[30.6473754,-26.6670719]]]},"attributes":{}}]},"query":{"url":"https:\/\/api.twitter.com\/1.1\/geo\/search.json?ip=127.0.0.1&query=AMSTERDAM&include_entities=true&include_ext_alt_text=true&tweet_mode=extended","type":"search","params":{"accuracy":0,"granularity":"neighborhood","query":"AMSTERDAM","autocomplete":false,"trim_place":false,"ip":"127.0.0.1"}}} diff --git a/functions/function/twitter-function/src/test/resources/response/search_users.json b/functions/function/twitter-function/src/test/resources/response/search_users.json deleted file mode 100644 index 6ad473bee..000000000 --- a/functions/function/twitter-function/src/test/resources/response/search_users.json +++ /dev/null @@ -1 +0,0 @@ -[{"id":710705860343963648,"id_str":"710705860343963648","name":"Marto Tzolov","screen_name":"TzolovMarto","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":10,"friends_count":34,"listed_count":0,"created_at":"Fri Mar 18 05:54:16 +0000 2016","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"bg","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":326896547,"id_str":"326896547","name":"Mladen Tzolov","screen_name":"Rabotnik57","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":0,"friends_count":1,"listed_count":0,"created_at":"Thu Jun 30 17:25:54 +0000 2011","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":267603736,"id_str":"267603736","name":"Anatoli Tzolov","screen_name":"antzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":4,"friends_count":0,"listed_count":0,"created_at":"Thu Mar 17 06:40:31 +0000 2011","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":781497571629989888,"id_str":"781497571629989888","name":"Peyo Tzolov","screen_name":"peyo_tzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":1,"friends_count":34,"listed_count":0,"created_at":"Thu Sep 29 14:15:15 +0000 2016","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":838754923,"id_str":"838754923","name":"Ivan Stoyanov Tzolov","screen_name":"ivantzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":2,"friends_count":1,"listed_count":0,"created_at":"Fri Sep 21 23:43:31 +0000 2012","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":17,"lang":"en","status":{"created_at":"Fri Sep 12 08:47:45 +0000 2014","id":510349029784694784,"id_str":"510349029784694784","full_text":"http:\/\/t.co\/DAjlG8Nb2H","truncated":false,"display_text_range":[0,22],"entities":{"hashtags":[],"symbols":[],"user_mentions":[],"urls":[{"url":"http:\/\/t.co\/DAjlG8Nb2H","expanded_url":"http:\/\/startupmedia.net\/goodenough.html","display_url":"startupmedia.net\/goodenough.html","indices":[0,22]}]},"source":"\u003ca href=\"http:\/\/twitter.com\" rel=\"nofollow\"\u003eTwitter Web Client\u003c\/a\u003e","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":0,"favorite_count":0,"favorited":false,"retweeted":false,"possibly_sensitive":false,"lang":"und"},"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":891056763964977152,"id_str":"891056763964977152","name":"Marto Tzolov","screen_name":"marto_tzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":0,"friends_count":1,"listed_count":0,"created_at":"Fri Jul 28 22:04:22 +0000 2017","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"bg","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":765259099,"id_str":"765259099","name":"Veronique Tzolov","screen_name":"Veronica_Tzolov","location":"","description":"Animal lover, love One Direction, fun and outgoing","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":2,"friends_count":12,"listed_count":0,"created_at":"Sat Aug 18 07:22:00 +0000 2012","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":1,"lang":"en","status":{"created_at":"Wed Jan 02 14:23:37 +0000 2013","id":286477854449995776,"id_str":"286477854449995776","full_text":"@Harry_Styles \nHey Harry... I'm sure you get this a lot but you seem like a really sweet and fun guy... I hope one day I'll get to meet you","truncated":false,"display_text_range":[0,139],"entities":{"hashtags":[],"symbols":[],"user_mentions":[{"screen_name":"Harry_Styles","name":"Harry Styles.","id":181561712,"id_str":"181561712","indices":[0,13]}],"urls":[]},"source":"\u003ca href=\"http:\/\/twitter.com\" rel=\"nofollow\"\u003eTwitter Web Client\u003c\/a\u003e","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":181561712,"in_reply_to_user_id_str":"181561712","in_reply_to_screen_name":"Harry_Styles","geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":0,"favorite_count":0,"favorited":false,"retweeted":false,"lang":"en"},"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":4236673876,"id_str":"4236673876","name":"Marto Tzolov","screen_name":"MartoTzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":10,"friends_count":59,"listed_count":0,"created_at":"Fri Nov 20 19:05:04 +0000 2015","favourites_count":43,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":1,"lang":"bg","status":{"created_at":"Sat Nov 21 16:54:02 +0000 2015","id":668110130723057664,"id_str":"668110130723057664","full_text":"https:\/\/t.co\/1sSG8HlbYP","truncated":false,"display_text_range":[0,23],"entities":{"hashtags":[],"symbols":[],"user_mentions":[],"urls":[{"url":"https:\/\/t.co\/1sSG8HlbYP","expanded_url":"https:\/\/youtu.be\/4oxrPkgUuFI","display_url":"youtu.be\/4oxrPkgUuFI","indices":[0,23]}]},"source":"\u003ca href=\"http:\/\/twitter.com\/download\/android\" rel=\"nofollow\"\u003eTwitter for Android\u003c\/a\u003e","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":1,"favorite_count":23,"favorited":false,"retweeted":false,"possibly_sensitive":false,"lang":"und"},"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":725201088262103040,"id_str":"725201088262103040","name":"Aleksandar Tzolov Ma","screen_name":"ma_tzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":0,"friends_count":0,"listed_count":0,"created_at":"Wed Apr 27 05:53:07 +0000 2016","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"bg","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":2191899364,"id_str":"2191899364","name":"Philip Tzolov","screen_name":"holymolyass","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":1,"friends_count":0,"listed_count":0,"created_at":"Fri Nov 22 21:06:24 +0000 2013","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":3367523121,"id_str":"3367523121","name":"Sharoma Tzolov","screen_name":"SharomaT","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":4,"friends_count":0,"listed_count":0,"created_at":"Thu Jul 09 12:43:28 +0000 2015","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":1272912426,"id_str":"1272912426","name":"Elisa Tzolov","screen_name":"Ellu_xx","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":4,"friends_count":3,"listed_count":0,"created_at":"Sat Mar 16 17:57:20 +0000 2013","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"fi","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":1440863107,"id_str":"1440863107","name":"Veronica Tzolov","screen_name":"VeronicaTzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":2,"friends_count":8,"listed_count":0,"created_at":"Sun May 19 10:45:38 +0000 2013","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":65257397,"id_str":"65257397","name":"Rosko Tzolov","screen_name":"oldblindpew","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":0,"friends_count":1,"listed_count":0,"created_at":"Thu Aug 13 03:14:32 +0000 2009","favourites_count":2,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":555653883,"id_str":"555653883","name":"Ves Tzolov","screen_name":"VesTzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":1,"friends_count":0,"listed_count":0,"created_at":"Tue Apr 17 01:29:31 +0000 2012","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":3388816924,"id_str":"3388816924","name":"Sharoma Tzolov","screen_name":"sharoma_tzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":11,"friends_count":197,"listed_count":0,"created_at":"Thu Jul 23 07:57:45 +0000 2015","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":1,"lang":"en","status":{"created_at":"Thu Jul 23 08:06:47 +0000 2015","id":624128509959630848,"id_str":"624128509959630848","full_text":"Pick me to win tickets to experience the first NBA game in Africa #947NBA http:\/\/t.co\/BEn2v37H7D #947NBA via @947","truncated":false,"display_text_range":[0,113],"entities":{"hashtags":[{"text":"947NBA","indices":[66,73]},{"text":"947NBA","indices":[97,104]}],"symbols":[],"user_mentions":[{"screen_name":"947","name":"947","id":41780491,"id_str":"41780491","indices":[109,113]}],"urls":[{"url":"http:\/\/t.co\/BEn2v37H7D","expanded_url":"http:\/\/bit.ly\/1Vazq3V","display_url":"bit.ly\/1Vazq3V","indices":[74,96]}]},"source":"\u003ca href=\"http:\/\/mobile.twitter.com\" rel=\"nofollow\"\u003eMobile Web\u003c\/a\u003e","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":0,"favorite_count":5,"favorited":false,"retweeted":false,"possibly_sensitive":false,"lang":"en"},"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":497663678,"id_str":"497663678","name":"Maria Tzolov","screen_name":"mariatzolov","location":"","description":"i like dylan o'brien and mac n cheese","url":"http:\/\/t.co\/Vp5OnuTrT0","entities":{"url":{"urls":[{"url":"http:\/\/t.co\/Vp5OnuTrT0","expanded_url":"http:\/\/purelymaria.tumblr.com","display_url":"purelymaria.tumblr.com","indices":[0,22]}]},"description":{"urls":[]}},"protected":false,"followers_count":83,"friends_count":235,"listed_count":0,"created_at":"Mon Feb 20 07:09:19 +0000 2012","favourites_count":41,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":428,"lang":"en","status":{"created_at":"Tue Jan 20 08:03:02 +0000 2015","id":557448200227803136,"id_str":"557448200227803136","full_text":"RT @zaynmalik: So were in NZ how's every1 doing ?","truncated":false,"display_text_range":[0,49],"entities":{"hashtags":[],"symbols":[],"user_mentions":[{"screen_name":"zaynmalik","name":"zayn","id":176566242,"id_str":"176566242","indices":[3,13]}],"urls":[]},"source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Thu Apr 19 06:57:07 +0000 2012","id":192869420853497856,"id_str":"192869420853497856","full_text":"So were in NZ how's every1 doing ?","truncated":false,"display_text_range":[0,34],"entities":{"hashtags":[],"symbols":[],"user_mentions":[],"urls":[]},"source":"\u003ca href=\"http:\/\/blackberry.com\/twitter\" rel=\"nofollow\"\u003eTwitter for BlackBerry\u00ae\u003c\/a\u003e","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":13160,"favorite_count":10940,"favorited":false,"retweeted":false,"lang":"en"},"is_quote_status":false,"retweet_count":13160,"favorite_count":0,"favorited":false,"retweeted":false,"lang":"en"},"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/517266103186120704\/JdAwOnwF_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/517266103186120704\/JdAwOnwF_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/497663678\/1412160774","profile_image_extensions_alt_text":null,"profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":false,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":467216441,"id_str":"467216441","name":"George Tzolov","screen_name":"gtzolov","location":"Bulgaria","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":true,"followers_count":4,"friends_count":175,"listed_count":1,"created_at":"Wed Jan 18 07:18:51 +0000 2012","favourites_count":34,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":168,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/841011507508965376\/lkmRA0q2_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/841011507508965376\/lkmRA0q2_normal.jpg","profile_image_extensions_alt_text":null,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":949949047,"id_str":"949949047","name":"Milen Tzolov","screen_name":"MilenTzolov","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":true,"followers_count":0,"friends_count":0,"listed_count":0,"created_at":"Thu Nov 15 15:28:24 +0000 2012","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":true,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/2852620531\/6eeafeb0c22030e59c2226a6a508fc74_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/2852620531\/6eeafeb0c22030e59c2226a6a508fc74_normal.jpeg","profile_image_extensions_alt_text":null,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"},{"id":454245310,"id_str":"454245310","name":"Simeon Tzolov","screen_name":"simeon190","location":"","description":"","url":null,"entities":{"description":{"urls":[]}},"protected":true,"followers_count":0,"friends_count":13,"listed_count":0,"created_at":"Tue Jan 03 19:44:35 +0000 2012","favourites_count":0,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":0,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"has_extended_profile":false,"default_profile":true,"default_profile_image":true,"following":false,"follow_request_sent":false,"notifications":false,"translator_type":"none"}] diff --git a/functions/function/twitter-function/src/test/resources/response/trends.json b/functions/function/twitter-function/src/test/resources/response/trends.json deleted file mode 100644 index 769015272..000000000 --- a/functions/function/twitter-function/src/test/resources/response/trends.json +++ /dev/null @@ -1 +0,0 @@ -[{"trends":[{"name":"Phillip Danault","url":"http:\/\/twitter.com\/search?q=%22Phillip+Danault%22","promoted_content":null,"query":"%22Phillip+Danault%22","tweet_volume":null},{"name":"Mali","url":"http:\/\/twitter.com\/search?q=Mali","promoted_content":null,"query":"Mali","tweet_volume":26504},{"name":"Paul Byron","url":"http:\/\/twitter.com\/search?q=%22Paul+Byron%22","promoted_content":null,"query":"%22Paul+Byron%22","tweet_volume":null},{"name":"Adonis Stevenson","url":"http:\/\/twitter.com\/search?q=%22Adonis+Stevenson%22","promoted_content":null,"query":"%22Adonis+Stevenson%22","tweet_volume":null},{"name":"Marcus Stroman","url":"http:\/\/twitter.com\/search?q=%22Marcus+Stroman%22","promoted_content":null,"query":"%22Marcus+Stroman%22","tweet_volume":null},{"name":"Mike Smith","url":"http:\/\/twitter.com\/search?q=%22Mike+Smith%22","promoted_content":null,"query":"%22Mike+Smith%22","tweet_volume":null},{"name":"Brandon Pirri","url":"http:\/\/twitter.com\/search?q=%22Brandon+Pirri%22","promoted_content":null,"query":"%22Brandon+Pirri%22","tweet_volume":null},{"name":"#WASvsTEN","url":"http:\/\/twitter.com\/search?q=%23WASvsTEN","promoted_content":null,"query":"%23WASvsTEN","tweet_volume":null},{"name":"#stlblues","url":"http:\/\/twitter.com\/search?q=%23stlblues","promoted_content":null,"query":"%23stlblues","tweet_volume":null},{"name":"Tatar","url":"http:\/\/twitter.com\/search?q=Tatar","promoted_content":null,"query":"Tatar","tweet_volume":null},{"name":"Bob Cole","url":"http:\/\/twitter.com\/search?q=%22Bob+Cole%22","promoted_content":null,"query":"%22Bob+Cole%22","tweet_volume":null},{"name":"Patrice Bergeron","url":"http:\/\/twitter.com\/search?q=%22Patrice+Bergeron%22","promoted_content":null,"query":"%22Patrice+Bergeron%22","tweet_volume":null},{"name":"#ResignTrump","url":"http:\/\/twitter.com\/search?q=%23ResignTrump","promoted_content":null,"query":"%23ResignTrump","tweet_volume":330688},{"name":"Christmas","url":"http:\/\/twitter.com\/search?q=Christmas","promoted_content":null,"query":"Christmas","tweet_volume":2824948},{"name":"Happy Holidays","url":"http:\/\/twitter.com\/search?q=%22Happy+Holidays%22","promoted_content":null,"query":"%22Happy+Holidays%22","tweet_volume":178360},{"name":"#WinterSolstice","url":"http:\/\/twitter.com\/search?q=%23WinterSolstice","promoted_content":null,"query":"%23WinterSolstice","tweet_volume":27358},{"name":"#The80sTaughtMe","url":"http:\/\/twitter.com\/search?q=%23The80sTaughtMe","promoted_content":null,"query":"%23The80sTaughtMe","tweet_volume":14044},{"name":"Atlas","url":"http:\/\/twitter.com\/search?q=Atlas","promoted_content":null,"query":"Atlas","tweet_volume":29402},{"name":"Liverpool","url":"http:\/\/twitter.com\/search?q=Liverpool","promoted_content":null,"query":"Liverpool","tweet_volume":141820},{"name":"manchester united","url":"http:\/\/twitter.com\/search?q=%22manchester+united%22","promoted_content":null,"query":"%22manchester+united%22","tweet_volume":93060},{"name":"Rocket","url":"http:\/\/twitter.com\/search?q=Rocket","promoted_content":null,"query":"Rocket","tweet_volume":23740},{"name":"F\u00EAtes","url":"http:\/\/twitter.com\/search?q=F%C3%AAtes","promoted_content":null,"query":"F%C3%AAtes","tweet_volume":47651},{"name":"Maple Valley","url":"http:\/\/twitter.com\/search?q=%22Maple+Valley%22","promoted_content":null,"query":"%22Maple+Valley%22","tweet_volume":null},{"name":"Omar Khadr","url":"http:\/\/twitter.com\/search?q=%22Omar+Khadr%22","promoted_content":null,"query":"%22Omar+Khadr%22","tweet_volume":null},{"name":"La Presse","url":"http:\/\/twitter.com\/search?q=%22La+Presse%22","promoted_content":null,"query":"%22La+Presse%22","tweet_volume":null},{"name":"Man City","url":"http:\/\/twitter.com\/search?q=%22Man+City%22","promoted_content":null,"query":"%22Man+City%22","tweet_volume":56236},{"name":"Yorkville","url":"http:\/\/twitter.com\/search?q=Yorkville","promoted_content":null,"query":"Yorkville","tweet_volume":null},{"name":"Kawhi Leonard","url":"http:\/\/twitter.com\/search?q=%22Kawhi+Leonard%22","promoted_content":null,"query":"%22Kawhi+Leonard%22","tweet_volume":null},{"name":"Lotto Max","url":"http:\/\/twitter.com\/search?q=%22Lotto+Max%22","promoted_content":null,"query":"%22Lotto+Max%22","tweet_volume":null},{"name":"Real Madrid","url":"http:\/\/twitter.com\/search?q=%22Real+Madrid%22","promoted_content":null,"query":"%22Real+Madrid%22","tweet_volume":120739},{"name":"White Rock","url":"http:\/\/twitter.com\/search?q=%22White+Rock%22","promoted_content":null,"query":"%22White+Rock%22","tweet_volume":null},{"name":"Pogba","url":"http:\/\/twitter.com\/search?q=Pogba","promoted_content":null,"query":"Pogba","tweet_volume":70788},{"name":"Rangers","url":"http:\/\/twitter.com\/search?q=Rangers","promoted_content":null,"query":"Rangers","tweet_volume":17174},{"name":"Arsenal","url":"http:\/\/twitter.com\/search?q=Arsenal","promoted_content":null,"query":"Arsenal","tweet_volume":151598},{"name":"Boxing Day","url":"http:\/\/twitter.com\/search?q=%22Boxing+Day%22","promoted_content":null,"query":"%22Boxing+Day%22","tweet_volume":23196},{"name":"York University","url":"http:\/\/twitter.com\/search?q=%22York+University%22","promoted_content":null,"query":"%22York+University%22","tweet_volume":null},{"name":"Slovakia","url":"http:\/\/twitter.com\/search?q=Slovakia","promoted_content":null,"query":"Slovakia","tweet_volume":null},{"name":"Finance Department","url":"http:\/\/twitter.com\/search?q=%22Finance+Department%22","promoted_content":null,"query":"%22Finance+Department%22","tweet_volume":null},{"name":"Chelsea","url":"http:\/\/twitter.com\/search?q=Chelsea","promoted_content":null,"query":"Chelsea","tweet_volume":97038},{"name":"New Year","url":"http:\/\/twitter.com\/search?q=%22New+Year%22","promoted_content":null,"query":"%22New+Year%22","tweet_volume":197747},{"name":"Bria Myles","url":"http:\/\/twitter.com\/search?q=%22Bria+Myles%22","promoted_content":null,"query":"%22Bria+Myles%22","tweet_volume":47871},{"name":"Parliament","url":"http:\/\/twitter.com\/search?q=Parliament","promoted_content":null,"query":"Parliament","tweet_volume":51213},{"name":"#BirdBox","url":"http:\/\/twitter.com\/search?q=%23BirdBox","promoted_content":null,"query":"%23BirdBox","tweet_volume":30984},{"name":"#BCstorm","url":"http:\/\/twitter.com\/search?q=%23BCstorm","promoted_content":null,"query":"%23BCstorm","tweet_volume":null},{"name":"#MCICRY","url":"http:\/\/twitter.com\/search?q=%23MCICRY","promoted_content":null,"query":"%23MCICRY","tweet_volume":43210},{"name":"#Caturday","url":"http:\/\/twitter.com\/search?q=%23Caturday","promoted_content":null,"query":"%23Caturday","tweet_volume":13967},{"name":"#CARMUN","url":"http:\/\/twitter.com\/search?q=%23CARMUN","promoted_content":null,"query":"%23CARMUN","tweet_volume":92394},{"name":"#Aquaman","url":"http:\/\/twitter.com\/search?q=%23Aquaman","promoted_content":null,"query":"%23Aquaman","tweet_volume":40278},{"name":"#ReasonsNotToBuildTheWall","url":"http:\/\/twitter.com\/search?q=%23ReasonsNotToBuildTheWall","promoted_content":null,"query":"%23ReasonsNotToBuildTheWall","tweet_volume":31522},{"name":"#AllStars4","url":"http:\/\/twitter.com\/search?q=%23AllStars4","promoted_content":null,"query":"%23AllStars4","tweet_volume":25908}],"as_of":"2018-12-30T16:38:56Z","created_at":"2018-12-22T23:47:56Z","locations":[{"name":"Winnipeg","woeid":2972}]}] diff --git a/functions/pom.xml b/functions/pom.xml deleted file mode 100644 index db3d8bee6..000000000 --- a/functions/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - org.springframework.cloud.fn - java-functions-parent - 5.0.0-SNAPSHOT - java-functions-parent - Java Functions Parent - pom - - - spring-functions-parent - common - consumer - function - supplier - function-dependencies - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.0.0 - - true - - - - - diff --git a/functions/spring-functions-parent/pom.xml b/functions/spring-functions-parent/pom.xml deleted file mode 100644 index 4fd7dd259..000000000 --- a/functions/spring-functions-parent/pom.xml +++ /dev/null @@ -1,104 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.stream.app - stream-applications-build - 5.0.0-SNAPSHOT - ../../stream-applications-build/pom.xml - - - org.springframework.cloud.fn - spring-functions-parent - pom - - 3.19.6 - - - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - org.springframework.cloud.fn - config-common - ${project.version} - - - - org.springframework.boot - spring-boot-configuration-processor - provided - - - org.springframework.boot - spring-boot-starter-validation - - - org.springframework.boot - spring-boot-starter-json - - - org.springframework.boot - spring-boot-starter-integration - - - com.jayway.jsonpath - json-path - - - - org.springframework.boot - spring-boot-starter-test - test - - - org.junit.jupiter - junit-jupiter - test - - - org.springframework.integration - spring-integration-test - test - - - io.projectreactor - reactor-test - test - - - org.testcontainers - testcontainers - ${testcontainers.version} - test - - - org.testcontainers - junit-jupiter - ${testcontainers.version} - test - - - org.awaitility - awaitility - test - - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.0.0 - - false - - - - - diff --git a/functions/supplier/debezium-supplier/pom.xml b/functions/supplier/debezium-supplier/pom.xml deleted file mode 100644 index a36df9fdb..000000000 --- a/functions/supplier/debezium-supplier/pom.xml +++ /dev/null @@ -1,146 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - debezium-supplier - debezium-supplier - Debezium Supplier - - - 2.3.3.Final - - - - - org.springframework.cloud.fn - debezium-autoconfigure - ${project.version} - - - - - io.debezium - debezium-connector-mysql - ${version.debezium} - - - slf4j-log4j12 - org.slf4j - - - - - io.debezium - debezium-connector-mongodb - ${version.debezium} - - - slf4j-log4j12 - org.slf4j - - - - - io.debezium - debezium-connector-postgres - ${version.debezium} - - - slf4j-log4j12 - org.slf4j - - - - - io.debezium - debezium-connector-oracle - ${version.debezium} - - - slf4j-log4j12 - org.slf4j - - - - - io.debezium - debezium-connector-sqlserver - ${version.debezium} - - - slf4j-log4j12 - org.slf4j - - - - - io.debezium - debezium-connector-db2 - ${version.debezium} - - - slf4j-log4j12 - org.slf4j - - - - - io.debezium - debezium-connector-vitess - ${version.debezium} - - - slf4j-log4j12 - org.slf4j - - - - - io.debezium - debezium-connector-spanner - ${version.debezium} - - - slf4j-log4j12 - org.slf4j - - - true - - - - - com.zaxxer - HikariCP - 4.0.3 - test - - - org.springframework - spring-jdbc - test - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 3.0.0 - - 1 - 1 - integration - - - - - diff --git a/functions/supplier/debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/debezium/DebeziumReactiveConsumerConfiguration.java b/functions/supplier/debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/debezium/DebeziumReactiveConsumerConfiguration.java deleted file mode 100644 index ed8ad039a..000000000 --- a/functions/supplier/debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/debezium/DebeziumReactiveConsumerConfiguration.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.debezium; - -import java.lang.reflect.Field; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Consumer; -import java.util.function.Supplier; - -import io.debezium.engine.ChangeEvent; -import io.debezium.engine.DebeziumEngine; -import io.debezium.engine.DebeziumEngine.Builder; -import io.debezium.engine.Header; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Sinks; - -import org.springframework.beans.factory.BeanClassLoaderAware; -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.debezium.DebeziumEngineBuilderAutoConfiguration; -import org.springframework.cloud.fn.common.debezium.DebeziumProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.util.ClassUtils; -import org.springframework.util.MimeTypeUtils; - -/** - * - * @author Christian Tzolov - * @author Artem Bilan - */ -@AutoConfiguration(after = DebeziumEngineBuilderAutoConfiguration.class) -@EnableConfigurationProperties({ DebeziumSupplierProperties.class }) -@ConditionalOnBean(DebeziumEngine.Builder.class) -public class DebeziumReactiveConsumerConfiguration implements BeanClassLoaderAware { - - private static final Log logger = LogFactory.getLog(DebeziumReactiveConsumerConfiguration.class); - /** - * ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL. - */ - public static final String ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL = "org.springframework.kafka.support.KafkaNull"; - - private Object kafkaNull = null; - - @Override - public void setBeanClassLoader(ClassLoader classLoader) { - try { - Class clazz = ClassUtils.forName(ORG_SPRINGFRAMEWORK_KAFKA_SUPPORT_KAFKA_NULL, classLoader); - Field field = clazz.getDeclaredField("INSTANCE"); - this.kafkaNull = field.get(null); - } - catch (ClassNotFoundException | NoSuchFieldException | IllegalAccessException e) { - } - } - - /** - * Reactive Streams, single subscriber, sink used to push down the change event signals received from the Debezium - * Engine. - */ - private final Sinks.Many> eventSink = Sinks.many().unicast().onBackpressureError(); - - /** - * Debezium Engine is designed to be submitted to an {@link ExecutorService} for execution by a single thread, and a - * running connector can be stopped either by calling {@link #stop()} from another thread or by interrupting the - * running thread (e.g., as is the case with {@link ExecutorService#shutdownNow()}). - */ - private final ExecutorService debeziumExecutor = Executors.newSingleThreadExecutor(); - - @Bean - public DebeziumEngine> debeziumEngine( - Consumer> changeEventConsumer, - Builder> debeziumEngineBuilder) { - - return debeziumEngineBuilder.notifying(changeEventConsumer).build(); - } - - @Bean - public Supplier>> debeziumSupplier(DebeziumEngine> debeziumEngine) { - - return () -> this.eventSink.asFlux() - .doOnRequest(r -> debeziumExecutor.execute(debeziumEngine)) - .doOnTerminate(debeziumExecutor::shutdownNow); - } - - @Bean - @ConditionalOnMissingBean - public Consumer> changeEventConsumer(DebeziumProperties engineProperties, - DebeziumSupplierProperties supplierProperties) { - - return new ChangeEventConsumer(engineProperties.getPayloadFormat().contentType(), - supplierProperties.isCopyHeaders(), - this.eventSink); - } - - /** - * Format-agnostic change event consumer. - */ - private final class ChangeEventConsumer implements Consumer> { - - private final String contentType; - private final boolean copyHeaders; - private final Sinks.Many> eventSink; - - private ChangeEventConsumer(String contentType, boolean copyHeaders, Sinks.Many> eventSink) { - this.contentType = contentType; - this.copyHeaders = copyHeaders; - this.eventSink = eventSink; - } - - @Override - public void accept(ChangeEvent changeEvent) { - if (logger.isDebugEnabled()) { - logger.debug("[Debezium Event]: " + changeEvent.key()); - } - - Object key = changeEvent.key(); - Object payload = changeEvent.value(); - String destination = changeEvent.destination(); - - // When the tombstone event is enabled, Debezium serializes the payload to null (e.g. empty payload) - // while the metadata information is carried through the headers (debezium_key). - // Note: Event for none flattened responses, when the debezium.properties.tombstones.on.delete=true - // (default), tombstones are generate by Debezium and handled by the code below. - if (payload == null) { - payload = DebeziumReactiveConsumerConfiguration.this.kafkaNull; - } - - // If payload is still null ignore the message. - if (payload == null) { - logger.info("Dropped null payload message"); - return; - } - - MessageBuilder messageBuilder = MessageBuilder - .withPayload(payload) - .setHeader("debezium_key", key) - .setHeader("debezium_destination", destination) - .setHeader(MessageHeaders.CONTENT_TYPE, - (payload.equals(DebeziumReactiveConsumerConfiguration.this.kafkaNull)) - ? MimeTypeUtils.TEXT_PLAIN_VALUE - : this.contentType); - - if (this.copyHeaders) { - List> headers = changeEvent.headers(); - if (headers != null && !headers.isEmpty()) { - Iterator> itr = headers.iterator(); - while (itr.hasNext()) { - Header header = itr.next(); - messageBuilder.setHeader(header.getKey(), header.getValue()); - } - } - } - - this.eventSink.tryEmitNext(messageBuilder.build()); - } - } - -} diff --git a/functions/supplier/debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/debezium/DebeziumSupplierProperties.java b/functions/supplier/debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/debezium/DebeziumSupplierProperties.java deleted file mode 100644 index 55ab2018d..000000000 --- a/functions/supplier/debezium-supplier/src/main/java/org/springframework/cloud/fn/supplier/debezium/DebeziumSupplierProperties.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.debezium; - -import org.springframework.boot.context.properties.ConfigurationProperties; - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("debezium.supplier") -public class DebeziumSupplierProperties { - - /** - * Copy Change Event headers into Message headers. - */ - private boolean copyHeaders = true; - - public boolean isCopyHeaders() { - return copyHeaders; - } - - public void setCopyHeaders(boolean copyHeaders) { - this.copyHeaders = copyHeaders; - } -} diff --git a/functions/supplier/debezium-supplier/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/supplier/debezium-supplier/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index 2c4a031c1..000000000 --- a/functions/supplier/debezium-supplier/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.supplier.debezium.DebeziumReactiveConsumerConfiguration diff --git a/functions/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/DebeziumReactiveConsumerConfigurationTests.java b/functions/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/DebeziumReactiveConsumerConfigurationTests.java deleted file mode 100644 index a11cefcf8..000000000 --- a/functions/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/DebeziumReactiveConsumerConfigurationTests.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.debezium.it.supplier; - -import io.debezium.engine.DebeziumEngine; -import org.junit.jupiter.api.Test; - -import org.springframework.boot.autoconfigure.AutoConfigurations; -import org.springframework.boot.test.context.FilteredClassLoader; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.cloud.fn.common.debezium.DebeziumEngineBuilderAutoConfiguration; -import org.springframework.cloud.fn.supplier.debezium.DebeziumReactiveConsumerConfiguration; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Tests for {@link DebeziumReactiveConsumerConfiguration}. - * - * @author Christian Tzolov - */ -public class DebeziumReactiveConsumerConfigurationTests { - - private final ApplicationContextRunner contextRunner = new ApplicationContextRunner() - .withConfiguration(AutoConfigurations.of(DebeziumEngineBuilderAutoConfiguration.class, - DebeziumReactiveConsumerConfiguration.class)); - - // We have the debezium connectors on the classpath by default. - - @Test - void noConnectorNoProperty() { - this.contextRunner.run((context) -> { - assertThat(context).doesNotHaveBean(DebeziumEngine.class); - }); - } - - @Test - void noConnectorWithProperty() { - this.contextRunner.withPropertyValues("debezium.properties.connector.class=Dummy") - .withClassLoader(new FilteredClassLoader("io.debezium.engine.DebeziumEngine$Builder")) - .run((context) -> { - assertThat(context).doesNotHaveBean(DebeziumEngine.class); - }); - } - - @Test - void withConnectorWithProperty() { - this.contextRunner.withPropertyValues("debezium.properties.connector.class=Dummy").run((context) -> { - assertThat(context).hasSingleBean(DebeziumEngine.class); - }); - } - -} diff --git a/functions/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/DebeziumSupplierIntegrationTest.java b/functions/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/DebeziumSupplierIntegrationTest.java deleted file mode 100644 index f19dc99ef..000000000 --- a/functions/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/DebeziumSupplierIntegrationTest.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.debezium.it.supplier; - -import java.util.function.Supplier; - -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Import; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.messaging.Message; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { - "spring.cloud.function.definition=debeziumSupplier", - - // https://debezium.io/documentation/reference/transformations/event-flattening.html - "debezium.properties.transforms=unwrap", - "debezium.properties.transforms.unwrap.type=io.debezium.transforms.ExtractNewRecordState", - "debezium.properties.transforms.unwrap.drop.tombstones=true", - "debezium.properties.transforms.unwrap.delete.handling.mode=rewrite", - "debezium.properties.transforms.unwrap.add.fields=name,db,op,table", - - "debezium.properties.schema.history.internal=io.debezium.relational.history.MemorySchemaHistory", - "debezium.properties.offset.storage=org.apache.kafka.connect.storage.MemoryOffsetBackingStore", - - // Drop schema from the message payload. - "debezium.properties.key.converter.schemas.enable=false", - "debezium.properties.value.converter.schemas.enable=false", - - "debezium.properties.topic.prefix=my-topic", - "debezium.properties.name=my-connector", - "debezium.properties.database.server.id=85744", - "debezium.properties.connector.class=io.debezium.connector.mysql.MySqlConnector", - "debezium.properties.database.user=debezium", - "debezium.properties.database.password=dbz", - "debezium.properties.database.hostname=localhost", - - "debezium.properties.table.include.list=inventory.customers, inventory.addresses", - - // JdbcTemplate config. - "app.datasource.username=root", - "app.datasource.password=debezium", - "app.datasource.driver-class-name=com.mysql.cj.jdbc.Driver", - "app.datasource.type=com.zaxxer.hikari.HikariDataSource" -}) -@Testcontainers -public class DebeziumSupplierIntegrationTest { - - public static final String IMAGE_TAG = "2.3.3.Final"; - public static final String DEBEZIUM_EXAMPLE_MYSQL_IMAGE = "debezium/example-mysql:" + IMAGE_TAG; - - @Container - static GenericContainer debeziumMySQL = new GenericContainer<>(DEBEZIUM_EXAMPLE_MYSQL_IMAGE) - .withEnv("MYSQL_ROOT_PASSWORD", "debezium") - .withEnv("MYSQL_USER", "mysqluser") - .withEnv("MYSQL_PASSWORD", "mysqlpw") - .withExposedPorts(3306); - - @DynamicPropertySource - static void mysqlDbProperties(DynamicPropertyRegistry registry) { - registry.add("debezium.properties.database.port", () -> debeziumMySQL.getMappedPort(3306)); - registry.add("app.datasource.url", - () -> String.format("jdbc:mysql://localhost:%d/%s?enabledTLSProtocols=TLSv1.2", - debeziumMySQL.getMappedPort(3306), "inventory")); // JdbcTemplate config. - } - - @Autowired - private Supplier>> debeziumSupplier; - - @Autowired - private JdbcTemplate jdbcTemplate; - - @Test - void testDebeziumSupplier() { - - jdbcTemplate.update( - "INSERT INTO `customers`(`first_name`,`last_name`,`email`) VALUES('Test666', 'Test666', 'Test666@spring.org')"); - - Flux> messageFlux = this.debeziumSupplier.get(); - - // Message size should correspond to the number of insert statements in: - // https://github.com/debezium/container-images/blob/main/examples/mysql/2.3/inventory.sql - // filtered by Customers and Addresses table. - StepVerifier.create(messageFlux) - .expectNextCount(16) // Skip the DDL transaction logs. - - // Customers table - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":1001,\"first_name\":\"Sally\",\"last_name\":\"Thomas\",\"email\":\"sally.thomas@acme.com\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"customers\",\"__deleted\":\"false\"}")) - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":1002,\"first_name\":\"George\",\"last_name\":\"Bailey\",\"email\":\"gbailey@foobar.com\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"customers\",\"__deleted\":\"false\"}")) - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":1003,\"first_name\":\"Edward\",\"last_name\":\"Walker\",\"email\":\"ed@walker.com\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"customers\",\"__deleted\":\"false\"}")) - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":1004,\"first_name\":\"Anne\",\"last_name\":\"Kretchmar\",\"email\":\"annek@noanswer.org\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"customers\",\"__deleted\":\"false\"}")) - - // NEW Customer Insert - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":1005,\"first_name\":\"Test666\",\"last_name\":\"Test666\",\"email\":\"Test666@spring.org\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"customers\",\"__deleted\":\"false\"}")) - - // Addresses table - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":10,\"customer_id\":1001,\"street\":\"3183 Moore Avenue\",\"city\":\"Euless\",\"state\":\"Texas\",\"zip\":\"76036\",\"type\":\"SHIPPING\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"addresses\",\"__deleted\":\"false\"}")) - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":11,\"customer_id\":1001,\"street\":\"2389 Hidden Valley Road\",\"city\":\"Harrisburg\",\"state\":\"Pennsylvania\",\"zip\":\"17116\",\"type\":\"BILLING\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"addresses\",\"__deleted\":\"false\"}")) - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":12,\"customer_id\":1002,\"street\":\"281 Riverside Drive\",\"city\":\"Augusta\",\"state\":\"Georgia\",\"zip\":\"30901\",\"type\":\"BILLING\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"addresses\",\"__deleted\":\"false\"}")) - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":13,\"customer_id\":1003,\"street\":\"3787 Brownton Road\",\"city\":\"Columbus\",\"state\":\"Mississippi\",\"zip\":\"39701\",\"type\":\"SHIPPING\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"addresses\",\"__deleted\":\"false\"}")) - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":14,\"customer_id\":1003,\"street\":\"2458 Lost Creek Road\",\"city\":\"Bethlehem\",\"state\":\"Pennsylvania\",\"zip\":\"18018\",\"type\":\"SHIPPING\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"addresses\",\"__deleted\":\"false\"}")) - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":15,\"customer_id\":1003,\"street\":\"4800 Simpson Square\",\"city\":\"Hillsdale\",\"state\":\"Oklahoma\",\"zip\":\"73743\",\"type\":\"BILLING\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"addresses\",\"__deleted\":\"false\"}")) - .assertNext((message) -> assertThat(payloadString(message)) - .isEqualTo( - "{\"id\":16,\"customer_id\":1004,\"street\":\"1289 University Hill Road\",\"city\":\"Canehill\",\"state\":\"Arkansas\",\"zip\":\"72717\",\"type\":\"LIVING\",\"__name\":\"my-topic\",\"__db\":\"inventory\",\"__op\":\"r\",\"__table\":\"addresses\",\"__deleted\":\"false\"}")) - .thenCancel() - .verify(); - - } - - private String payloadString(Message message) { - return new String((byte[]) message.getPayload()); - } - - @SpringBootApplication(exclude = { MongoAutoConfiguration.class }) - @Import({TestJdbcTemplateConfiguration.class}) - static class DebeziumSupplierTestApplication { - } - -} diff --git a/functions/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/TestJdbcTemplateConfiguration.java b/functions/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/TestJdbcTemplateConfiguration.java deleted file mode 100644 index 9e7918ae4..000000000 --- a/functions/supplier/debezium-supplier/src/test/java/org/springframework/cloud/fn/supplier/debezium/it/supplier/TestJdbcTemplateConfiguration.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.debezium.it.supplier; - -import javax.sql.DataSource; - -import com.zaxxer.hikari.HikariDataSource; - -import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.core.JdbcTemplate; - -/** - * - * @author Christian Tzolov - */ -@Configuration -public class TestJdbcTemplateConfiguration { - @Bean - public JdbcTemplate myJdbcTemplate(DataSource dataSource) { - return new JdbcTemplate(dataSource); - } - - @Bean - @Primary - @ConfigurationProperties("app.datasource") - public DataSourceProperties dataSourceProperties() { - return new DataSourceProperties(); - } - - @Bean - public HikariDataSource dataSource(DataSourceProperties dataSourceProperties) { - return dataSourceProperties.initializeDataSourceBuilder() - .type(HikariDataSource.class) - .build(); - } - -} diff --git a/functions/supplier/debezium-supplier/src/test/resources/logback-test.xml b/functions/supplier/debezium-supplier/src/test/resources/logback-test.xml deleted file mode 100644 index 227674db7..000000000 --- a/functions/supplier/debezium-supplier/src/test/resources/logback-test.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n - - - - - - - - - - diff --git a/functions/supplier/file-supplier/README.adoc b/functions/supplier/file-supplier/README.adoc deleted file mode 100644 index 1179253c9..000000000 --- a/functions/supplier/file-supplier/README.adoc +++ /dev/null @@ -1,37 +0,0 @@ -# File Supplier - -This module provides a File supplier that can be reused and composed in other applications. -The `Supplier` uses the `FileInboundChannelAdapter` from Spring Integration. -`FileSupplier` is implemented as a `java.util.function.Supplier`. -This supplier gives you a reactive stream of files from the provided directory as the supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and receive the data. - -## Beans for injection - -You can import the `FileSupplierConfiguration` in the application and then inject the following bean. - -`fileSupplier` - -You need to inject this as `Supplier>>`. - -You can use `fileSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `file.supplier`. -There are also properties that need to be used with the prefix `file.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierProperties.java[FileSupplierProperties]. -See link:src/main/java/org/springframework/cloud/fn/supplier/file/FileConsumerProperties.java[this] also. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `FileInboundChannelAdapterSpec` configuration used by the `fileSupplier`. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/file[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/file-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a File Source. diff --git a/functions/supplier/file-supplier/pom.xml b/functions/supplier/file-supplier/pom.xml deleted file mode 100644 index dd7e65210..000000000 --- a/functions/supplier/file-supplier/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - file-supplier - file-supplier - file supplier - - - - org.springframework.integration - spring-integration-file - - - org.springframework.cloud.fn - metadata-store-common - ${project.version} - - - org.springframework.cloud.fn - file-common - ${project.version} - - - - org.springframework.integration - spring-integration-jdbc - test - - - org.springframework.boot - spring-boot-starter-jdbc - test - - - org.hsqldb - hsqldb - test - - - - diff --git a/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java deleted file mode 100644 index da8dfc4c9..000000000 --- a/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierConfiguration.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2020-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.file; - -import java.io.File; -import java.util.function.Supplier; - -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.cloud.fn.common.file.FileConsumerProperties; -import org.springframework.cloud.fn.common.file.FileReadingMode; -import org.springframework.cloud.fn.common.file.FileUtils; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Lazy; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.dsl.IntegrationFlowBuilder; -import org.springframework.integration.file.FileReadingMessageSource; -import org.springframework.integration.file.dsl.FileInboundChannelAdapterSpec; -import org.springframework.integration.file.dsl.Files; -import org.springframework.integration.file.filters.ChainFileListFilter; -import org.springframework.integration.file.filters.FileListFilter; -import org.springframework.integration.file.filters.FileSystemPersistentAcceptOnceFileListFilter; -import org.springframework.integration.file.filters.RegexPatternFileListFilter; -import org.springframework.integration.file.filters.SimplePatternFileListFilter; -import org.springframework.integration.metadata.ConcurrentMetadataStore; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.util.StringUtils; - -/** - * @author Artem Bilan - * @author Soby Chacko - */ -@Configuration -@EnableConfigurationProperties({FileSupplierProperties.class, FileConsumerProperties.class}) -public class FileSupplierConfiguration { - - private static final String METADATA_STORE_PREFIX = "local-file-system-metadata-"; - - private final FileSupplierProperties fileSupplierProperties; - - private final FileConsumerProperties fileConsumerProperties; - - @Autowired - @Lazy - @Qualifier("fileMessageSource") - private FileReadingMessageSource fileMessageSource; - - public FileSupplierConfiguration(FileSupplierProperties fileSupplierProperties, - FileConsumerProperties fileConsumerProperties) { - - this.fileSupplierProperties = fileSupplierProperties; - this.fileConsumerProperties = fileConsumerProperties; - } - - @Bean - public ChainFileListFilter filter(ConcurrentMetadataStore metadataStore) { - ChainFileListFilter chainFilter = new ChainFileListFilter<>(); - if (StringUtils.hasText(this.fileSupplierProperties.getFilenamePattern())) { - chainFilter.addFilter(new SimplePatternFileListFilter(this.fileSupplierProperties.getFilenamePattern())); - } - else if (this.fileSupplierProperties.getFilenameRegex() != null) { - chainFilter.addFilter(new RegexPatternFileListFilter(this.fileSupplierProperties.getFilenameRegex())); - } - - if (this.fileSupplierProperties.isPreventDuplicates()) { - chainFilter.addFilter(new FileSystemPersistentAcceptOnceFileListFilter(metadataStore, METADATA_STORE_PREFIX)); - } - - return chainFilter; - } - - @Bean - public FileInboundChannelAdapterSpec fileMessageSource(FileListFilter fileListFilter, - @Nullable ComponentCustomizer fileInboundChannelAdapterSpecCustomizer) { - - FileInboundChannelAdapterSpec adapterSpec = - Files.inboundAdapter(this.fileSupplierProperties.getDirectory()) - .filter(fileListFilter); - if (fileInboundChannelAdapterSpecCustomizer != null) { - fileInboundChannelAdapterSpecCustomizer.customize(adapterSpec); - } - return adapterSpec; - } - - @Bean - public Flux> fileMessageFlux() { - return Mono.>create(monoSink -> - monoSink.onRequest(value -> - monoSink.success(this.fileMessageSource.receive()))) - .subscribeOn(Schedulers.boundedElastic()) - .repeatWhenEmpty(it -> it.delayElements(this.fileSupplierProperties.getDelayWhenEmpty())) - .repeat() - .doOnRequest(r -> this.fileMessageSource.start()); - } - - @Bean - @ConditionalOnExpression("environment['file.consumer.mode'] != 'ref'") - public Publisher> fileReadingFlow() { - IntegrationFlowBuilder flowBuilder = IntegrationFlow.from(fileMessageFlux()); - return FileUtils.enhanceFlowForReadingMode(flowBuilder, this.fileConsumerProperties) - .toReactivePublisher(); - } - - @Bean - public Supplier>> fileSupplier() { - if (this.fileConsumerProperties.getMode() == FileReadingMode.ref) { - return this::fileMessageFlux; - } - else { - return () -> Flux.from(fileReadingFlow()); - } - } - -} diff --git a/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierProperties.java b/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierProperties.java deleted file mode 100644 index d0870b644..000000000 --- a/functions/supplier/file-supplier/src/main/java/org/springframework/cloud/fn/supplier/file/FileSupplierProperties.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.file; - -import java.io.File; -import java.time.Duration; -import java.util.regex.Pattern; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * Properties for the file supplier. - * - * @author Gary Russell - * @author Artem Bilan - * @author Soby Chacko - */ -@ConfigurationProperties("file.supplier") -@Validated -public class FileSupplierProperties { - - private static final String DEFAULT_DIR = System.getProperty("java.io.tmpdir") + - File.separator + "file-supplier"; - - /** - * The directory to poll for new files. - */ - private File directory = new File(DEFAULT_DIR); - - /** - * Set to true to include an AcceptOnceFileListFilter which prevents duplicates. - */ - private boolean preventDuplicates = true; - - /** - * A simple ant pattern to match files. - */ - private String filenamePattern; - - /** - * A regex pattern to match files. - */ - private Pattern filenameRegex; - - /** - * Duration of delay when no new files are detected. - */ - private Duration delayWhenEmpty = Duration.ofSeconds(1); - - public File getDirectory() { - return this.directory; - } - - public void setDirectory(File directory) { - this.directory = directory; - } - - public boolean isPreventDuplicates() { - return this.preventDuplicates; - } - - public void setPreventDuplicates(boolean preventDuplicates) { - this.preventDuplicates = preventDuplicates; - } - - public String getFilenamePattern() { - return this.filenamePattern; - } - - public void setFilenamePattern(String filenamePattern) { - this.filenamePattern = filenamePattern; - } - - public Pattern getFilenameRegex() { - return this.filenameRegex; - } - - public void setFilenameRegex(Pattern filenameRegex) { - this.filenameRegex = filenameRegex; - } - - //@AssertTrue(message = "filenamePattern and filenameRegex are mutually exclusive") - - public boolean isExclusivePatterns() { - return !(this.filenamePattern != null && this.filenameRegex != null); - } - - public Duration getDelayWhenEmpty() { - return delayWhenEmpty; - } - - public void setDelayWhenEmpty(Duration delayWhenEmpty) { - this.delayWhenEmpty = delayWhenEmpty; - } - - -} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/AbstractFileSupplierTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/AbstractFileSupplierTests.java deleted file mode 100644 index aa3c1e5b6..000000000 --- a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/AbstractFileSupplierTests.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.file; - -import java.nio.file.Path; -import java.util.Date; -import java.util.function.Supplier; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.io.TempDir; -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.integration.file.FileReadingMessageSource; -import org.springframework.integration.file.dsl.FileInboundChannelAdapterSpec; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; - -/** - * @author Gary Russell - * @author Artem Bilan - * @author Soby Chacko - */ -@SpringBootTest -@DirtiesContext -public class AbstractFileSupplierTests { - - @TempDir - static Path tempDir; - - @Autowired - Supplier>> fileSupplier; - - @BeforeAll - public static void beforeAll() { - System.setProperty("file.supplier.directory", tempDir.toAbsolutePath().toString()); - } - - @AfterAll - public static void afterAll() { - System.clearProperty("file.supplier.directory"); - } - - @SpringBootApplication - static class FileSupplierTestApplication { - - @Bean - ComponentCustomizer fileInboundChannelAdapterSpecCustomizer() { - return (adapterSpec) -> adapterSpec.watchEvents(FileReadingMessageSource.WatchEventType.DELETE); - } - - @Bean - ComponentCustomizer fakeCustomizer() { - return (date) -> { - throw new RuntimeException("Must not happen"); - }; - } - - } - -} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java deleted file mode 100644 index ad11e8872..000000000 --- a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/DefaultFileSupplierTests.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.file; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.integration.file.FileHeaders; -import org.springframework.integration.file.FileReadingMessageSource; -import org.springframework.integration.jdbc.metadata.JdbcMetadataStore; -import org.springframework.integration.metadata.ConcurrentMetadataStore; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Gary Russell - * @author Artem Bilan - * @author Soby Chacko - */ -@TestPropertySource(properties = "metadata.store.type = jdbc") -public class DefaultFileSupplierTests extends AbstractFileSupplierTests { - - @Autowired - @Qualifier("fileMessageSource") - private FileReadingMessageSource fileMessageSource; - - @Autowired - private ConcurrentMetadataStore metadataStore; - - @Autowired - private JdbcTemplate jdbcTemplate; - - @Test - public void testBasicFlow() throws IOException { - Path firstFile = tempDir.resolve("first.file"); - Files.write(firstFile, "first.file".getBytes()); - - final Flux> messageFlux = fileSupplier.get(); - - //create file after subscription - Path tempFile = tempDir.resolve("test.file"); - - StepVerifier stepVerifier = - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(message.getPayload()) - .isEqualTo("first.file".getBytes()); - assertThat(message.getHeaders()) - .containsEntry(FileHeaders.FILENAME, "first.file"); - assertThat(message.getHeaders()) - .containsEntry(FileHeaders.RELATIVE_PATH, "first.file"); - assertThat(message.getHeaders()) - .containsEntry(FileHeaders.ORIGINAL_FILE, firstFile.toFile()); - } - ) - .assertNext((message) -> { - assertThat(message.getPayload()) - .isEqualTo("testing".getBytes()); - assertThat(message.getHeaders()) - .containsEntry(FileHeaders.FILENAME, "test.file"); - assertThat(message.getHeaders()) - .containsEntry(FileHeaders.RELATIVE_PATH, "test.file"); - assertThat(message.getHeaders()) - .containsEntry(FileHeaders.ORIGINAL_FILE, tempFile.toFile()); - }) - .thenCancel() - .verifyLater(); - Files.write(tempFile, "testing".getBytes()); - stepVerifier.verify(); - - assertThat(this.fileMessageSource.isRunning()).isTrue(); - - assertThat(this.metadataStore).isInstanceOf(JdbcMetadataStore.class); - - List metadataStoreContent = - jdbcTemplate.queryForList("SELECT metadata_key FROM int_metadata_store", String.class); - - assertThat(metadataStoreContent).hasSize(2); - assertThat(metadataStoreContent.get(0)).startsWith("local-file-system-metadata-"); - assertThat(metadataStoreContent.get(0)).endsWith("first.file"); - assertThat(metadataStoreContent.get(1)).endsWith("test.file"); - - /* See AbstractFileSupplierTests.FileSupplierTestApplication.fileInboundChannelAdapterSpecCustomizer() - - through the ComponentCustomizer and @CustomizationAware on the FileSupplierConfiguration.fileMessageSource() - the provided customization is populated down to the bean under testing. - */ - assertThat(TestUtils.getPropertyValue(this.fileMessageSource, "watchEvents", - FileReadingMessageSource.WatchEventType[].class)) - .isEqualTo(new FileReadingMessageSource.WatchEventType[]{ FileReadingMessageSource.WatchEventType.DELETE }); - } - -} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FileModeRefTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FileModeRefTests.java deleted file mode 100644 index 493705fbe..000000000 --- a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FileModeRefTests.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.file; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.integration.file.FileHeaders; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Gary Russell - * @author Artem Bilan - * @author Soby Chacko - */ -@TestPropertySource(properties = "file.consumer.mode=ref") -public class FileModeRefTests extends AbstractFileSupplierTests { - - @Test - public void testBasicFlow() throws IOException { - - Path firstFile = tempDir.resolve("first.file"); - Files.write(firstFile, "first.file".getBytes()); - - final Flux> messageFlux = fileSupplier.get(); - - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(message.getPayload()) - .isEqualTo(firstFile.toAbsolutePath().toFile()); - assertThat(message.getHeaders()) - .containsEntry(FileHeaders.FILENAME, "first.file"); - assertThat(message.getHeaders()) - .containsEntry(FileHeaders.RELATIVE_PATH, "first.file"); - assertThat(message.getHeaders()) - .containsEntry(FileHeaders.ORIGINAL_FILE, firstFile.toFile()); - } - ) - .thenCancel() - .verify(); - } -} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithPatternTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithPatternTests.java deleted file mode 100644 index 7a7e1e611..000000000 --- a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithPatternTests.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.file; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Gary Russell - * @author Artem Bilan - * @author Soby Chacko - */ -@TestPropertySource(properties = {"file.consumer.mode=ref", "file.supplier.filenamePattern = *.txt"}) -public class FilePayloadWithPatternTests extends AbstractFileSupplierTests { - - @Test - public void testPattern() throws IOException { - - Path txtFile1 = tempDir.resolve("test1.txt"); - Files.write(txtFile1, "one".getBytes()); - Path nonTxtExtension = tempDir.resolve("hello.bin"); - Files.write(nonTxtExtension, ByteBuffer.allocate(4).putInt(1).array()); - Path txtFile2 = tempDir.resolve("test2.txt"); - Files.write(txtFile2, "two".getBytes()); - - final Flux> messageFlux = fileSupplier.get(); - - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(message.getPayload()) - .isEqualTo(txtFile1.toAbsolutePath().toFile()); - } - ) - .assertNext((message) -> { - assertThat(message.getPayload()) - .isEqualTo(txtFile2.toAbsolutePath().toFile()); - }) - .expectNoEvent(Duration.ofSeconds(1)) - .thenCancel() - .verify(); - } -} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithRegexTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithRegexTests.java deleted file mode 100644 index 136ebf76f..000000000 --- a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/FilePayloadWithRegexTests.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.file; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Gary Russell - * @author Artem Bilan - * @author Soby Chacko - */ -@TestPropertySource(properties = {"file.consumer.mode=ref", "file.supplier.filenameRegex = t.*.txt"}) -public class FilePayloadWithRegexTests extends AbstractFileSupplierTests { - - @Test - public void testRegexPattern() throws IOException { - - Path txtFile1 = tempDir.resolve("test1.txt"); - Files.write(txtFile1, "one".getBytes()); - Path nonTxtExtension = tempDir.resolve("hello.bin"); - Files.write(nonTxtExtension, ByteBuffer.allocate(4).putInt(1).array()); - Path txtFile2 = tempDir.resolve("abc.txt"); - Files.write(txtFile2, "two".getBytes()); - - final Flux> messageFlux = fileSupplier.get(); - - StepVerifier stepVerifier = - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(message.getPayload()) - .isEqualTo(txtFile1.toAbsolutePath().toFile()); - } - ) - .expectNoEvent(Duration.ofSeconds(1)) - .expectNoEvent(Duration.ofSeconds(1)) - .thenCancel() - .verifyLater(); - - stepVerifier.verify(); - } -} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesAndMarkersAsJsonPayloadTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesAndMarkersAsJsonPayloadTests.java deleted file mode 100644 index bc34cd4be..000000000 --- a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesAndMarkersAsJsonPayloadTests.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.file; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.integration.file.splitter.FileSplitter; -import org.springframework.integration.json.JsonPathUtils; -import org.springframework.integration.support.json.JsonObjectMapperProvider; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Gary Russell - * @author Artem Bilan - * @author Soby Chacko - */ -@TestPropertySource(properties = {"file.consumer.mode=lines", "file.consumer.withMarkers = true"}) -public class LinesAndMarkersAsJsonPayloadTests extends AbstractFileSupplierTests { - - @Test - public void testLinesWithMarkers() throws Exception { - Path firstFile = tempDir.resolve("test.file"); - Files.write(firstFile, "first line\n".getBytes()); - Files.write(firstFile, "second line\n".getBytes(), StandardOpenOption.APPEND); - - final Flux> messageFlux = fileSupplier.get(); - - StepVerifier.create(messageFlux) - .assertNext((message) -> { - try { - final Object evaluate = JsonPathUtils.evaluate(message.getPayload(), "$.mark"); - assertThat(evaluate).isEqualTo(FileSplitter.FileMarker.Mark.START.name()); - } - catch (IOException e) { - // passt through - } - } - ) - .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("first line")) - .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("second line")) - .assertNext((message) -> { - try { - final Object fileMarker = JsonPathUtils.evaluate(message.getPayload(), "$.mark"); - assertThat(fileMarker).isEqualTo(FileSplitter.FileMarker.Mark.END.name()); - FileSplitter.FileMarker fileMarker1 = JsonObjectMapperProvider.newInstance() - .fromJson(fileMarker, FileSplitter.FileMarker.class); - assertThat(FileSplitter.FileMarker.Mark.END).isEqualTo(fileMarker1.getMark()); - assertThat(firstFile.toAbsolutePath()).isEqualTo(fileMarker1.getFilePath()); - assertThat(fileMarker1.getLineCount()).isEqualTo(2); - } - catch (IOException e) { - // passt through - } - } - ) - .thenCancel() - .verify(); - } -} diff --git a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesPayloadTests.java b/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesPayloadTests.java deleted file mode 100644 index dd222c169..000000000 --- a/functions/supplier/file-supplier/src/test/java/org/springframework/cloud/fn/supplier/file/LinesPayloadTests.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.file; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Gary Russell - * @author Artem Bilan - * @author Soby Chacko - */ -@TestPropertySource(properties = "file.consumer.mode=lines") -public class LinesPayloadTests extends AbstractFileSupplierTests { - - @Test - public void testLines() throws IOException { - Path firstFile = tempDir.resolve("test.file"); - Files.write(firstFile, "first line\n".getBytes()); - Files.write(firstFile, "second line\n".getBytes(), StandardOpenOption.APPEND); - - final Flux> messageFlux = fileSupplier.get(); - - StepVerifier.create(messageFlux) - .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("first line")) - .assertNext((message) -> assertThat(message.getPayload()).isEqualTo("second line")) - .thenCancel() - .verify(); - - } -} diff --git a/functions/supplier/ftp-supplier/README.adoc b/functions/supplier/ftp-supplier/README.adoc deleted file mode 100644 index abca0e511..000000000 --- a/functions/supplier/ftp-supplier/README.adoc +++ /dev/null @@ -1,37 +0,0 @@ -# FTP Supplier - -This module provides a FTP supplier that can be reused and composed in other applications. -The `Supplier` uses the `FtpInboundChannelAdapter` from Spring Integration. -`FtpSupplier` is implemented as a `java.util.function.Supplier`. -This supplier gives you a reactive stream of files from the provided directory as the supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and receive the data. - -## Beans for injection - -You can import the `FtpSupplierConfiguration` in the application and then inject the following bean. - -`ftpSupplier` - -You need to inject this as `Supplier>>`. - -You can use `ftpSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `ftp.supplier`. -There are also properties that need to be used with the prefix `file.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierProperties.java[FtpSupplierProperties]. -See link:src/main/java/org/springframework/cloud/fn/supplier/file/FileConsumerProperties.java[this] also. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `FtpInboundChannelAdapterSpec` configuration used by the `ftpSupplier`. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierTests.java[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/ftp-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a File Source. diff --git a/functions/supplier/ftp-supplier/pom.xml b/functions/supplier/ftp-supplier/pom.xml deleted file mode 100644 index 5c7265441..000000000 --- a/functions/supplier/ftp-supplier/pom.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - ftp-supplier - ftp-supplier - ftp supplier - - - - org.springframework.integration - spring-integration-ftp - - - org.springframework.cloud.fn - file-common - ${project.version} - - - org.springframework.cloud.fn - ftp-common - ${project.version} - - - org.springframework.cloud.fn - metadata-store-common - ${project.version} - - - org.springframework.cloud.fn - function-test-support - ${project.version} - test - - - - diff --git a/functions/supplier/ftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierConfiguration.java b/functions/supplier/ftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierConfiguration.java deleted file mode 100644 index 58870a236..000000000 --- a/functions/supplier/ftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierConfiguration.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright 2015-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.ftp; - -import java.util.function.Supplier; -import java.util.regex.Pattern; - -import org.apache.commons.net.ftp.FTPFile; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.core.scheduler.Schedulers; - -import org.springframework.beans.factory.BeanInitializationException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.cloud.fn.common.file.FileConsumerProperties; -import org.springframework.cloud.fn.common.file.FileReadingMode; -import org.springframework.cloud.fn.common.file.FileUtils; -import org.springframework.cloud.fn.common.ftp.FtpSessionFactoryConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Lazy; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.file.filters.ChainFileListFilter; -import org.springframework.integration.file.remote.session.SessionFactory; -import org.springframework.integration.ftp.dsl.Ftp; -import org.springframework.integration.ftp.dsl.FtpInboundChannelAdapterSpec; -import org.springframework.integration.ftp.filters.FtpPersistentAcceptOnceFileListFilter; -import org.springframework.integration.ftp.filters.FtpRegexPatternFileListFilter; -import org.springframework.integration.ftp.filters.FtpSimplePatternFileListFilter; -import org.springframework.integration.ftp.inbound.FtpInboundFileSynchronizingMessageSource; -import org.springframework.integration.metadata.ConcurrentMetadataStore; -import org.springframework.integration.util.IntegrationReactiveUtils; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.util.StringUtils; - -/** - * @author David Turanski - * @author Marius Bogoevici - * @author Gary Russell - * @author Artem Bilan - * @author Christian Tzolov - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties({ FtpSupplierProperties.class, FileConsumerProperties.class }) -@Import(FtpSessionFactoryConfiguration.class) -public class FtpSupplierConfiguration { - - private final FtpSupplierProperties ftpSupplierProperties; - - private final FileConsumerProperties fileConsumerProperties; - - private final ConcurrentMetadataStore metadataStore; - - SessionFactory ftpSessionFactory; - - @Autowired - @Lazy - @Qualifier("ftpMessageSource") - private FtpInboundFileSynchronizingMessageSource ftpMessageSource; - - public FtpSupplierConfiguration(FtpSupplierProperties ftpSupplierProperties, - FileConsumerProperties fileConsumerProperties, - ConcurrentMetadataStore metadataStore, - SessionFactory ftpSessionFactory) { - - this.ftpSupplierProperties = ftpSupplierProperties; - this.fileConsumerProperties = fileConsumerProperties; - this.metadataStore = metadataStore; - this.ftpSessionFactory = ftpSessionFactory; - } - - @Bean - public FtpInboundChannelAdapterSpec ftpMessageSource( - @Nullable ComponentCustomizer ftpInboundChannelAdapterSpecCustomizer) { - - FtpInboundChannelAdapterSpec messageSourceBuilder = Ftp.inboundAdapter(ftpSessionFactory) - .preserveTimestamp(this.ftpSupplierProperties.isPreserveTimestamp()) - .remoteDirectory(this.ftpSupplierProperties.getRemoteDir()) - .remoteFileSeparator(this.ftpSupplierProperties.getRemoteFileSeparator()) - .localDirectory(this.ftpSupplierProperties.getLocalDir()) - .autoCreateLocalDirectory(this.ftpSupplierProperties.isAutoCreateLocalDir()) - .temporaryFileSuffix(this.ftpSupplierProperties.getTmpFileSuffix()) - .deleteRemoteFiles(this.ftpSupplierProperties.isDeleteRemoteFiles()); - - ChainFileListFilter chainFileListFilter = new ChainFileListFilter<>(); - - String filenamePattern = this.ftpSupplierProperties.getFilenamePattern(); - Pattern filenameRegex = this.ftpSupplierProperties.getFilenameRegex(); - if (StringUtils.hasText(filenamePattern)) { - chainFileListFilter.addFilter(new FtpSimplePatternFileListFilter(filenamePattern)); - } - else if (filenameRegex != null) { - chainFileListFilter.addFilter(new FtpRegexPatternFileListFilter(filenameRegex)); - } - - chainFileListFilter.addFilter(new FtpPersistentAcceptOnceFileListFilter(this.metadataStore, "ftpSource/")); - - messageSourceBuilder.filter(chainFileListFilter); - if (ftpInboundChannelAdapterSpecCustomizer != null) { - ftpInboundChannelAdapterSpecCustomizer.customize(messageSourceBuilder); - } - return messageSourceBuilder; - } - - @Bean - public Flux> ftpMessageFlux() { - return Mono.>create(monoSink -> - monoSink.onRequest(value -> - monoSink.success(this.ftpMessageSource.receive()))) - .subscribeOn(Schedulers.boundedElastic()) - .repeatWhenEmpty(it -> it.delayElements(this.ftpSupplierProperties.getDelayWhenEmpty())) - .repeat(); - } - - @Bean - @ConditionalOnExpression("environment['file.consumer.mode'] != 'ref'") - public Publisher> ftpReadingFlow(FtpInboundFileSynchronizingMessageSource ftpMessageSource) { - return FileUtils.enhanceFlowForReadingMode(IntegrationFlow - .from(IntegrationReactiveUtils.messageSourceToFlux(ftpMessageSource)), fileConsumerProperties) - .toReactivePublisher(); - } - - @Bean - public Supplier>> ftpSupplier(@Nullable Publisher> ftpReadingFlow) { - if (this.fileConsumerProperties.getMode() == FileReadingMode.ref) { - return this::ftpMessageFlux; - } - else if (ftpReadingFlow != null) { - return () -> Flux.from(ftpReadingFlow); - } - else { - throw new BeanInitializationException( - "Cannot creat 'ftpSupplier' bean: no 'ftpReadingFlow' dependency and is not 'FileReadingMode.ref'."); - } - } - -} diff --git a/functions/supplier/ftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierProperties.java b/functions/supplier/ftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierProperties.java deleted file mode 100644 index a156c4f14..000000000 --- a/functions/supplier/ftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierProperties.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.ftp; - -import java.io.File; -import java.time.Duration; -import java.util.regex.Pattern; - -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * @author David Turanski - * @author Gary Russell - * @author Artem Bilan - */ -@ConfigurationProperties("ftp.supplier") -@Validated -public class FtpSupplierProperties { - - /** - * The remote FTP directory. - */ - private String remoteDir = "/"; - - /** - * The suffix to use while the transfer is in progress. - */ - private String tmpFileSuffix = ".tmp"; - - /** - * The remote file separator. - */ - private String remoteFileSeparator = "/"; - - /** - * Set to true to delete remote files after successful transfer. - */ - private boolean deleteRemoteFiles = false; - - /** - * The local directory to use for file transfers. - */ - private File localDir = new File(System.getProperty("java.io.tmpdir"), "ftp-source"); - - /** - * Set to true to create the local directory if it does not exist. - */ - private boolean autoCreateLocalDir = true; - - /** - * A filter pattern to match the names of files to transfer. - */ - private String filenamePattern; - - /** - * A filter regex pattern to match the names of files to transfer. - */ - private Pattern filenameRegex; - - /** - * Set to true to preserve the original timestamp. - */ - private boolean preserveTimestamp = true; - - /** - * Duration of delay when no new files are detected. - */ - private Duration delayWhenEmpty = Duration.ofSeconds(1); - - public boolean isAutoCreateLocalDir() { - return this.autoCreateLocalDir; - } - - public void setAutoCreateLocalDir(boolean autoCreateLocalDir) { - this.autoCreateLocalDir = autoCreateLocalDir; - } - - public boolean isDeleteRemoteFiles() { - return this.deleteRemoteFiles; - } - - public void setDeleteRemoteFiles(boolean deleteRemoteFiles) { - this.deleteRemoteFiles = deleteRemoteFiles; - } - - @NotNull - public File getLocalDir() { - return this.localDir; - } - - public final void setLocalDir(File localDir) { - this.localDir = localDir; - } - - public String getFilenamePattern() { - return this.filenamePattern; - } - - public void setFilenamePattern(String filenamePattern) { - this.filenamePattern = filenamePattern; - } - - public Pattern getFilenameRegex() { - return this.filenameRegex; - } - - public void setFilenameRegex(Pattern filenameRegex) { - this.filenameRegex = filenameRegex; - } - - public boolean isPreserveTimestamp() { - return this.preserveTimestamp; - } - - public void setPreserveTimestamp(boolean preserveTimestamp) { - this.preserveTimestamp = preserveTimestamp; - } - - @NotBlank - public String getRemoteDir() { - return this.remoteDir; - } - - public final void setRemoteDir(String remoteDir) { - this.remoteDir = remoteDir; - } - - @NotBlank - public String getTmpFileSuffix() { - return this.tmpFileSuffix; - } - - public void setTmpFileSuffix(String tmpFileSuffix) { - this.tmpFileSuffix = tmpFileSuffix; - } - - @NotBlank - public String getRemoteFileSeparator() { - return this.remoteFileSeparator; - } - - public void setRemoteFileSeparator(String remoteFileSeparator) { - this.remoteFileSeparator = remoteFileSeparator; - } - - @AssertTrue(message = "filenamePattern and filenameRegex are mutually exclusive") - public boolean isExclusivePatterns() { - return !(this.filenamePattern != null && this.filenameRegex != null); - } - - public Duration getDelayWhenEmpty() { - return delayWhenEmpty; - } - - public void setDelayWhenEmpty(Duration delayWhenEmpty) { - this.delayWhenEmpty = delayWhenEmpty; - } - -} diff --git a/functions/supplier/ftp-supplier/src/test/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierPropertiesTests.java b/functions/supplier/ftp-supplier/src/test/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierPropertiesTests.java deleted file mode 100644 index efe08e727..000000000 --- a/functions/supplier/ftp-supplier/src/test/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierPropertiesTests.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.ftp; - -import java.io.File; - -import org.junit.jupiter.api.Test; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.boot.test.util.TestPropertyValues; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.springframework.context.annotation.Configuration; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author David Turanski - * @author Gary Russell - * @author Artem Bilan - */ -public class FtpSupplierPropertiesTests { - - @Test - public void localDirCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.supplier.localDir:local") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpSupplierProperties properties = context.getBean(FtpSupplierProperties.class); - assertThat(properties.getLocalDir()).isEqualTo(new File("local")); - context.close(); - } - - @Test - public void remoteDirCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.supplier.remoteDir:/remote") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpSupplierProperties properties = context.getBean(FtpSupplierProperties.class); - assertThat(properties.getRemoteDir()).isEqualTo("/remote"); - context.close(); - } - - @Test - public void deleteRemoteFilesCanBeEnabled() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.supplier.deleteRemoteFiles:true") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpSupplierProperties properties = context.getBean(FtpSupplierProperties.class); - assertThat(properties.isDeleteRemoteFiles()).isTrue(); - context.close(); - } - - @Test - public void autoCreateLocalDirCanBeDisabled() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.supplier.autoCreateLocalDir:false") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpSupplierProperties properties = context.getBean(FtpSupplierProperties.class); - assertThat(!properties.isAutoCreateLocalDir()).isTrue(); - context.close(); - } - - @Test - public void tmpFileSuffixCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.supplier.tmpFileSuffix:.foo") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpSupplierProperties properties = context.getBean(FtpSupplierProperties.class); - assertThat(properties.getTmpFileSuffix()).isEqualTo(".foo"); - context.close(); - } - - @Test - public void filenamePatternCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.supplier.filenamePattern:*.foo") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpSupplierProperties properties = context.getBean(FtpSupplierProperties.class); - assertThat(properties.getFilenamePattern()).isEqualTo("*.foo"); - context.close(); - } - - @Test - public void remoteFileSeparatorCanBeCustomized() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.supplier.remoteFileSeparator:\\") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpSupplierProperties properties = context.getBean(FtpSupplierProperties.class); - assertThat(properties.getRemoteFileSeparator()).isEqualTo("\\"); - context.close(); - } - - - @Test - public void preserveTimestampDirCanBeDisabled() { - AnnotationConfigApplicationContext context = new AnnotationConfigApplicationContext(); - TestPropertyValues.of("ftp.supplier.preserveTimestamp:false") - .applyTo(context); - context.register(Conf.class); - context.refresh(); - FtpSupplierProperties properties = context.getBean(FtpSupplierProperties.class); - assertThat(!properties.isPreserveTimestamp()).isTrue(); - context.close(); - } - - @Configuration - @EnableConfigurationProperties(FtpSupplierProperties.class) - static class Conf { - - } -} diff --git a/functions/supplier/ftp-supplier/src/test/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierTests.java b/functions/supplier/ftp-supplier/src/test/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierTests.java deleted file mode 100644 index e74e8bf80..000000000 --- a/functions/supplier/ftp-supplier/src/test/java/org/springframework/cloud/fn/supplier/ftp/FtpSupplierTests.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.ftp; - -import java.io.File; -import java.util.function.Supplier; - -import org.apache.commons.net.ftp.FTPFile; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.test.support.ftp.FtpTestSupport; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Primary; -import org.springframework.integration.file.remote.session.CachingSessionFactory; -import org.springframework.integration.file.remote.session.SessionFactory; -import org.springframework.integration.metadata.ConcurrentMetadataStore; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { - "ftp.factory.username = foo", - "ftp.factory.password = foo", - "file.consumer.mode = ref", - "ftp.factory.cacheSessions = true" - }) -@DirtiesContext -public class FtpSupplierTests extends FtpTestSupport { - - @Autowired - Supplier>> ftpSupplier; - - @Autowired - private ConcurrentMetadataStore metadataStore; - - @Autowired - FtpSupplierProperties config; - - @Autowired - SessionFactory sessionFactory; - - @Test - public void testSourceFileAsRef() { - final Flux> messageFlux = ftpSupplier.get(); - assertThat(this.sessionFactory).isInstanceOf(CachingSessionFactory.class); - StepVerifier stepVerifier = - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(new File(message.getPayload().toString().replaceAll("\"", ""))).isEqualTo( - new File(this.config.getLocalDir(), "ftpSource1.txt")); - } - ) - .assertNext((message) -> { - assertThat(new File(message.getPayload().toString().replaceAll("\"", ""))).isEqualTo( - new File(this.config.getLocalDir(), "ftpSource2.txt")); - }) - .thenCancel() - .verifyLater(); - stepVerifier.verify(); - } - - @SpringBootApplication - static class FtpSupplierTestApplication { - - // These properties can be moved into the SpringBootApplication annotation, but providing here - // as a way to demonstrate how we can provide ConfigurationProperties as a bean in the application itself - // This way, it is less error-prone from typos. - @Bean - @Primary - public FtpSupplierProperties ftpSupplierProperties() { - final FtpSupplierProperties ftpSupplierProperties = new FtpSupplierProperties(); - ftpSupplierProperties.setRemoteDir("ftpSource"); - ftpSupplierProperties.setFilenamePattern("*"); - return ftpSupplierProperties; - } - } -} diff --git a/functions/supplier/http-supplier/README.adoc b/functions/supplier/http-supplier/README.adoc deleted file mode 100644 index 9725e100f..000000000 --- a/functions/supplier/http-supplier/README.adoc +++ /dev/null @@ -1,35 +0,0 @@ -# HTTP Supplier - -This module provides an HTTP supplier that can be reused and composed in other applications. -The `Supplier` uses the `WebFluxInboundEndpoint` from Spring Integration. -`httpSupplier` is implemented as a `java.util.function.Supplier`. -This supplier gives you a reactive stream from the http endpoint. The supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and then receive the data. - -## Beans for injection - -You can import the `HttpSupplierConfiguration` in the application and then inject the following bean. - -`httpSupplier` - -You need to inject this as `Supplier>>`. - -You can use `httpSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `http`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierProperties.java[HttpSupplierProperties]. - -The `HeaderMapper` bean can be provided in the target configuration to override a default one in the `HttpSupplierConfiguration`. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/http/HttpSupplierApplicationTests.java[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/http-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes an HTTP Source. \ No newline at end of file diff --git a/functions/supplier/http-supplier/pom.xml b/functions/supplier/http-supplier/pom.xml deleted file mode 100644 index 6c5ac9e29..000000000 --- a/functions/supplier/http-supplier/pom.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - http-supplier - http-supplier - HTTP Supplier - - - - org.springframework.boot - spring-boot-starter-webflux - - - org.springframework.integration - spring-integration-webflux - - - - diff --git a/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierConfiguration.java b/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierConfiguration.java deleted file mode 100644 index 2cbef92ab..000000000 --- a/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierConfiguration.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright 2011-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.http; - -import java.util.function.Supplier; - -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.codec.ServerCodecConfigurer; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.expression.ValueExpression; -import org.springframework.integration.http.support.DefaultHttpHeaderMapper; -import org.springframework.integration.mapping.HeaderMapper; -import org.springframework.integration.webflux.dsl.WebFlux; -import org.springframework.integration.webflux.inbound.WebFluxInboundEndpoint; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; - -/** - * Configuration for the HTTP Supplier. - * - * @author Artem Bilan - */ -@EnableConfigurationProperties(HttpSupplierProperties.class) -@Configuration -public class HttpSupplierConfiguration { - - @Bean - @ConditionalOnMissingBean - public HeaderMapper httpHeaderMapper(HttpSupplierProperties httpSupplierProperties) { - DefaultHttpHeaderMapper defaultHttpHeaderMapper = DefaultHttpHeaderMapper.inboundMapper(); - defaultHttpHeaderMapper.setInboundHeaderNames(httpSupplierProperties.getMappedRequestHeaders()); - return defaultHttpHeaderMapper; - } - - @Bean - public Publisher> httpSupplierFlow(HttpSupplierProperties httpSupplierProperties, - HeaderMapper httpHeaderMapper, - ServerCodecConfigurer serverCodecConfigurer) { - - return IntegrationFlow.from( - WebFlux.inboundChannelAdapter(httpSupplierProperties.getPathPattern()) - .requestPayloadType(byte[].class) - .statusCodeExpression(new ValueExpression<>(HttpStatus.ACCEPTED)) - .headerMapper(httpHeaderMapper) - .codecConfigurer(serverCodecConfigurer) - .crossOrigin(crossOrigin -> - crossOrigin.origin(httpSupplierProperties.getCors().getAllowedOrigins()) - .allowedHeaders(httpSupplierProperties.getCors().getAllowedHeaders()) - .allowCredentials(httpSupplierProperties.getCors().getAllowCredentials())) - .autoStartup(false)) - .enrichHeaders((headers) -> - headers.headerFunction(MessageHeaders.CONTENT_TYPE, - (message) -> - (MediaType.APPLICATION_FORM_URLENCODED.equals( - message.getHeaders().get(MessageHeaders.CONTENT_TYPE, MediaType.class))) - ? MediaType.APPLICATION_JSON - : null, - true)) - .toReactivePublisher(); - } - - @Bean - public Supplier>> httpSupplier( - Publisher> httpRequestPublisher, - WebFluxInboundEndpoint webFluxInboundEndpoint) { - - return () -> Flux.from(httpRequestPublisher) - .doOnSubscribe((subscription) -> webFluxInboundEndpoint.start()) - .doOnTerminate(webFluxInboundEndpoint::stop); - } - -} diff --git a/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierProperties.java b/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierProperties.java deleted file mode 100644 index 76f5a3454..000000000 --- a/functions/supplier/http-supplier/src/main/java/org/springframework/cloud/fn/supplier/http/HttpSupplierProperties.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.http; - -import jakarta.validation.constraints.NotEmpty; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.integration.http.support.DefaultHttpHeaderMapper; -import org.springframework.validation.annotation.Validated; -import org.springframework.web.cors.CorsConfiguration; - -/** - * Configuration properties for the HTTP Supplier. - * - * @author Artem Bilan - */ -@ConfigurationProperties("http") -@Validated -public class HttpSupplierProperties { - - /** - * HTTP endpoint path mapping. - */ - private String pathPattern = "/"; - - /** - * Headers that will be mapped. - */ - private String[] mappedRequestHeaders = {DefaultHttpHeaderMapper.HTTP_REQUEST_HEADER_NAME_PATTERN}; - - /** - * CORS properties. - */ - private Cors cors = new Cors(); - - @NotEmpty - public String getPathPattern() { - return this.pathPattern; - } - - public void setPathPattern(String pathPattern) { - this.pathPattern = pathPattern; - } - - public String[] getMappedRequestHeaders() { - return this.mappedRequestHeaders; - } - - public void setMappedRequestHeaders(String[] mappedRequestHeaders) { - this.mappedRequestHeaders = mappedRequestHeaders; - } - - public Cors getCors() { - return this.cors; - } - - public void setCors(Cors cors) { - this.cors = cors; - } - - public static class Cors { - - /** - * List of allowed origins, e.g. https://domain1.com. - */ - private String[] allowedOrigins = {CorsConfiguration.ALL}; - - /** - * List of request headers that can be used during the actual request. - */ - private String[] allowedHeaders = {CorsConfiguration.ALL}; - - /** - * Whether the browser should include any cookies associated with the domain of the request being annotated. - */ - private Boolean allowCredentials; - - @NotEmpty - public String[] getAllowedOrigins() { - return this.allowedOrigins; - } - - public void setAllowedOrigins(String[] allowedOrigins) { - this.allowedOrigins = allowedOrigins; - } - - @NotEmpty - public String[] getAllowedHeaders() { - return this.allowedHeaders; - } - - public void setAllowedHeaders(String[] allowedHeaders) { - this.allowedHeaders = allowedHeaders; - } - - public Boolean getAllowCredentials() { - return allowCredentials; - } - - public void setAllowCredentials(Boolean allowCredentials) { - this.allowCredentials = allowCredentials; - } - - } - -} diff --git a/functions/supplier/http-supplier/src/main/resources/application.properties b/functions/supplier/http-supplier/src/main/resources/application.properties deleted file mode 100644 index 8b1378917..000000000 --- a/functions/supplier/http-supplier/src/main/resources/application.properties +++ /dev/null @@ -1 +0,0 @@ - diff --git a/functions/supplier/http-supplier/src/test/java/org/springframework/cloud/fn/supplier/http/HttpSupplierApplicationTests.java b/functions/supplier/http-supplier/src/test/java/org/springframework/cloud/fn/supplier/http/HttpSupplierApplicationTests.java deleted file mode 100644 index 968e9cde7..000000000 --- a/functions/supplier/http-supplier/src/test/java/org/springframework/cloud/fn/supplier/http/HttpSupplierApplicationTests.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright 2020-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.http; - -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.function.Supplier; - -import javax.net.ssl.SSLException; - -import io.netty.handler.ssl.SslContext; -import io.netty.handler.ssl.SslContextBuilder; -import io.netty.handler.ssl.SslProvider; -import io.netty.handler.ssl.util.InsecureTrustManagerFactory; -import org.assertj.core.api.InstanceOfAssertFactories; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.netty.http.client.HttpClient; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.web.server.LocalServerPort; -import org.springframework.http.MediaType; -import org.springframework.http.client.reactive.ReactorClientHttpConnector; -import org.springframework.http.codec.ServerCodecConfigurer; -import org.springframework.http.codec.json.AbstractJackson2Decoder; -import org.springframework.integration.http.HttpHeaders; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.integration.webflux.inbound.WebFluxInboundEndpoint; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.web.reactive.function.client.WebClient; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * The test for HTTP Supplier. - * - * @author Artem Bilan - */ -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = { - "server.ssl.key-store=classpath:test.jks", - "server.ssl.key-password=password", - "server.ssl.trust-store=classpath:test.jks", - "server.ssl.client-auth=want", - "spring.codec.max-in-memory-size=10MB" - }) -@DirtiesContext -public class HttpSupplierApplicationTests { - - @Autowired - private Supplier>> httpSupplier; - - @Autowired - private WebFluxInboundEndpoint webFluxInboundEndpoint; - - @LocalServerPort - private int port; - - @Test - public void testHttpSupplier() throws SSLException { - ServerCodecConfigurer codecConfigurer = - TestUtils.getPropertyValue(this.webFluxInboundEndpoint, "codecConfigurer", ServerCodecConfigurer.class); - - final ServerCodecConfigurer.ServerDefaultCodecs serverDefaultCodecs = codecConfigurer.defaultCodecs(); - assertThat(TestUtils.getPropertyValue(serverDefaultCodecs, "maxInMemorySize", Integer.class)) - .isEqualTo(1024 * 1024 * 10); - - AbstractJackson2Decoder jackson2JsonDecoder = - TestUtils.getPropertyValue(serverDefaultCodecs, "jackson2JsonDecoder", AbstractJackson2Decoder.class); - - assertThat(jackson2JsonDecoder).isNotNull() - .extracting("maxInMemorySize") - .isEqualTo(1024 * 1024 * 10); - - Flux> messageFlux = this.httpSupplier.get(); - - StepVerifier stepVerifier = - StepVerifier.create(messageFlux) - .assertNext((message) -> - assertThat(message) - .satisfies((msg) -> assertThat(msg) - .extracting(Message::getPayload) - .isEqualTo("test1".getBytes())) - .satisfies((msg) -> assertThat(msg.getHeaders()) - .containsEntry(MessageHeaders.CONTENT_TYPE, - new MediaType("text", "plain", StandardCharsets.UTF_8)) - .extractingByKey(HttpHeaders.REQUEST_URL).asString() - .startsWith("https://")) - ) - .assertNext((message) -> - assertThat(message) - .extracting(Message::getPayload) - .isEqualTo("{\"name\":\"test2\"}".getBytes())) - .assertNext((message) -> - assertThat(message) - .extracting(Message::getPayload) - .isEqualTo("{\"name\":\"test3\"}".getBytes())) - .assertNext((message) -> - assertThat(message) - .satisfies((msg) -> assertThat(msg) - .extracting(Message::getPayload) - .asInstanceOf(InstanceOfAssertFactories.MAP) - .isEmpty()) - .satisfies((msg) -> assertThat(msg.getHeaders()) - .doesNotContainKey(MessageHeaders.CONTENT_TYPE))) - .thenCancel() - .verifyLater(); - - SslContext sslContext = - SslContextBuilder.forClient() - .sslProvider(SslProvider.JDK) - .trustManager(InsecureTrustManagerFactory.INSTANCE) - .build(); - - HttpClient httpClient = - HttpClient.create() - .secure(sslSpec -> sslSpec.sslContext(sslContext)); - - WebClient webClient = - WebClient.builder() - .clientConnector(new ReactorClientHttpConnector(httpClient)) - .baseUrl("https://localhost:" + port) - .build(); - - webClient.post() - .uri("/") - .bodyValue("test1") - .retrieve() - .toBodilessEntity() - .block(Duration.ofSeconds(10)); - - webClient.post() - .uri("/") - .bodyValue(new TestPojo("test2")) - .retrieve() - .toBodilessEntity() - .block(Duration.ofSeconds(10)); - - webClient.post() - .uri("/") - .bodyValue(new TestPojo("test3")) - .retrieve() - .toBodilessEntity() - .block(Duration.ofSeconds(10)); - - webClient.post().uri("/") - .retrieve() - .toBodilessEntity() - .block(Duration.ofSeconds(10)); - - stepVerifier.verify(); - } - - private static class TestPojo { - - private String name; - - TestPojo() { - } - - TestPojo(String name) { - this.name = name; - } - - public String getName() { - return this.name; - } - - public void setName(String name) { - this.name = name; - } - - } - - @SpringBootApplication - static class HttpSupplierTestApplication { - - } - -} diff --git a/functions/supplier/http-supplier/src/test/resources/test.jks b/functions/supplier/http-supplier/src/test/resources/test.jks deleted file mode 100644 index 0fc3e802f..000000000 Binary files a/functions/supplier/http-supplier/src/test/resources/test.jks and /dev/null differ diff --git a/functions/supplier/jdbc-supplier/README.adoc b/functions/supplier/jdbc-supplier/README.adoc deleted file mode 100644 index beba5d737..000000000 --- a/functions/supplier/jdbc-supplier/README.adoc +++ /dev/null @@ -1,36 +0,0 @@ -# JDBC Supplier - -This module provides a JDBC supplier that can be reused and composed in other applications. -The `Supplier` uses the `JdbcPollingChannelAdapter` from Spring Integration. -`JdbcSupplier` is implemented as a `java.util.function.Supplier`. -When you have use-cases such as periodical execution of a Database query, based on some external trigger such as a REST endpoint call for example, then you can use this `Supplier` to query the underlying relational database. - -## Beans for injection - -You can import the `JdbcSupplierConfiguration` in the application and then inject the following bean. - -`jdbcSupplier` - -You can inject it as `Supplier` when you are not splitting the rows. -If you are splitting the output, you need to inject this as `Supplier>>`. - -You can use `jdbcSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it. -In the case of splitting you get a `Flux` which you have to subscribe in your applications. - -## Configuration Options - -All configuration properties are prefixed with `jdbc.supplier`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierProperties.java[JdbcSupplierProperties] - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `JdbcPollingChannelAdapter` configuration used by the `jdbcSupplier`. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/jdbc[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/jdbc-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a JDBC Source. diff --git a/functions/supplier/jdbc-supplier/pom.xml b/functions/supplier/jdbc-supplier/pom.xml deleted file mode 100644 index 749f41442..000000000 --- a/functions/supplier/jdbc-supplier/pom.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - jdbc-supplier - jdbc-supplier - JDBC supplier - - - - org.springframework.integration - spring-integration-jdbc - - - org.springframework.boot - spring-boot-starter-jdbc - - - org.springframework.cloud - spring-cloud-function-context - ${spring-cloud-function.version} - - - org.springframework.cloud.fn - splitter-function - ${project.version} - - - org.hsqldb - hsqldb - runtime - - - com.h2database - h2 - runtime - - - org.mariadb.jdbc - mariadb-java-client - runtime - - - org.postgresql - postgresql - runtime - - - com.microsoft.sqlserver - mssql-jdbc - runtime - - - com.microsoft.azure - azure-keyvault - - - - - - diff --git a/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierConfiguration.java b/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierConfiguration.java deleted file mode 100644 index a10013b1b..000000000 --- a/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierConfiguration.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2019-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.jdbc; - -import java.util.List; -import java.util.function.Function; -import java.util.function.Supplier; - -import javax.sql.DataSource; - -import reactor.core.publisher.Flux; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.cloud.fn.splitter.SplitterFunctionConfiguration; -import org.springframework.cloud.function.context.PollableBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.integration.core.MessageSource; -import org.springframework.integration.jdbc.JdbcPollingChannelAdapter; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; - -/** - * @author Soby Chacko - * @author Artem Bilan - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties(JdbcSupplierProperties.class) -@Import(SplitterFunctionConfiguration.class) -public class JdbcSupplierConfiguration { - - private final JdbcSupplierProperties properties; - - private final DataSource dataSource; - - public JdbcSupplierConfiguration(JdbcSupplierProperties properties, DataSource dataSource) { - this.properties = properties; - this.dataSource = dataSource; - } - - @Bean - public MessageSource jdbcMessageSource( - @Nullable ComponentCustomizer jdbcPollingChannelAdapterCustomizer) { - - JdbcPollingChannelAdapter jdbcPollingChannelAdapter = - new JdbcPollingChannelAdapter(this.dataSource, this.properties.getQuery()); - jdbcPollingChannelAdapter.setMaxRows(this.properties.getMaxRows()); - jdbcPollingChannelAdapter.setUpdateSql(this.properties.getUpdate()); - if (jdbcPollingChannelAdapterCustomizer != null) { - jdbcPollingChannelAdapterCustomizer.customize(jdbcPollingChannelAdapter); - } - return jdbcPollingChannelAdapter; - } - - @Bean(name = "jdbcSupplier") - @PollableBean - @ConditionalOnProperty(prefix = "jdbc.supplier", name = "split", matchIfMissing = true) - public Supplier>> splittedSupplier(MessageSource jdbcMessageSource, - Function, List>> splitterFunction) { - return () -> { - Message received = jdbcMessageSource.receive(); - if (received != null) { - return Flux.fromIterable(splitterFunction.apply(received)); // multiple Message> - } - else { - return Flux.empty(); - } - }; - } - - @Bean - @ConditionalOnProperty(prefix = "jdbc.supplier", name = "split", havingValue = "false") - public Supplier> jdbcSupplier(MessageSource jdbcMessageSource) { - return jdbcMessageSource::receive; - } - -} diff --git a/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierProperties.java b/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierProperties.java deleted file mode 100644 index 45d0ed8fb..000000000 --- a/functions/supplier/jdbc-supplier/src/main/java/org/springframework/cloud/fn/supplier/jdbc/JdbcSupplierProperties.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2019-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.jdbc; - -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * @author Soby Chacko - * @author Artem Bilan - */ -@ConfigurationProperties("jdbc.supplier") -@Validated -public class JdbcSupplierProperties { - - /** - * The query to use to select data. - */ - private String query; - - /** - * An SQL update statement to execute for marking polled messages as 'seen'. - */ - private String update; - - /** - * Whether to split the SQL result as individual messages. - */ - private boolean split = true; - - /** - * Max numbers of rows to process for query. - */ - private int maxRows = 0; - - @NotNull - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public String getUpdate() { - return update; - } - - public void setUpdate(String update) { - this.update = update; - } - - public boolean isSplit() { - return split; - } - - public void setSplit(boolean split) { - this.split = split; - } - - public int getMaxRows() { - return maxRows; - } - - public void setMaxRows(int maxRows) { - this.maxRows = maxRows; - } - -} diff --git a/functions/supplier/jdbc-supplier/src/main/resources/application.properties b/functions/supplier/jdbc-supplier/src/main/resources/application.properties deleted file mode 100644 index 532da3b2d..000000000 --- a/functions/supplier/jdbc-supplier/src/main/resources/application.properties +++ /dev/null @@ -1 +0,0 @@ -spring.integration.jdbc.initialize-schema=NEVER diff --git a/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/DefaultJdbcSupplierTests.java b/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/DefaultJdbcSupplierTests.java deleted file mode 100644 index 44073f558..000000000 --- a/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/DefaultJdbcSupplierTests.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.jdbc; - -import java.util.Map; -import java.util.function.Supplier; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.jdbc.BadSqlGrammarException; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatExceptionOfType; - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = "jdbc.supplier.query=select id, name from test order by id") -@DirtiesContext -public class DefaultJdbcSupplierTests { - - @Autowired - Supplier>> jdbcSupplier; - - @Autowired - JdbcTemplate jdbcTemplate; - - @Test - void testExtraction() { - final Flux> messageFlux = jdbcSupplier.get(); - StepVerifier stepVerifier = - StepVerifier.create(messageFlux) - .assertNext((message) -> - assertThat(message) - .satisfies((msg) -> assertThat(msg) - .extracting(Message::getPayload) - .matches(o -> { - Map map = (Map) o; - return map.get("ID").equals(1L) && map.get("NAME").equals("Bob"); - }) - )) - .assertNext((message) -> - assertThat(message) - .satisfies((msg) -> assertThat(msg) - .extracting(Message::getPayload) - .matches(o -> { - Map map = (Map) o; - return map.get("ID").equals(2L) && map.get("NAME").equals("Jane"); - }) - )) - .assertNext((message) -> - assertThat(message) - .satisfies((msg) -> assertThat(msg) - .extracting(Message::getPayload) - .matches(o -> { - Map map = (Map) o; - return map.get("ID").equals(3L) && map.get("NAME").equals("John"); - }) - )) - .thenCancel() - .verifyLater(); - stepVerifier.verify(); - } - - /* - The test to verify that DB is not initialized with Spring Integration DDL - (spring.integration.jdbc.initialize-schema=NEVER) what happens by default via IntegrationAutoConfiguration.IntegrationJdbcConfiguration. - This is not a functionality of this JDBC Supplier. - */ - @Test - void verifyNoIntMessageGroupTable() { - assertThatExceptionOfType(BadSqlGrammarException.class) - .isThrownBy(() -> this.jdbcTemplate.queryForList("SELECT * FROM INT_MESSAGE_GROUP")) - .havingCause() - .withMessageContaining("Table \"INT_MESSAGE_GROUP\" not found;"); - } - - @SpringBootApplication - static class JdbcSupplierTestApplication { - } -} diff --git a/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/NonSplitJdbcSupplierTests.java b/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/NonSplitJdbcSupplierTests.java deleted file mode 100644 index eb6f550f0..000000000 --- a/functions/supplier/jdbc-supplier/src/test/java/org/springframework/cloud/fn/supplier/jdbc/NonSplitJdbcSupplierTests.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.jdbc; - -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; - -import org.junit.jupiter.api.Test; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Soby Chacko - * @author Artem Bilan - */ -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = {"jdbc.supplier.query=select id, name from test order by id", "jdbc.supplier.split=false"}) -@DirtiesContext -public class NonSplitJdbcSupplierTests { - - @Autowired - Supplier> jdbcSupplier; - - @Test - void testExtraction() { - final Message message = jdbcSupplier.get(); - final List> payload = (List>) message.getPayload(); - assertThat(payload.size()).isEqualTo(3); - Map map = payload.get(0); - assertThat(map.get("ID")).isEqualTo(1L); - assertThat(map.get("NAME")).isEqualTo("Bob"); - map = payload.get(1); - assertThat(map.get("ID")).isEqualTo(2L); - assertThat(map.get("NAME")).isEqualTo("Jane"); - map = payload.get(2); - assertThat(map.get("ID")).isEqualTo(3L); - assertThat(map.get("NAME")).isEqualTo("John"); - } - - @SpringBootApplication - static class JdbcSupplierTestApplication { - } -} diff --git a/functions/supplier/jdbc-supplier/src/test/resources/schema.sql b/functions/supplier/jdbc-supplier/src/test/resources/schema.sql deleted file mode 100644 index bb5835cb8..000000000 --- a/functions/supplier/jdbc-supplier/src/test/resources/schema.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Run by default by Boot infrastructure - -create table test( - id bigint, - name varchar (2000), - tag char(1) -); -insert into test values (1, 'Bob', NULL); -insert into test values (2, 'Jane', NULL); -insert into test values (3, 'John', NULL); \ No newline at end of file diff --git a/functions/supplier/jms-supplier/README.adoc b/functions/supplier/jms-supplier/README.adoc deleted file mode 100644 index 4aa187546..000000000 --- a/functions/supplier/jms-supplier/README.adoc +++ /dev/null @@ -1,35 +0,0 @@ -# JMS Supplier - -This module provides a JMS supplier that can be reused and composed in other applications. -The `Supplier` uses the JMS support provided by Spring Framework and Spring Integration under the covers. -`jmsSupplier` is implemented as a `java.util.function.Supplier`. -This supplier gives you a reactive stream from JMS sources. The supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and then receive the data. - -## Beans for injection - -You can import the `JmsSupplierConfiguration` in the application and then inject the following bean. - -`jmsSupplier` - -You need to inject this as `Supplier>>`. - -You can use `jmsSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `jms`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/jms/JmsSupplierProperties.java[JmsSupplierProperties]. - -A `ComponentCustomizer>` bean can be added in the target project to provide any custom options for the `JmsMessageDrivenChannelAdapterSpec` configuration used by the `jmsSupplier`. - -## Examples - -See this link:src/test/java/org/springframework/cloud/fn/supplier/jms/[test suite] for the various ways, this supplier is used. - -## Other usage - -See this link:../../../applications/source/jms-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream JMS Source. diff --git a/functions/supplier/jms-supplier/pom.xml b/functions/supplier/jms-supplier/pom.xml deleted file mode 100644 index b115f536d..000000000 --- a/functions/supplier/jms-supplier/pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - jms-supplier - jms-supplier - jms supplier - - - - org.springframework.integration - spring-integration-jms - - - jakarta.jms - jakarta.jms-api - provided - - - org.springframework.boot - spring-boot-starter-artemis - test - - - org.apache.activemq - artemis-jms-server - 2.26.0 - test - - - diff --git a/functions/supplier/jms-supplier/src/main/java/org/springframework/cloud/fn/supplier/jms/JmsSupplierConfiguration.java b/functions/supplier/jms-supplier/src/main/java/org/springframework/cloud/fn/supplier/jms/JmsSupplierConfiguration.java deleted file mode 100644 index d4e040a0d..000000000 --- a/functions/supplier/jms-supplier/src/main/java/org/springframework/cloud/fn/supplier/jms/JmsSupplierConfiguration.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.jms; - -import java.util.function.Supplier; - -import jakarta.jms.ConnectionFactory; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.jms.JmsProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.jms.dsl.Jms; -import org.springframework.integration.jms.dsl.JmsMessageDrivenChannelAdapterSpec; -import org.springframework.jms.listener.AbstractMessageListenerContainer; -import org.springframework.jms.listener.DefaultMessageListenerContainer; -import org.springframework.jms.listener.SimpleMessageListenerContainer; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; - -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties(JmsSupplierProperties.class) -public class JmsSupplierConfiguration { - - @Autowired - JmsSupplierProperties properties; - - @Autowired - private JmsProperties jmsProperties; - - @Autowired - private ConnectionFactory connectionFactory; - - @Bean - public Supplier>> jmsSupplier(Publisher> jmsPublisher) { - return () -> Flux.from(jmsPublisher); - } - - @Bean - public Publisher> jmsPublisher( - AbstractMessageListenerContainer container, - @Nullable ComponentCustomizer> - jmsMessageDrivenChannelAdapterSpecCustomizer) { - - JmsMessageDrivenChannelAdapterSpec messageProducerSpec = Jms.messageDrivenChannelAdapter(container); - - if (jmsMessageDrivenChannelAdapterSpecCustomizer != null) { - jmsMessageDrivenChannelAdapterSpecCustomizer.customize(messageProducerSpec); - } - - return IntegrationFlow.from(messageProducerSpec) - .toReactivePublisher(true); - } - - @Bean - public AbstractMessageListenerContainer container() { - AbstractMessageListenerContainer container; - JmsProperties.Listener listenerProperties = this.jmsProperties.getListener(); - if (this.properties.isSessionTransacted()) { - DefaultMessageListenerContainer dmlc = new DefaultMessageListenerContainer(); - dmlc.setSessionTransacted(true); - if (listenerProperties.getConcurrency() != null) { - dmlc.setConcurrentConsumers(listenerProperties.getConcurrency()); - } - if (listenerProperties.getMaxConcurrency() != null) { - dmlc.setMaxConcurrentConsumers(listenerProperties.getMaxConcurrency()); - } - container = dmlc; - } - else { - SimpleMessageListenerContainer smlc = new SimpleMessageListenerContainer(); - smlc.setSessionTransacted(false); - if (listenerProperties != null && listenerProperties.getConcurrency() != null) { - smlc.setConcurrentConsumers(listenerProperties.getConcurrency()); - } - container = smlc; - } - container.setConnectionFactory(this.connectionFactory); - if (this.properties.getClientId() != null) { - container.setClientId(this.properties.getClientId()); - } - container.setDestinationName(this.properties.getDestination()); - if (this.properties.getMessageSelector() != null) { - container.setMessageSelector(this.properties.getMessageSelector()); - } - container.setPubSubDomain(this.jmsProperties.isPubSubDomain()); - if (this.properties.getMessageSelector() != null - && listenerProperties.getAcknowledgeMode() != null) { - container.setSessionAcknowledgeMode(listenerProperties.getAcknowledgeMode().getMode()); - } - if (this.properties.getSubscriptionDurable() != null) { - container.setSubscriptionDurable(this.properties.getSubscriptionDurable()); - } - if (this.properties.getSubscriptionName() != null) { - container.setSubscriptionName(this.properties.getSubscriptionName()); - } - if (this.properties.getSubscriptionShared() != null) { - container.setSubscriptionShared(this.properties.getSubscriptionShared()); - } - return container; - } - -} diff --git a/functions/supplier/jms-supplier/src/main/java/org/springframework/cloud/fn/supplier/jms/JmsSupplierProperties.java b/functions/supplier/jms-supplier/src/main/java/org/springframework/cloud/fn/supplier/jms/JmsSupplierProperties.java deleted file mode 100644 index 245f9cb8c..000000000 --- a/functions/supplier/jms-supplier/src/main/java/org/springframework/cloud/fn/supplier/jms/JmsSupplierProperties.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.jms; - -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * Properties for the JMS Supplier. - * - * @author Gary Russell - * - */ -@ConfigurationProperties(prefix = "jms.supplier") -@Validated -public class JmsSupplierProperties { - - /** - * True to enable transactions and select a DefaultMessageListenerContainer, false to - * select a SimpleMessageListenerContainer. - */ - private boolean sessionTransacted = true; - - /** - * Client id for durable subscriptions. - */ - private String clientId; - - /** - * The destination from which to receive messages (queue or topic). - */ - private String destination; - - /** - * The name of a durable or shared subscription. - */ - private String subscriptionName; - - /** - * A selector for messages. - */ - private String messageSelector = null; - - /** - * True for a durable subscription. - */ - private Boolean subscriptionDurable; - - /** - * True for a shared subscription. - */ - private Boolean subscriptionShared; - - public String getClientId() { - return this.clientId; - } - - public void setClientId(String clientId) { - this.clientId = clientId; - } - - @NotNull - public String getDestination() { - return this.destination; - } - - public void setDestination(String destination) { - this.destination = destination; - } - - public String getSubscriptionName() { - return this.subscriptionName; - } - - public void setSubscriptionName(String subscriptionName) { - this.subscriptionName = subscriptionName; - } - - public String getMessageSelector() { - return this.messageSelector; - } - - public void setMessageSelector(String messageSelector) { - this.messageSelector = messageSelector; - } - - public Boolean getSubscriptionDurable() { - return this.subscriptionDurable; - } - - public void setSubscriptionDurable(Boolean subscriptionDurable) { - this.subscriptionDurable = subscriptionDurable; - } - - public Boolean getSubscriptionShared() { - return this.subscriptionShared; - } - - public void setSubscriptionShared(Boolean subscriptionShared) { - this.subscriptionShared = subscriptionShared; - } - - public boolean isSessionTransacted() { - return this.sessionTransacted; - } - - public void setSessionTransacted(boolean sessionTransacted) { - this.sessionTransacted = sessionTransacted; - } -} diff --git a/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/AbstractJmsSupplierTests.java b/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/AbstractJmsSupplierTests.java deleted file mode 100644 index 81d8675d5..000000000 --- a/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/AbstractJmsSupplierTests.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.jms; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.integration.jms.JmsMessageDrivenEndpoint; -import org.springframework.test.annotation.DirtiesContext; - -@SpringBootTest -@DirtiesContext -public class AbstractJmsSupplierTests { - - @Autowired - protected JmsMessageDrivenEndpoint endpoint; - - @SpringBootApplication - public static class JmsSupplierTestApplication { - - } -} diff --git a/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/PropertiesPopulated1Tests.java b/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/PropertiesPopulated1Tests.java deleted file mode 100644 index 4b38d482c..000000000 --- a/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/PropertiesPopulated1Tests.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.jms; - -import jakarta.jms.Session; -import org.junit.jupiter.api.Test; - -import org.springframework.integration.test.util.TestUtils; -import org.springframework.jms.listener.AbstractMessageListenerContainer; -import org.springframework.jms.listener.SimpleMessageListenerContainer; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { - "jms.supplier.sessionTransacted = false", - "jms.supplier.destination = topic", - "jms.supplier.messageSelector = JMSCorrelationId=foo", - "jms.supplier.subscriptionDurable = false", - "jms.supplier.subscriptionShared = false", - "spring.jms.listener.acknowledgeMode = DUPS_OK", - "spring.jms.listener.concurrency = 3", - "spring.jms.listener.maxConcurrency = 4", - "spring.jms.pubSubDomain = true" -}) -public class PropertiesPopulated1Tests extends AbstractJmsSupplierTests { - - @Test - public void test() { - AbstractMessageListenerContainer container = TestUtils.getPropertyValue(this.endpoint, "listenerContainer", - AbstractMessageListenerContainer.class); - assertThat(container).isInstanceOf(SimpleMessageListenerContainer.class); - assertThat(TestUtils.getPropertyValue(container, "sessionAcknowledgeMode")).isEqualTo(Session.DUPS_OK_ACKNOWLEDGE); - assertThat(TestUtils.getPropertyValue(container, "sessionTransacted", Boolean.class)).isFalse(); - assertThat(TestUtils.getPropertyValue(container, "clientId")).isNull(); - assertThat(TestUtils.getPropertyValue(container, "destination")).isEqualTo("topic"); - assertThat(TestUtils.getPropertyValue(container, "messageSelector")).isEqualTo("JMSCorrelationId=foo"); - assertThat(TestUtils.getPropertyValue(container, "subscriptionDurable", Boolean.class)).isFalse(); - assertThat(TestUtils.getPropertyValue(container, "subscriptionShared", Boolean.class)).isFalse(); - assertThat(TestUtils.getPropertyValue(container, "concurrentConsumers")).isEqualTo(3); - assertThat(TestUtils.getPropertyValue(container, "pubSubDomain", Boolean.class)).isTrue(); - } -} diff --git a/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/PropertiesPopulated2Tests.java b/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/PropertiesPopulated2Tests.java deleted file mode 100644 index c9b7e0cba..000000000 --- a/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/PropertiesPopulated2Tests.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.jms; - -import jakarta.jms.Session; -import org.junit.jupiter.api.Test; - -import org.springframework.integration.test.util.TestUtils; -import org.springframework.jms.listener.AbstractMessageListenerContainer; -import org.springframework.jms.listener.DefaultMessageListenerContainer; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { - "jms.supplier.sessionTransacted = true", "jms.supplier.clientId = client", "jms.supplier.destination = topic", - "jms.supplier.subscriptionName = subName", "jms.supplier.subscriptionDurable = true", - "jms.supplier.subscriptionShared = false", "spring.jms.listener.acknowledgeMode = AUTO", - "spring.jms.listener.concurrency = 3", - "spring.jms.listener.maxConcurrency = 4" -}) -public class PropertiesPopulated2Tests extends AbstractJmsSupplierTests { - - @Test - public void test() { - - AbstractMessageListenerContainer container = TestUtils.getPropertyValue(this.endpoint, "listenerContainer", - AbstractMessageListenerContainer.class); - assertThat(container).isInstanceOf(DefaultMessageListenerContainer.class); - - assertThat(TestUtils.getPropertyValue(container, "sessionAcknowledgeMode")).isEqualTo(Session.AUTO_ACKNOWLEDGE); - assertThat(TestUtils.getPropertyValue(container, "sessionTransacted", Boolean.class)).isTrue(); - assertThat(TestUtils.getPropertyValue(container, "clientId")).isEqualTo("client"); - assertThat(TestUtils.getPropertyValue(container, "destination")).isEqualTo("topic"); - assertThat(TestUtils.getPropertyValue(container, "subscriptionDurable", Boolean.class)).isTrue(); - assertThat(TestUtils.getPropertyValue(container, "subscriptionName")).isEqualTo("subName"); - assertThat(TestUtils.getPropertyValue(container, "subscriptionShared", Boolean.class)).isFalse(); - assertThat(TestUtils.getPropertyValue(container, "concurrentConsumers")).isEqualTo(3); - assertThat(TestUtils.getPropertyValue(container, "maxConcurrentConsumers")).isEqualTo(4); - assertThat(TestUtils.getPropertyValue(container, "pubSubDomain", Boolean.class)).isTrue(); - } -} diff --git a/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/PropertiesPopulated3Tests.java b/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/PropertiesPopulated3Tests.java deleted file mode 100644 index d1b826e91..000000000 --- a/functions/supplier/jms-supplier/src/test/java/org/springframework/cloud/fn/supplier/jms/PropertiesPopulated3Tests.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.jms; - -import java.util.function.Supplier; - -import jakarta.jms.Session; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.jms.core.JmsTemplate; -import org.springframework.jms.listener.AbstractMessageListenerContainer; -import org.springframework.jms.listener.DefaultMessageListenerContainer; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { - "jms.supplier.sessionTransacted = true", "jms.supplier.destination = jmssource.test.queue", - "jms.supplier.messageSelector = JMSCorrelationId=foo", - "jms.supplier.subscriptionDurable = false", "jms.supplier.subscriptionShared = false", - "spring.jms.listener.acknowledgeMode = AUTO", - "spring.jms.listener.concurrency = 3", - "spring.jms.listener.maxConcurrency = 4", - "spring.jms.pubSubDomain = false" }) -public class PropertiesPopulated3Tests extends AbstractJmsSupplierTests { - - @Autowired - private JmsTemplate template; - - @Autowired - private Supplier>> jmsSupplier; - - @Test - public void test() throws Exception { - AbstractMessageListenerContainer container = TestUtils.getPropertyValue(this.endpoint, "listenerContainer", - AbstractMessageListenerContainer.class); - assertThat(container).isInstanceOf(DefaultMessageListenerContainer.class); - assertThat(TestUtils.getPropertyValue(container, "sessionAcknowledgeMode")).isEqualTo(Session.AUTO_ACKNOWLEDGE); - assertThat(TestUtils.getPropertyValue(container, "sessionTransacted", Boolean.class)).isTrue(); - assertThat(TestUtils.getPropertyValue(container, "destination")).isEqualTo("jmssource.test.queue"); - assertThat(TestUtils.getPropertyValue(container, "messageSelector")).isEqualTo("JMSCorrelationId=foo"); - assertThat(TestUtils.getPropertyValue(container, "subscriptionDurable", Boolean.class)).isFalse(); - assertThat(TestUtils.getPropertyValue(container, "subscriptionShared", Boolean.class)).isFalse(); - assertThat(TestUtils.getPropertyValue(container, "concurrentConsumers")).isEqualTo(3); - assertThat(TestUtils.getPropertyValue(container, "maxConcurrentConsumers")).isEqualTo(4); - assertThat(TestUtils.getPropertyValue(container, "pubSubDomain", Boolean.class)).isFalse(); - - final Flux> messageFlux = jmsSupplier.get(); - - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(message.getPayload()) - .isEqualTo("Hello, world!"); - } - ) - .thenCancel() - .verifyLater(); - - template.convertAndSend("jmssource.test.queue", "Hello, world!"); - - stepVerifier.verify(); - } -} diff --git a/functions/supplier/kafka-supplier/README.adoc b/functions/supplier/kafka-supplier/README.adoc deleted file mode 100644 index f84bc8c82..000000000 --- a/functions/supplier/kafka-supplier/README.adoc +++ /dev/null @@ -1,28 +0,0 @@ -# Apache Kafka (Consumer) Supplier - -A `Supplier` that allows to consume messages from Apache Kafka topic. - - -## Beans for injection - -The `KafkaSupplierConfiguration` is an auto-configuration, so no need to import anything. - -The `Supplier>> kafkaSupplier` bean can be injected into the target service for consuming data from Kafka topics. - -## Configuration Options - -All configuration properties are prefixed with `kafka.supplier`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/kafka/KafkaSupplierProperties.java[KafkaSupplierProperties]. -Also, this artifact fully depends on Spring for Apache Kafka auto-configuration and injects a `ConcurrentKafkaListenerContainerFactory` from there. - -A `ComponentCustomizer>` bean can be added in the target project to provide any custom options for the `KafkaMessageDrivenChannelAdapterSpec` configuration used by the `kafkaSupplier`. - -See more information about `KafkaMessageDrivenChannelAdapter` configuration and behavior in Spring Integration https://docs.spring.io/spring-integration/docs/current/reference/html/kafka.html#kafka-inbound[documentation]. - -## Tests - - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/kafka-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes an Apache Kafka source. diff --git a/functions/supplier/kafka-supplier/pom.xml b/functions/supplier/kafka-supplier/pom.xml deleted file mode 100644 index 815d26256..000000000 --- a/functions/supplier/kafka-supplier/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - - kafka-supplier - kafka-supplier - Apache Kafka Supplier based on Apache Kafka Consumer - - - - org.springframework.integration - spring-integration-kafka - - - - org.springframework.kafka - spring-kafka-test - test - - - - diff --git a/functions/supplier/kafka-supplier/src/main/java/org/springframework/cloud/fn/supplier/kafka/KafkaSupplierConfiguration.java b/functions/supplier/kafka-supplier/src/main/java/org/springframework/cloud/fn/supplier/kafka/KafkaSupplierConfiguration.java deleted file mode 100644 index e55aceebf..000000000 --- a/functions/supplier/kafka-supplier/src/main/java/org/springframework/cloud/fn/supplier/kafka/KafkaSupplierConfiguration.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.kafka; - -import java.util.function.Supplier; -import java.util.regex.Pattern; - -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.boot.autoconfigure.AutoConfiguration; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.expression.spel.support.StandardEvaluationContext; -import org.springframework.integration.context.IntegrationContextUtils; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.kafka.dsl.Kafka; -import org.springframework.integration.kafka.dsl.KafkaMessageDrivenChannelAdapterSpec; -import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; -import org.springframework.kafka.listener.adapter.RecordFilterStrategy; -import org.springframework.kafka.support.converter.BatchMessageConverter; -import org.springframework.kafka.support.converter.BatchMessagingMessageConverter; -import org.springframework.kafka.support.converter.RecordMessageConverter; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; - -/** - * An auto-configuration for Kafka Supplier. - * Uses a {@link KafkaMessageDrivenChannelAdapterSpec} to consumer records from Kafka topic. - * - * @author Artem Bilan - * - * @since 4.0 - */ -@AutoConfiguration(after = KafkaAutoConfiguration.class) -@EnableConfigurationProperties(KafkaSupplierProperties.class) -public class KafkaSupplierConfiguration { - - @Bean - public Supplier>> kafkaSupplier(Publisher> kafkaSupplierPublisher) { - return () -> Flux.from(kafkaSupplierPublisher); - } - - @Bean - public Publisher> kafkaSupplierPublisher( - KafkaMessageDrivenChannelAdapterSpec kafkaMessageDrivenChannelAdapterSpec) { - - return IntegrationFlow.from(kafkaMessageDrivenChannelAdapterSpec) - .toReactivePublisher(true); - } - - @Bean - public KafkaMessageDrivenChannelAdapterSpec kafkaMessageDrivenChannelAdapterSpec( - KafkaSupplierProperties kafkaSupplierProperties, - ConcurrentKafkaListenerContainerFactory kafkaListenerContainerFactory, - ObjectProvider recordMessageConverterProvider, - ObjectProvider> recordFilterStrategyProvider, - ObjectProvider batchMessageConverterProvider, - @Nullable ComponentCustomizer> kafkaChannelAdapterComponentCustomizer) { - - ConcurrentMessageListenerContainer container; - - Pattern topicPattern = kafkaSupplierProperties.getTopicPattern(); - if (topicPattern != null) { - container = kafkaListenerContainerFactory.createContainer(topicPattern); - } - else { - container = kafkaListenerContainerFactory.createContainer(kafkaSupplierProperties.getTopics()); - } - - KafkaMessageDrivenChannelAdapter.ListenerMode listenerMode = - Boolean.TRUE.equals(kafkaListenerContainerFactory.isBatchListener()) - ? KafkaMessageDrivenChannelAdapter.ListenerMode.batch - : KafkaMessageDrivenChannelAdapter.ListenerMode.record; - - KafkaMessageDrivenChannelAdapterSpec kafkaMessageDrivenChannelAdapterSpec = - Kafka.messageDrivenChannelAdapter(container, listenerMode) - .ackDiscarded(kafkaSupplierProperties.isAckDiscarded()) - .autoStartup(false); - - RecordMessageConverter recordMessageConverter = recordMessageConverterProvider.getIfUnique(); - - if (KafkaMessageDrivenChannelAdapter.ListenerMode.batch.equals(listenerMode)) { - BatchMessageConverter batchMessageConverter = - batchMessageConverterProvider.getIfUnique( - () -> new BatchMessagingMessageConverter(recordMessageConverter)); - - kafkaMessageDrivenChannelAdapterSpec.batchMessageConverter(batchMessageConverter); - } - else if (recordMessageConverter != null) { - kafkaMessageDrivenChannelAdapterSpec.recordMessageConverter(recordMessageConverter); - } - - recordFilterStrategyProvider.ifUnique(kafkaMessageDrivenChannelAdapterSpec::recordFilterStrategy); - - if (kafkaChannelAdapterComponentCustomizer != null) { - kafkaChannelAdapterComponentCustomizer.customize(kafkaMessageDrivenChannelAdapterSpec); - } - - return kafkaMessageDrivenChannelAdapterSpec; - } - - @Bean - @ConditionalOnMissingBean - @ConditionalOnProperty("kafka.supplier.recordFilter") - RecordFilterStrategy recordFilterStrategy(KafkaSupplierProperties kafkaSupplierProperties, - BeanFactory beanFactory) { - - StandardEvaluationContext evaluationContext = IntegrationContextUtils.getEvaluationContext(beanFactory); - - return consumerRecord -> - Boolean.TRUE.equals( - kafkaSupplierProperties.getRecordFilter() - .getValue(evaluationContext, consumerRecord, Boolean.class)); - } - -} diff --git a/functions/supplier/kafka-supplier/src/main/java/org/springframework/cloud/fn/supplier/kafka/KafkaSupplierProperties.java b/functions/supplier/kafka-supplier/src/main/java/org/springframework/cloud/fn/supplier/kafka/KafkaSupplierProperties.java deleted file mode 100644 index 6a509ac6a..000000000 --- a/functions/supplier/kafka-supplier/src/main/java/org/springframework/cloud/fn/supplier/kafka/KafkaSupplierProperties.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.kafka; - -import java.util.regex.Pattern; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; - -/** - * Auto-configuration properties for the Kafka Supplier. - * - * @author Artem Bilan - * - * @since 4.0 - */ -@ConfigurationProperties("kafka.supplier") -public class KafkaSupplierProperties { - - /** - * Apache Kafka topics to subscribe. - */ - private String[] topics; - - /** - * Apache Kafka topics pattern to subscribe. - */ - private Pattern topicPattern; - - /** - * Whether to acknowledge discarded records after 'RecordFilterStrategy'. - */ - private boolean ackDiscarded; - - /** - * SpEL expression for 'RecordFilterStrategy' with a 'ConsumerRecord' as a root object. - */ - Expression recordFilter; - - public String[] getTopics() { - return this.topics; - } - - public void setTopics(String[] topics) { - this.topics = topics; - } - - public Pattern getTopicPattern() { - return this.topicPattern; - } - - public void setTopicPattern(Pattern topicPattern) { - this.topicPattern = topicPattern; - } - - public boolean isAckDiscarded() { - return this.ackDiscarded; - } - - public void setAckDiscarded(boolean ackDiscarded) { - this.ackDiscarded = ackDiscarded; - } - - public Expression getRecordFilter() { - return this.recordFilter; - } - - public void setRecordFilter(Expression recordFilter) { - this.recordFilter = recordFilter; - } - -} diff --git a/functions/supplier/kafka-supplier/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/functions/supplier/kafka-supplier/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports deleted file mode 100644 index 3a9a43ac0..000000000 --- a/functions/supplier/kafka-supplier/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports +++ /dev/null @@ -1 +0,0 @@ -org.springframework.cloud.fn.supplier.kafka.KafkaSupplierConfiguration diff --git a/functions/supplier/kafka-supplier/src/test/java/org/springframework/cloud/fn/supplier/kafka/KafkaSupplierTests.java b/functions/supplier/kafka-supplier/src/test/java/org/springframework/cloud/fn/supplier/kafka/KafkaSupplierTests.java deleted file mode 100644 index b4621378b..000000000 --- a/functions/supplier/kafka-supplier/src/test/java/org/springframework/cloud/fn/supplier/kafka/KafkaSupplierTests.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2023-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.kafka; - -import java.time.Duration; -import java.util.List; -import java.util.function.Supplier; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.boot.autoconfigure.AutoConfigurations; -import org.springframework.boot.autoconfigure.integration.IntegrationAutoConfiguration; -import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.cloud.fn.common.config.SpelExpressionConverterConfiguration; -import org.springframework.context.ApplicationContext; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.test.EmbeddedKafkaBroker; -import org.springframework.messaging.Message; - -/** - * @author Artem Bilan - * - * @since 4.0 - */ -public class KafkaSupplierTests { - - static final EmbeddedKafkaBroker EMBEDDED_KAFKA = - new EmbeddedKafkaBroker(1, true, 1) - .brokerListProperty("spring.kafka.bootstrap-servers"); - - final ApplicationContextRunner contextRunner = new ApplicationContextRunner() - .withConfiguration(AutoConfigurations.of( - IntegrationAutoConfiguration.class, - KafkaAutoConfiguration.class, - KafkaSupplierConfiguration.class, - SpelExpressionConverterConfiguration.class)); - - @BeforeAll - static void initializeEmbeddedKafka() { - EMBEDDED_KAFKA.afterPropertiesSet(); - } - - @Test - void recordModeAndTopicPattern() { - this.contextRunner.withPropertyValues( - "spring.kafka.template.defaultTopic=defaultTopic", - "spring.kafka.consumer.group-id=test-group1", - "spring.kafka.consumer.auto-offset-reset=earliest", - "kafka.supplier.topicPattern=default.+") - .run((context) -> { - String testPayload1 = "test data #1"; - String testPayload2 = "test data #2"; - - KafkaTemplate kafkaTemplate = getKafkaTemplate(context); - kafkaTemplate.sendDefault(testPayload1); - kafkaTemplate.sendDefault(testPayload2); - kafkaTemplate.flush(); - - Supplier>> kafkaSupplier = getKafkaSupplier(context); - StepVerifier.create( - kafkaSupplier.get() - .map(Message::getPayload) - .cast(String.class)) - .expectNext(testPayload1, testPayload2) - .thenCancel() - .verify(Duration.ofSeconds(30)); - }); - } - - @Test - void batchMode() { - this.contextRunner.withPropertyValues( - "spring.kafka.consumer.group-id=test-group2", - "spring.kafka.consumer.auto-offset-reset=earliest", - "spring.kafka.listener.type=BATCH", - "kafka.supplier.topics=testTopic1,testTopic2", - "kafka.supplier.recordFilter=value() == 'test data #2'") - .run((context) -> { - String testPayload1 = "test data #1"; - String testPayload2 = "test data #2"; - - Supplier>> kafkaSupplier = getKafkaSupplier(context); - StepVerifier stepVerifier = StepVerifier.create( - kafkaSupplier.get() - .map(Message::getPayload) - .cast(List.class)) - .expectNext(List.of(testPayload1)) - .thenCancel() - .verifyLater(); - - - KafkaTemplate kafkaTemplate = getKafkaTemplate(context); - kafkaTemplate.send("testTopic1", testPayload1); - kafkaTemplate.send("testTopic2", testPayload2); - kafkaTemplate.flush(); - - stepVerifier.verify(Duration.ofSeconds(30)); - }); - } - - @SuppressWarnings("unchecked") - private static KafkaTemplate getKafkaTemplate(ApplicationContext applicationContext) { - return applicationContext.getBean(KafkaTemplate.class); - } - - @SuppressWarnings("unchecked") - private static Supplier>> getKafkaSupplier(ApplicationContext applicationContext) { - return (Supplier>>) applicationContext.getBean("kafkaSupplier"); - } - -} diff --git a/functions/supplier/mail-supplier/README.adoc b/functions/supplier/mail-supplier/README.adoc deleted file mode 100644 index e5f1f4a16..000000000 --- a/functions/supplier/mail-supplier/README.adoc +++ /dev/null @@ -1,36 +0,0 @@ -# Mail Supplier - -This module provides a File supplier that can be reused and composed in other applications. -The `Supplier` uses the mail IMAP and POP3 support from Spring Integration. -The `mailSupplier` bean is implemented as a `java.util.function.Supplier`. -This supplier gives you a reactive stream of emails from the provided url (imap or pop3). -The supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and receive the data. - -## Beans for injection - -You can import the `MailSupplierConfiguration` in the application and then inject the following bean. - -`mailSupplier` - -You need to inject this as `Supplier>>`. - -You can use `mailSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `mail.supplier`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/mail/MailSupplierProperties.java[MailSupplierProperties]. - -A `ComponentCustomizer>` (or `ComponentCustomizer` when `mail.supplier.idle-imap = true`) bean can be added in the target project to provide any custom options for the `MailInboundChannelAdapterSpec` (or `ImapIdleChannelAdapterSpec`, respectively) configuration used by the `mailSupplier`. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/mail[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/mail-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a Mail Source. diff --git a/functions/supplier/mail-supplier/pom.xml b/functions/supplier/mail-supplier/pom.xml deleted file mode 100644 index bfc0447f4..000000000 --- a/functions/supplier/mail-supplier/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - mail-supplier - mail-supplier - Mail Supplier - - - - org.springframework.integration - spring-integration-mail - - - org.eclipse.angus - jakarta.mail - 1.0.0 - - - com.icegreen - greenmail - 2.0.0 - test - - - diff --git a/functions/supplier/mail-supplier/src/main/java/org/springframework/cloud/fn/supplier/mail/MailSupplierConfiguration.java b/functions/supplier/mail-supplier/src/main/java/org/springframework/cloud/fn/supplier/mail/MailSupplierConfiguration.java deleted file mode 100644 index 35ba4e825..000000000 --- a/functions/supplier/mail-supplier/src/main/java/org/springframework/cloud/fn/supplier/mail/MailSupplierConfiguration.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mail; - -import java.util.Arrays; -import java.util.List; -import java.util.Properties; -import java.util.function.Supplier; - -import jakarta.mail.URLName; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.core.MessageSource; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.dsl.MessageProducerSpec; -import org.springframework.integration.dsl.MessageSourceSpec; -import org.springframework.integration.endpoint.MessageProducerSupport; -import org.springframework.integration.endpoint.ReactiveMessageSourceProducer; -import org.springframework.integration.mail.MailHeaders; -import org.springframework.integration.mail.dsl.ImapIdleChannelAdapterSpec; -import org.springframework.integration.mail.dsl.Mail; -import org.springframework.integration.mail.dsl.MailInboundChannelAdapterSpec; -import org.springframework.integration.transformer.support.AbstractHeaderValueMessageProcessor; -import org.springframework.integration.transformer.support.HeaderValueMessageProcessor; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; - -/** - * Mail supplier components. - * - * @author Amol - * @author Artem Bilan - * @author Chris Schaefer - * @author Soby Chacko - * @author Corneil du Plessis - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties(MailSupplierProperties.class) -public class MailSupplierConfiguration { - - - final private MailSupplierProperties properties; - - public MailSupplierConfiguration(MailSupplierProperties properties) { - this.properties = properties; - } - - @Bean - public Publisher> mailInboundFlow(MessageProducerSupport messageProducer) { - - return IntegrationFlow.from(messageProducer) - .transform(Mail.toStringTransformer(this.properties.getCharset())) - .enrichHeaders(h -> h - .defaultOverwrite(true) - .header(MailHeaders.TO, arrayToListProcessor(MailHeaders.TO)) - .header(MailHeaders.CC, arrayToListProcessor(MailHeaders.CC)) - .header(MailHeaders.BCC, arrayToListProcessor(MailHeaders.BCC))) - .toReactivePublisher(true); - } - - @Bean - public Supplier>> mailSupplier(Publisher> messagePublisher) { - return () -> Flux.from(messagePublisher); - } - - private HeaderValueMessageProcessor arrayToListProcessor(final String header) { - return new AbstractHeaderValueMessageProcessor>() { - - @Override - public List processMessage(Message message) { - return Arrays.asList(message.getHeaders().get(header, String[].class)); - } - - }; - } - - @Bean("mailChannelAdapter") - @ConditionalOnProperty("mail.supplier.idle-imap") - MessageProducerSpec imapIdleProducer( - @Nullable ComponentCustomizer imapIdleChannelAdapterSpecCustomizer) { - - URLName urlName = this.properties.getUrl(); - ImapIdleChannelAdapterSpec imapIdleChannelAdapterSpec = Mail.imapIdleAdapter(urlName.toString()) - .shouldDeleteMessages(this.properties.isDelete()) - .userFlag(this.properties.getUserFlag()) - .javaMailProperties(getJavaMailProperties(urlName)) - .selectorExpression(this.properties.getExpression()) - .shouldMarkMessagesAsRead(this.properties.isMarkAsRead()); - - if (imapIdleChannelAdapterSpecCustomizer != null) { - imapIdleChannelAdapterSpecCustomizer.customize(imapIdleChannelAdapterSpec); - } - return imapIdleChannelAdapterSpec; - } - - @Bean - @ConditionalOnProperty(value = "mail.supplier.idle-imap", matchIfMissing = true, havingValue = "false") - MessageSourceSpec mailMessageSource( - @Nullable ComponentCustomizer> - mailInboundChannelAdapterSpecCustomizer) { - - MailInboundChannelAdapterSpec adapterSpec; - URLName urlName = this.properties.getUrl(); - switch (urlName.getProtocol().toUpperCase()) { - case "IMAP": - case "IMAPS": - adapterSpec = getImapChannelAdapterSpec(urlName); - break; - case "POP3": - case "POP3S": - adapterSpec = getPop3ChannelAdapterSpec(urlName); - break; - default: - throw new IllegalArgumentException( - "Unsupported mail protocol: " + urlName.getProtocol()); - } - adapterSpec.javaMailProperties(getJavaMailProperties(urlName)) - .userFlag(this.properties.getUserFlag()) - .selectorExpression(this.properties.getExpression()) - .shouldDeleteMessages(this.properties.isDelete()); - - if (mailInboundChannelAdapterSpecCustomizer != null) { - mailInboundChannelAdapterSpecCustomizer.customize(adapterSpec); - } - - return adapterSpec; - } - - @Bean("mailChannelAdapter") - @ConditionalOnProperty(value = "mail.supplier.idle-imap", matchIfMissing = true, havingValue = "false") - MessageProducerSupport mailMessageProducer(MessageSource mailMessageSource) { - return new ReactiveMessageSourceProducer(mailMessageSource); - } - - /** - * Method to build Mail Channel Adapter for POP3. - * @param urlName Mail source URL. - * @return Mail Channel for POP3 - */ - @SuppressWarnings("rawtypes") - private MailInboundChannelAdapterSpec getPop3ChannelAdapterSpec(URLName urlName) { - return Mail.pop3InboundAdapter(urlName.toString()); - } - - /** - * Method to build Mail Channel Adapter for IMAP. - * @param urlName Mail source URL. - * @return Mail Channel for IMAP - */ - @SuppressWarnings("rawtypes") - private MailInboundChannelAdapterSpec getImapChannelAdapterSpec(URLName urlName) { - return Mail.imapInboundAdapter(urlName.toString()) - .shouldMarkMessagesAsRead(this.properties.isMarkAsRead()); - } - - private Properties getJavaMailProperties(URLName urlName) { - Properties javaMailProperties = new Properties(); - - switch (urlName.getProtocol().toUpperCase()) { - case "IMAP": - javaMailProperties.setProperty("mail.imap.socketFactory.class", "javax.net.SocketFactory"); - javaMailProperties.setProperty("mail.imap.socketFactory.fallback", "false"); - javaMailProperties.setProperty("mail.store.protocol", "imap"); - break; - - case "IMAPS": - javaMailProperties.setProperty("mail.imap.socketFactory.class", "javax.net.ssl.SSLSocketFactory"); - javaMailProperties.setProperty("mail.imap.socketFactory.fallback", "false"); - javaMailProperties.setProperty("mail.store.protocol", "imaps"); - break; - - case "POP3": - javaMailProperties.setProperty("mail.pop3.socketFactory.class", "javax.net.SocketFactory"); - javaMailProperties.setProperty("mail.pop3.socketFactory.fallback", "false"); - javaMailProperties.setProperty("mail.store.protocol", "pop3"); - break; - - case "POP3S": - javaMailProperties.setProperty("mail.pop3.socketFactory.class", "javax.net.ssl.SSLSocketFactory"); - javaMailProperties.setProperty("mail.pop3.socketFactory.fallback", "false"); - javaMailProperties.setProperty("mail.store.protocol", "pop3s"); - break; - } - - javaMailProperties.putAll(this.properties.getJavaMailProperties()); - return javaMailProperties; - } - -} diff --git a/functions/supplier/mail-supplier/src/main/java/org/springframework/cloud/fn/supplier/mail/MailSupplierProperties.java b/functions/supplier/mail-supplier/src/main/java/org/springframework/cloud/fn/supplier/mail/MailSupplierProperties.java deleted file mode 100644 index 05ccde233..000000000 --- a/functions/supplier/mail-supplier/src/main/java/org/springframework/cloud/fn/supplier/mail/MailSupplierProperties.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mail; - -import java.util.Properties; - -import jakarta.mail.URLName; -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.integration.mail.AbstractMailReceiver; -import org.springframework.validation.annotation.Validated; - -/** - * Properties for the file supplier. - * - * @author Gary Russell - * @author Artem Bilan - * @author Soby Chacko - */ -@ConfigurationProperties("mail.supplier") -@Validated -public class MailSupplierProperties { - - /** - * Mail connection URL for connection to Mail server e.g. - * 'imaps://username:password@imap.server.com:993/Inbox'. - */ - private URLName url; - - /** - * Set to true to mark email as read. - */ - private boolean markAsRead = false; - - /** - * Set to true to delete email after download. - */ - private boolean delete = false; - - /** - * Set to true to use IdleImap Configuration. - */ - private boolean idleImap = false; - - /** - * JavaMail properties as a new line delimited string of name-value pairs, e.g. - * 'foo=bar\n baz=car'. - */ - private Properties javaMailProperties = new Properties(); - - /** - * Configure a SpEL expression to select messages. - */ - private String expression = "true"; - - /** - * The charset for byte[] mail-to-string transformation. - */ - private String charset = "UTF-8"; - - /** - * The flag to mark messages when the server does not support \Recent. - */ - private String userFlag = AbstractMailReceiver.DEFAULT_SI_USER_FLAG; - - /** - * @return the markAsRead - */ - public boolean isMarkAsRead() { - return this.markAsRead; - } - - /** - * @param markAsRead the markAsRead to set - */ - public void setMarkAsRead(boolean markAsRead) { - this.markAsRead = markAsRead; - } - - /** - * @return the delete - */ - public boolean isDelete() { - return this.delete; - } - - /** - * @param delete the delete to set - */ - public void setDelete(boolean delete) { - this.delete = delete; - } - - /** - * @return the idleImap - */ - public boolean isIdleImap() { - return this.idleImap; - } - - /** - * @param idleImap the idleImap to set - */ - public void setIdleImap(boolean idleImap) { - this.idleImap = idleImap; - } - - /** - * @return the javaMailProperties - */ - @NotNull - public Properties getJavaMailProperties() { - return this.javaMailProperties; - } - - /** - * @param javaMailProperties the javaMailProperties to set - */ - public void setJavaMailProperties(Properties javaMailProperties) { - this.javaMailProperties = javaMailProperties; - } - - /** - * @return the url - */ - @NotNull - public URLName getUrl() { - return this.url; - } - - /** - * @param url the url to set - */ - public void setUrl(URLName url) { - this.url = url; - } - - /** - * @return the expression - */ - @NotNull - public String getExpression() { - return this.expression; - } - - /** - * @param expression the expression to set - */ - public void setExpression(String expression) { - this.expression = expression; - } - - @NotNull - public String getCharset() { - return this.charset; - } - - public void setCharset(String charset) { - this.charset = charset; - } - - @NotNull - public String getUserFlag() { - return this.userFlag; - } - - public void setUserFlag(String userFlag) { - this.userFlag = userFlag; - } -} diff --git a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/AbstractMailSupplierTests.java b/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/AbstractMailSupplierTests.java deleted file mode 100644 index af5362cd3..000000000 --- a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/AbstractMailSupplierTests.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mail; - -import java.util.function.Supplier; - -import com.icegreen.greenmail.user.GreenMailUser; -import com.icegreen.greenmail.util.GreenMail; -import com.icegreen.greenmail.util.GreenMailUtil; -import com.icegreen.greenmail.util.ServerSetup; -import com.icegreen.greenmail.util.ServerSetupTest; -import org.junit.jupiter.api.BeforeAll; -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.integration.dsl.StandardIntegrationFlow; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = { - "mail.supplier.mark-as-read=true", - "mail.supplier.delete=false", - "mail.supplier.user-flag=testSIUserFlag", - "mail.supplier.java-mail-properties=mail.imap.socketFactory.fallback=true\\n mail.store.protocol=imap\\n mail.debug=true"}) -@DirtiesContext -public abstract class AbstractMailSupplierTests { - - protected static GreenMail mailServer; - - protected static GreenMailUser mailUser; - - @Autowired - protected Supplier>> mailSupplier; - - @Autowired - protected StandardIntegrationFlow integrationFlow; - - protected void sendMessage(String subject, String body) { - mailUser.deliver(GreenMailUtil.createTextEmail("bar@bax", "test@test", subject, body, mailServer.getSmtp().getServerSetup())); - } - - @BeforeAll - public static void setup() { - ServerSetup imap = ServerSetupTest.IMAP.dynamicPort(); - imap.setServerStartupTimeout(10000); - ServerSetup pop3 = ServerSetupTest.POP3.dynamicPort(); - pop3.setServerStartupTimeout(10000); - ServerSetup smtp = ServerSetupTest.SMTP.dynamicPort(); - smtp.setServerStartupTimeout(10000); - - mailServer = new GreenMail(new ServerSetup[] {imap, pop3, smtp}); - mailUser = mailServer.setUser("user", "pw"); - mailServer.start(); - } - - @DynamicPropertySource - static void mongoDbProperties(DynamicPropertyRegistry registry) { - registry.add("test.mail.server.imap.port", mailServer.getImap().getServerSetup()::getPort); - registry.add("test.mail.server.pop3.port", mailServer.getPop3().getServerSetup()::getPort); - registry.add("test.mail.server.smtp.port", mailServer.getSmtp().getServerSetup()::getPort); - } - - @SpringBootApplication - public static class MailSupplierTestApplication { - - } - -} diff --git a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapFailTests.java b/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapFailTests.java deleted file mode 100644 index 63b89f1a4..000000000 --- a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapFailTests.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mail; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.integration.mail.transformer.MailToStringTransformer; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { - "mail.supplier.url=imap://user:pw@localhost:${test.mail.server.imap.port}/INBOX", - "mail.supplier.charset=cp1251"}) -public class ImapFailTests extends AbstractMailSupplierTests { - - @Autowired - protected MailToStringTransformer mailToStringTransformer; - - - @Test - public void testSimpleTest() { - // given - sendMessage("test", "foo"); - // when - final Flux> messageFlux = mailSupplier.get(); - // then - assertThat(TestUtils.getPropertyValue(mailToStringTransformer, "charset").equals("cp1251")).isTrue(); - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(((String) message.getPayload())).isNotEqualTo("Test Mail"); - } - ) - .thenCancel() - .verify(); - } -} diff --git a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapIdleFailTests.java b/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapIdleFailTests.java deleted file mode 100644 index 001f041eb..000000000 --- a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapIdleFailTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mail; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { - "mail.supplier.idle-imap=true", - "mail.supplier.url=imap://user:pw@localhost:${test.mail.server.imap.port}/INBOX"}) -public class ImapIdleFailTests extends AbstractMailSupplierTests { - @Test - public void testSimpleTest() { - // given - sendMessage("test", "foo"); - // when - final Flux> messageFlux = mailSupplier.get(); - // then - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(((String) message.getPayload())).isNotEqualTo("Test Mail"); - } - ) - .thenCancel() - .verify(); - } -} diff --git a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapIdlePassTests.java b/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapIdlePassTests.java deleted file mode 100644 index 1ed7aec38..000000000 --- a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapIdlePassTests.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mail; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { - "mail.supplier.idle-imap=true", - "mail.supplier.url=imap://user:pw@localhost:${test.mail.server.imap.port}/INBOX"}) -public class ImapIdlePassTests extends AbstractMailSupplierTests { - - @Test - public void testSimpleTest() { - // given - sendMessage("test", "foo"); - // when - final Flux> messageFlux = mailSupplier.get(); - // then - StepVerifier.create(messageFlux) - .assertNext((message) -> { - System.out.println("Message:" + message); - assertThat(((String) message.getPayload())).isEqualTo("foo"); - } - ) - .thenCancel() - .verify(); - } -} diff --git a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapPassTests.java b/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapPassTests.java deleted file mode 100644 index 97cc2a3f2..000000000 --- a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/ImapPassTests.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mail; - -import java.io.UnsupportedEncodingException; -import java.util.List; - -import com.icegreen.greenmail.util.GreenMailUtil; -import jakarta.mail.MessagingException; -import jakarta.mail.internet.InternetAddress; -import jakarta.mail.internet.MimeMessage; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.integration.mail.MailHeaders; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = "mail.supplier.url=imap://user:pw@localhost:${test.mail.server.imap.port}/INBOX") -public class ImapPassTests extends AbstractMailSupplierTests { - - - @Test - public void testSimpleTest() throws UnsupportedEncodingException, MessagingException { - // given - MimeMessage mailMessage = GreenMailUtil.createTextEmail("bar@foo", "test@test", "test", "foo", mailServer.getSmtp().getServerSetup()); - mailMessage.addRecipients(jakarta.mail.Message.RecipientType.TO, new InternetAddress[]{new InternetAddress("foo@bar", "Foo")}); - mailMessage.addRecipients(jakarta.mail.Message.RecipientType.CC, new InternetAddress[]{new InternetAddress("a@b"), new InternetAddress("c@d")}); - mailMessage.addRecipients(jakarta.mail.Message.RecipientType.BCC, new InternetAddress[]{new InternetAddress("e@f"), new InternetAddress("g@h")}); - mailUser.deliver(mailMessage); - // when - final Flux> messageFlux = mailSupplier.get(); - // then - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(((String) message.getPayload())).isEqualTo("foo"); - MessageHeaders headers = message.getHeaders(); - assertThat(headers.get(MailHeaders.TO)).isInstanceOf(List.class); - assertThat(headers.get(MailHeaders.CC)).isInstanceOf(List.class); - assertThat(headers.get(MailHeaders.BCC)).isInstanceOf(List.class); - assertThat(headers.get(MailHeaders.TO).toString()).isEqualTo("[bar@foo, Foo ]"); - assertThat(headers.get(MailHeaders.CC).toString()).isEqualTo("[a@b, c@d]"); - assertThat(headers.get(MailHeaders.BCC).toString()).isEqualTo("[e@f, g@h]"); - } - ) - .thenCancel() - .verify(); - } -} diff --git a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/Pop3FailTests.java b/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/Pop3FailTests.java deleted file mode 100644 index 52a17c88d..000000000 --- a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/Pop3FailTests.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mail; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = "mail.supplier.url=pop3://user:pw@localhost:${test.mail.server.pop3.port}/INBOX") -public class Pop3FailTests extends AbstractMailSupplierTests { - - - @Test - public void testSimpleTest() { - // given - sendMessage("test", "foo"); - // when - final Flux> messageFlux = mailSupplier.get(); - // then - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(((String) message.getPayload())).isNotEqualTo("Test Mail"); - } - ) - .thenCancel() - .verify(); - } - -} diff --git a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/Pop3PassTests.java b/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/Pop3PassTests.java deleted file mode 100644 index fb1166d77..000000000 --- a/functions/supplier/mail-supplier/src/test/java/org/springframework/cloud/fn/supplier/mail/Pop3PassTests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mail; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = "mail.supplier.url=pop3://user:pw@localhost:${test.mail.server.pop3.port}/INBOX") -public class Pop3PassTests extends AbstractMailSupplierTests { - - - @Test - public void testSimpleTest() { - // given - sendMessage("test", "foo"); - final Flux> messageFlux = mailSupplier.get(); - - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(((String) message.getPayload())).contains("foo"); - } - ) - .thenCancel() - .verify(); - - } -} diff --git a/functions/supplier/mongodb-supplier/README.adoc b/functions/supplier/mongodb-supplier/README.adoc deleted file mode 100644 index 79fc60fcc..000000000 --- a/functions/supplier/mongodb-supplier/README.adoc +++ /dev/null @@ -1,39 +0,0 @@ -# MongoDB Supplier - -This module provides a MongoDB supplier that can be reused and composed in other applications. -The `Supplier` uses the `MongoDBMessageSource` from Spring Integration. -`MongoDBSupplier` is implemented as a `java.util.function.Supplier`. -When you have use-cases such as periodical execution of querying MongoDB, based on some external trigger such as a REST endpoint call for example, then you can use this `Supplier` to query. - -## Beans for injection - -You can import the `MongoDBSupplierConfiguration` in the application and then inject the following bean. - -`mongoDBSupplier` - -You can inject it as `Supplier` when you are not splitting the rows. -If you are splitting the output, you need to inject this as `Supplier>>`. - -You can use `mongoDBSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it. -In the case of splitting you get a `Flux` which you have to subscribe in your applications. - -## Configuration Options - -All configuration properties are prefixed with `mongodb.supplier`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierProperties.java[MongoDBSupplierProperties]. - -The `queryExpression` and `updateExpression` options may use Spring Data MongoDB query DSL from the `org.springframework.data.mongodb.core.query`, such as `Query` and `Update` factories respectively. -The `updateExpression` is optional and ca use an item from query result as a root evaluation object to extract some values to update from just fetched data. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `MongoDbMessageSource` configuration used by the `mongodbSupplier`. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierApplicationTests.java[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/mongodb-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a MongoDB Source. diff --git a/functions/supplier/mongodb-supplier/pom.xml b/functions/supplier/mongodb-supplier/pom.xml deleted file mode 100644 index 28d888d92..000000000 --- a/functions/supplier/mongodb-supplier/pom.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - mongodb-supplier - mongodb-supplier - Mongo DB supplier - - - - org.springframework.integration - spring-integration-mongodb - - - org.mongodb - mongodb-driver-sync - - - org.springframework.cloud - spring-cloud-function-context - ${spring-cloud-function.version} - - - org.springframework.cloud.fn - splitter-function - ${project.version} - - - org.testcontainers - mongodb - ${testcontainers.version} - test - - - org.springframework.cloud.fn - mongodb-common - ${project.version} - test-jar - test - - - - diff --git a/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierConfiguration.java b/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierConfiguration.java deleted file mode 100644 index f63ae1b00..000000000 --- a/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierConfiguration.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2019-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mongo; - -import java.util.List; -import java.util.function.Function; -import java.util.function.Supplier; - -import reactor.core.publisher.Flux; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.cloud.fn.splitter.SplitterFunctionConfiguration; -import org.springframework.cloud.function.context.PollableBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.expression.Expression; -import org.springframework.expression.common.LiteralExpression; -import org.springframework.integration.mongodb.inbound.MongoDbMessageSource; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; - -/** - * A configuration for MongoDB Source applications. Produces - * {@link MongoDbMessageSource} which polls collection with the query after startup - * according to the polling properties. - * - * @author Adam Zwickey - * @author Artem Bilan - * @author David Turanski - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties({ MongodbSupplierProperties.class }) -@Import(SplitterFunctionConfiguration.class) -public class MongodbSupplierConfiguration { - - private final MongodbSupplierProperties properties; - - private final MongoTemplate mongoTemplate; - - public MongodbSupplierConfiguration(MongodbSupplierProperties properties, MongoTemplate mongoTemplate) { - this.properties = properties; - this.mongoTemplate = mongoTemplate; - } - - @Bean(name = "mongodbSupplier") - @PollableBean - @ConditionalOnProperty(prefix = "mongodb", name = "split", matchIfMissing = true) - public Supplier>> splittedSupplier(MongoDbMessageSource mongoDbSource, - Function, List>> splitterFunction) { - - return () -> { - Message received = mongoDbSource.receive(); - if (received != null) { - return Flux.fromIterable(splitterFunction.apply(received)); // multiple Message> - } - else { - return Flux.empty(); - } - }; - } - - @Bean - @ConditionalOnProperty(prefix = "mongodb", name = "split", havingValue = "false") - public Supplier> mongodbSupplier(MongoDbMessageSource mongoDbSource) { - return mongoDbSource::receive; - } - - @Bean - public MongoDbMessageSource mongoDbSource( - @Nullable ComponentCustomizer mongoDbMessageSourceCustomizer) { - - Expression queryExpression = (this.properties.getQueryExpression() != null - ? this.properties.getQueryExpression() - : new LiteralExpression(this.properties.getQuery())); - MongoDbMessageSource mongoDbMessageSource = new MongoDbMessageSource(this.mongoTemplate, queryExpression); - mongoDbMessageSource.setCollectionNameExpression(new LiteralExpression(this.properties.getCollection())); - mongoDbMessageSource.setEntityClass(String.class); - mongoDbMessageSource.setUpdateExpression(this.properties.getUpdateExpression()); - - if (mongoDbMessageSourceCustomizer != null) { - mongoDbMessageSourceCustomizer.customize(mongoDbMessageSource); - } - - return mongoDbMessageSource; - } - -} diff --git a/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierProperties.java b/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierProperties.java deleted file mode 100644 index 67fd31c10..000000000 --- a/functions/supplier/mongodb-supplier/src/main/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierProperties.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2016-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mongo; - -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotEmpty; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.validation.annotation.Validated; - -/** - * @author Adam Zwickey - * @author Artem Bilan - * @author Chris Schaefer - * @author David Turanski - */ -@ConfigurationProperties("mongodb.supplier") -@Validated -public class MongodbSupplierProperties { - - /** - * The MongoDB collection to query. - */ - private String collection; - - /** - * The MongoDB query. - */ - private String query = "{ }"; - - /** - * The SpEL expression in MongoDB query DSL style. - */ - private Expression queryExpression; - - /** - * The SpEL expression in MongoDB update DSL style. - */ - private Expression updateExpression; - - /** - * Whether to split the query result as individual messages. - */ - private boolean split = true; - - @NotEmpty(message = "Query is required") - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public Expression getQueryExpression() { - return queryExpression; - } - - public void setQueryExpression(Expression queryExpression) { - this.queryExpression = queryExpression; - } - - @NotBlank(message = "Collection name is required") - public String getCollection() { - return collection; - } - - public void setCollection(String collection) { - this.collection = collection; - } - - public boolean isSplit() { - return split; - } - - public void setSplit(boolean split) { - this.split = split; - } - - public Expression getUpdateExpression() { - return this.updateExpression; - } - - public void setUpdateExpression(Expression updateExpression) { - this.updateExpression = updateExpression; - } - -} diff --git a/functions/supplier/mongodb-supplier/src/main/resources/application.properties b/functions/supplier/mongodb-supplier/src/main/resources/application.properties deleted file mode 100644 index 8b1378917..000000000 --- a/functions/supplier/mongodb-supplier/src/main/resources/application.properties +++ /dev/null @@ -1 +0,0 @@ - diff --git a/functions/supplier/mongodb-supplier/src/test/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierApplicationTests.java b/functions/supplier/mongodb-supplier/src/test/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierApplicationTests.java deleted file mode 100644 index 14305fb9f..000000000 --- a/functions/supplier/mongodb-supplier/src/test/java/org/springframework/cloud/fn/supplier/mongo/MongodbSupplierApplicationTests.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2019-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mongo; - -import java.util.Map; -import java.util.function.Supplier; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoDatabase; -import org.bson.Document; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.common.mongo.MongoDbTestContainerSupport; -import org.springframework.messaging.Message; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.entry; - -@SpringBootTest(properties = { - "mongodb.supplier.collection=testing", - "mongodb.supplier.query={ name: { $exists: true }}", - "mongodb.supplier.update-expression='{ $unset: { name: 0 } }'" -}) -class MongodbSupplierApplicationTests implements MongoDbTestContainerSupport { - - @DynamicPropertySource - static void mongoDbProperties(DynamicPropertyRegistry registry) { - registry.add("spring.data.mongodb.port", MONGO_CONTAINER::getFirstMappedPort); - registry.add("spring.data.mongodb.database", () -> "test"); - } - - private final ObjectMapper objectMapper = new ObjectMapper(); - - @Autowired - private Supplier>> mongodbSupplier; - - @Autowired - private MongoClient mongo; - - @BeforeEach - public void setUp() { - MongoDatabase database = this.mongo.getDatabase("test"); - database.createCollection("testing"); - MongoCollection collection = database.getCollection("testing"); - collection.insertOne( - new Document("greeting", "hello") - .append("name", "foo")); - collection.insertOne( - new Document("greeting", "hola") - .append("name", "bar")); - } - - @Test - void testMongodbSupplier() { - // given - Flux> messageFlux = this.mongodbSupplier.get(); - // when - StepVerifier.create(messageFlux) - // then - .assertNext((message) -> - assertThat(toMap(message)).contains( - entry("greeting", "hello"), - entry("name", "foo"))) - .assertNext((message) -> - assertThat(toMap(message)).contains( - entry("greeting", "hola"), - entry("name", "bar"))) - .thenCancel() - .verify(); - - assertThat(this.mongodbSupplier.get().collectList().block()).isEmpty(); - } - - private Map toMap(Message message) { - Map map = null; - try { - map = objectMapper.readValue(message.getPayload().toString(), Map.class); - } - catch (Exception e) { - e.printStackTrace(); - } - return map; - } - - @SpringBootApplication - static class MongoDbSupplierTestApplication { - } -} diff --git a/functions/supplier/mqtt-supplier/README.adoc b/functions/supplier/mqtt-supplier/README.adoc deleted file mode 100644 index 0fc7b2b7d..000000000 --- a/functions/supplier/mqtt-supplier/README.adoc +++ /dev/null @@ -1,42 +0,0 @@ -# MQTT Supplier - -This module provides an MQTT supplier that can be reused and composed in other applications. -The `Supplier` uses the `MqttPahoMessageDrivenChannelAdapter` from Spring Integration. -`mqttSupplier` is implemented as a `java.util.function.Supplier`. -This supplier gives you a reactive stream from MQTT sources. The supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and then receive the data. - -## Beans for injection - -You can import the `MqttSupplierConfiguration` in the application and then inject the following bean. - -`mqttSupplier` - -You need to inject this as `Supplier>>`. - -You can use `mqttSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `mqtt.supplier` and `mqtt`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/mqtt/MqttSupplierProperties.java[MqttSupplierProperties]. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `MqttPahoMessageDrivenChannelAdapter` configuration used by the `mqttSupplier`. - -## SSL Configuration - -The MQTT Paho client can accept an SSL configuration via `MqttConnectOptions.setSSLProperties()`. -These properties are exposed on the `MqttProperties.sslProperties` map. -The keys for these SSL properties should be taken from the `org.eclipse.paho.client.mqttv3.internal.security.SSLSocketFactoryFactory` constants, which all start with the `com.ibm.ssl.` prefix. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/mqtt/MqttSupplierTests.java[test suite] for the various ways, this supplier is used. -In addition to this, there is also link:../../common/mqtt-common/src/main/java/org/springframework/cloud/fn/common/mqtt/MqttProperties.java[this set of properties] to consider. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/mqtt-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes an MQTT Source. diff --git a/functions/supplier/mqtt-supplier/pom.xml b/functions/supplier/mqtt-supplier/pom.xml deleted file mode 100644 index 8c2436e04..000000000 --- a/functions/supplier/mqtt-supplier/pom.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - mqtt-supplier - mqtt-supplier - mqtt supplier - - - - org.springframework.cloud.fn - mqtt-common - ${project.version} - - - - diff --git a/functions/supplier/mqtt-supplier/src/main/java/org/springframework/cloud/fn/supplier/mqtt/MqttSupplierConfiguration.java b/functions/supplier/mqtt-supplier/src/main/java/org/springframework/cloud/fn/supplier/mqtt/MqttSupplierConfiguration.java deleted file mode 100644 index 82b1bc52c..000000000 --- a/functions/supplier/mqtt-supplier/src/main/java/org/springframework/cloud/fn/supplier/mqtt/MqttSupplierConfiguration.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2017-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mqtt; - -import java.util.function.Supplier; - -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.cloud.fn.common.mqtt.MqttConfiguration; -import org.springframework.cloud.fn.common.mqtt.MqttProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.mqtt.core.MqttPahoClientFactory; -import org.springframework.integration.mqtt.inbound.MqttPahoMessageDrivenChannelAdapter; -import org.springframework.integration.mqtt.support.DefaultPahoMessageConverter; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; - -/** - * A source module that receives data from Mqtt. - * - * @author Janne Valkealahti - * @author Soby Chacko - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties({MqttProperties.class, MqttSupplierProperties.class}) -@Import(MqttConfiguration.class) -public class MqttSupplierConfiguration { - - @Autowired - private MqttSupplierProperties properties; - - @Autowired - private MqttPahoClientFactory mqttClientFactory; - - @Autowired - private BeanFactory beanFactory; - - @Bean - public Supplier>> mqttSupplier(Publisher> mqttPublisher) { - return () -> Flux.from(mqttPublisher); - } - - @Bean - public MqttPahoMessageDrivenChannelAdapter mqttInbound( - @Nullable ComponentCustomizer mqttMessageProducerCustomizer) { - - MqttPahoMessageDrivenChannelAdapter adapter = - new MqttPahoMessageDrivenChannelAdapter(this.properties.getClientId(), this.mqttClientFactory, - this.properties.getTopics()); - adapter.setQos(this.properties.getQos()); - adapter.setConverter(pahoMessageConverter(this.beanFactory)); - adapter.setAutoStartup(false); - - if (mqttMessageProducerCustomizer != null) { - mqttMessageProducerCustomizer.customize(adapter); - } - - return adapter; - } - - @Bean - public Publisher> mqttPublisher(MqttPahoMessageDrivenChannelAdapter mqttInbound) { - return IntegrationFlow.from(mqttInbound) - .toReactivePublisher(true); - } - - private DefaultPahoMessageConverter pahoMessageConverter(BeanFactory beanFactory) { - DefaultPahoMessageConverter converter = new DefaultPahoMessageConverter(properties.getCharset()); - converter.setPayloadAsBytes(properties.isBinary()); - converter.setBeanFactory(beanFactory); - return converter; - } - -} diff --git a/functions/supplier/mqtt-supplier/src/main/java/org/springframework/cloud/fn/supplier/mqtt/MqttSupplierProperties.java b/functions/supplier/mqtt-supplier/src/main/java/org/springframework/cloud/fn/supplier/mqtt/MqttSupplierProperties.java deleted file mode 100644 index 25f671e66..000000000 --- a/functions/supplier/mqtt-supplier/src/main/java/org/springframework/cloud/fn/supplier/mqtt/MqttSupplierProperties.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2017-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mqtt; - -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.Size; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * Properties for the Mqtt Source. - * - * @author Janne Valkealahti - * @author Soby Chacko - * - */ -@Validated -@ConfigurationProperties("mqtt.supplier") -public class MqttSupplierProperties { - - /** - * identifies the client. - */ - private String clientId = "stream.client.id.source"; - - /** - * the topic(s) (comma-delimited) to which the source will subscribe. - */ - private String[] topics = new String[] { "stream.mqtt" }; - - /** - * the qos; a single value for all topics or a comma-delimited list to match the topics. - */ - private int[] qos = new int[] { 0 }; - - /** - * true to leave the payload as bytes. - */ - private boolean binary = false; - - /** - * the charset used to convert bytes to String (when binary is false). - */ - private String charset = "UTF-8"; - - @NotBlank - @Size(min = 1, max = 23) - public String getClientId() { - return this.clientId; - } - - public void setClientId(String clientId) { - this.clientId = clientId; - } - - public String[] getTopics() { - return this.topics; - } - - public void setTopics(String[] topics) { - this.topics = topics; - } - - public int[] getQos() { - return this.qos; - } - - public void setQos(int[] qos) { - this.qos = qos; - } - - public String getCharset() { - return this.charset; - } - - public void setCharset(String charset) { - this.charset = charset; - } - - public boolean isBinary() { - return this.binary; - } - - public void setBinary(boolean binary) { - this.binary = binary; - } - -} diff --git a/functions/supplier/mqtt-supplier/src/test/java/org/springframework/cloud/fn/supplier/mqtt/MqttSupplierTests.java b/functions/supplier/mqtt-supplier/src/test/java/org/springframework/cloud/fn/supplier/mqtt/MqttSupplierTests.java deleted file mode 100644 index d859b0b62..000000000 --- a/functions/supplier/mqtt-supplier/src/test/java/org/springframework/cloud/fn/supplier/mqtt/MqttSupplierTests.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2017-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.mqtt; - -import java.time.Duration; -import java.util.Properties; -import java.util.function.Supplier; - -import org.eclipse.paho.client.mqttv3.MqttConnectOptions; -import org.eclipse.paho.client.mqttv3.internal.security.SSLSocketFactoryFactory; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.GenericContainer; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.integration.mqtt.core.MqttPahoClientFactory; -import org.springframework.integration.mqtt.outbound.MqttPahoMessageHandler; -import org.springframework.integration.mqtt.support.DefaultPahoMessageConverter; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHandler; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Tests for Mqtt Supplier. - * - * @author Janne Valkealahti - * @author Gary Russell - * @author Soby Chacko - * @author Artem Bilan - * - */ -@SpringBootTest(properties = - { "mqtt.supplier.topics=test,fake", - "mqtt.supplier.qos=0,0", - "mqtt.ssl-properties.com.ibm.ssl.protocol=TLS", - "mqtt.ssl-properties.com.ibm.ssl.keyStoreType=TEST"}) -@DirtiesContext -@Tag("integration") -public class MqttSupplierTests { - - static { - GenericContainer mosquitto = - new GenericContainer<>("eclipse-mosquitto:2.0.13") - .withCommand("mosquitto -c /mosquitto-no-auth.conf") - .withReuse(true) - .withExposedPorts(1883) - .withStartupTimeout(Duration.ofSeconds(120)) - .withStartupAttempts(3); - mosquitto.start(); - final Integer mappedPort = mosquitto.getMappedPort(1883); - System.setProperty("mqtt.url", "tcp://localhost:" + mappedPort); - } - - @AfterAll - public static void cleanup() { - System.clearProperty("mqtt.url"); - } - - @Autowired - private Supplier>> mqttSupplier; - - @Autowired - private MqttPahoMessageHandler mqttOutbound; - - @Test - public void testBasicFlow() { - MqttConnectOptions connectionInfo = this.mqttOutbound.getConnectionInfo(); - Properties sslProperties = connectionInfo.getSSLProperties(); - assertThat(sslProperties) - .containsEntry(SSLSocketFactoryFactory.SSLPROTOCOL, SSLSocketFactoryFactory.DEFAULT_PROTOCOL) - .containsEntry(SSLSocketFactoryFactory.KEYSTORETYPE, "TEST"); - this.mqttOutbound.handleMessage(MessageBuilder.withPayload("hello").build()); - - final Flux> messageFlux = mqttSupplier.get(); - - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(message.getPayload()) - .isEqualTo("hello"); - } - ) - .thenCancel() - .verify(); - } - - @SpringBootApplication - static class MqttSupplierTestApplication { - - @Autowired - private MqttPahoClientFactory mqttClientFactory; - - @Bean - public MessageHandler mqttOutbound() { - MqttPahoMessageHandler messageHandler = new MqttPahoMessageHandler("test", mqttClientFactory); - messageHandler.setAsync(true); - messageHandler.setDefaultTopic("test"); - messageHandler.setConverter(producerConverter()); - return messageHandler; - } - - @Bean - public DefaultPahoMessageConverter producerConverter() { - return new DefaultPahoMessageConverter(1, true, "UTF-8"); - } - - } - -} diff --git a/functions/supplier/mqtt-supplier/src/test/resources/logback.xml b/functions/supplier/mqtt-supplier/src/test/resources/logback.xml deleted file mode 100644 index 8e52ae3ee..000000000 --- a/functions/supplier/mqtt-supplier/src/test/resources/logback.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - \ No newline at end of file diff --git a/functions/supplier/pom.xml b/functions/supplier/pom.xml deleted file mode 100644 index d51b83490..000000000 --- a/functions/supplier/pom.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - 4.0.0 - - org.springframework.cloud.fn - java-functions-supplier - 5.0.0-SNAPSHOT - java-functions-supplier - Java Functions Supplier - pom - - - file-supplier - ftp-supplier - http-supplier - jdbc-supplier - jms-supplier - kafka-supplier - mail-supplier - mongodb-supplier - mqtt-supplier - sftp-supplier - tcp-supplier - time-supplier - rabbit-supplier - websocket-supplier - s3-supplier - twitter-supplier - debezium-supplier - syslog-supplier - xmpp-supplier - zeromq-supplier - - - - - org.apache.maven.plugins - maven-deploy-plugin - 3.0.0 - - true - - - - - diff --git a/functions/supplier/rabbit-supplier/README.adoc b/functions/supplier/rabbit-supplier/README.adoc deleted file mode 100644 index c80150510..000000000 --- a/functions/supplier/rabbit-supplier/README.adoc +++ /dev/null @@ -1,34 +0,0 @@ -# RabbitMQ Supplier - -This module provides an RabbitMQ supplier that can be reused and composed in other applications. -The `Supplier` uses the RabbitMQ support provided by Spring Integration. -The `rabbitSupplier` is implemented as a `java.util.function.Supplier`. -This supplier gives you a reactive stream of files from the provided directory as the supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and receive the data. - -## Beans for injection - -You can import the `RabbitSupplierConfiguration` in the application and then inject the following bean. - -`rabbitSupplier` - -You need to inject this as `Supplier>>`. - -You can use `rabbitSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `rabbit.supplier`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/rabbit/RabbitSupplierProperties.java[RabbitSupplierProperties]. -Also see an auto-configuration for RabbitMQ connection and listener container options. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `AmqpInboundChannelAdapterSMLCSpec` configuration used by the `rabbitSupplier`. - -## Tests - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/rabbit-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a RabbitMQ Source. diff --git a/functions/supplier/rabbit-supplier/pom.xml b/functions/supplier/rabbit-supplier/pom.xml deleted file mode 100644 index 90b14f9be..000000000 --- a/functions/supplier/rabbit-supplier/pom.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - rabbit-supplier - rabbit-supplier - Rabbit supplier - - - - org.springframework.integration - spring-integration-amqp - - - org.springframework.boot - spring-boot-starter-amqp - - - org.springframework.boot - spring-boot-starter-web - - - - diff --git a/functions/supplier/rabbit-supplier/src/main/java/org/springframework/cloud/fn/supplier/rabbit/RabbitSupplierConfiguration.java b/functions/supplier/rabbit-supplier/src/main/java/org/springframework/cloud/fn/supplier/rabbit/RabbitSupplierConfiguration.java deleted file mode 100644 index c058e9af4..000000000 --- a/functions/supplier/rabbit-supplier/src/main/java/org/springframework/cloud/fn/supplier/rabbit/RabbitSupplierConfiguration.java +++ /dev/null @@ -1,234 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.rabbit; - -import java.util.function.Supplier; - -import com.rabbitmq.client.AMQP; -import com.rabbitmq.client.Envelope; -import com.rabbitmq.client.impl.CredentialsProvider; -import com.rabbitmq.client.impl.CredentialsRefreshService; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; - -import org.springframework.amqp.core.AcknowledgeMode; -import org.springframework.amqp.core.MessageProperties; -import org.springframework.amqp.rabbit.config.RetryInterceptorBuilder; -import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; -import org.springframework.amqp.rabbit.connection.ConnectionFactory; -import org.springframework.amqp.rabbit.connection.RabbitConnectionFactoryBean; -import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer; -import org.springframework.amqp.rabbit.retry.RejectAndDontRequeueRecoverer; -import org.springframework.amqp.rabbit.support.DefaultMessagePropertiesConverter; -import org.springframework.amqp.rabbit.support.MessagePropertiesConverter; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.amqp.CachingConnectionFactoryConfigurer; -import org.springframework.boot.autoconfigure.amqp.ConnectionFactoryCustomizer; -import org.springframework.boot.autoconfigure.amqp.RabbitConnectionFactoryBeanConfigurer; -import org.springframework.boot.autoconfigure.amqp.RabbitProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.io.ResourceLoader; -import org.springframework.integration.amqp.dsl.Amqp; -import org.springframework.integration.amqp.dsl.AmqpInboundChannelAdapterSMLCSpec; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.retry.interceptor.RetryOperationsInterceptor; -import org.springframework.util.Assert; - -/** - * A source module that receives data from RabbitMQ. - * - * @author Gary Russell - * @author Chris Schaefer - * @author Roger Perez - * @author Chris Bono - * @author Artem Bilan - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties(RabbitSupplierProperties.class) -public class RabbitSupplierConfiguration implements DisposableBean { - - private static final MessagePropertiesConverter inboundMessagePropertiesConverter = - new DefaultMessagePropertiesConverter() { - - @Override - public MessageProperties toMessageProperties(AMQP.BasicProperties source, - Envelope envelope, String charset) { - - MessageProperties properties = super.toMessageProperties(source, envelope, charset); - properties.setDeliveryMode(null); - return properties; - } - - }; - - @Autowired - private RabbitProperties rabbitProperties; - - @Autowired - private ResourceLoader resourceLoader; - - @Autowired - private ObjectProvider credentialsProvider; - - @Autowired - private ObjectProvider credentialsRefreshService; - - @Autowired - private ObjectProvider connectionFactoryCustomizers; - - @Autowired - private RabbitSupplierProperties properties; - - @Autowired - private ConnectionFactory rabbitConnectionFactory; - - private CachingConnectionFactory ownConnectionFactory; - - @Bean - public SimpleMessageListenerContainer container(RetryOperationsInterceptor rabbitSourceRetryInterceptor) { - ConnectionFactory connectionFactory = this.properties.isOwnConnection() - ? buildLocalConnectionFactory() - : this.rabbitConnectionFactory; - SimpleMessageListenerContainer container = new SimpleMessageListenerContainer(connectionFactory); - container.setAutoStartup(false); - RabbitProperties.SimpleContainer simpleContainer = this.rabbitProperties.getListener().getSimple(); - - AcknowledgeMode acknowledgeMode = simpleContainer.getAcknowledgeMode(); - if (acknowledgeMode != null) { - container.setAcknowledgeMode(acknowledgeMode); - } - Integer concurrency = simpleContainer.getConcurrency(); - if (concurrency != null) { - container.setConcurrentConsumers(concurrency); - } - Integer maxConcurrency = simpleContainer.getMaxConcurrency(); - if (maxConcurrency != null) { - container.setMaxConcurrentConsumers(maxConcurrency); - } - Integer prefetch = simpleContainer.getPrefetch(); - if (prefetch != null) { - container.setPrefetchCount(prefetch); - } - Integer transactionSize = simpleContainer.getBatchSize(); - if (transactionSize != null) { - container.setBatchSize(transactionSize); - } - container.setDefaultRequeueRejected(this.properties.getRequeue()); - container.setChannelTransacted(this.properties.getTransacted()); - String[] queues = this.properties.getQueues(); - Assert.state(queues.length > 0, "At least one queue is required"); - Assert.noNullElements(queues, "queues cannot have null elements"); - container.setQueueNames(queues); - if (this.properties.isEnableRetry()) { - container.setAdviceChain(rabbitSourceRetryInterceptor); - } - container.setMessagePropertiesConverter(inboundMessagePropertiesConverter); - return container; - } - - @Bean - public Publisher> rabbitPublisher(SimpleMessageListenerContainer container, - @Nullable ComponentCustomizer amqpMessageProducerCustomizer) { - - AmqpInboundChannelAdapterSMLCSpec messageProducerSpec = - Amqp.inboundAdapter(container) - .mappedRequestHeaders(properties.getMappedRequestHeaders()); - - if (amqpMessageProducerCustomizer != null) { - amqpMessageProducerCustomizer.customize(messageProducerSpec); - } - - return IntegrationFlow.from(messageProducerSpec) - .toReactivePublisher(true); - } - - @Bean - public Supplier>> rabbitSupplier(Publisher> rabbitPublisher) { - return () -> Flux.from(rabbitPublisher); - } - - @Bean - public RetryOperationsInterceptor rabbitSourceRetryInterceptor() { - return RetryInterceptorBuilder.stateless() - .maxAttempts(this.properties.getMaxAttempts()) - .backOffOptions(this.properties.getInitialRetryInterval(), this.properties.getRetryMultiplier(), - this.properties.getMaxRetryInterval()) - .recoverer(new RejectAndDontRequeueRecoverer()) - .build(); - } - - @Override - public void destroy() { - if (this.ownConnectionFactory != null) { - this.ownConnectionFactory.destroy(); - } - } - - private ConnectionFactory buildLocalConnectionFactory() { - try { - this.ownConnectionFactory = rabbitConnectionFactory( - this.rabbitProperties, this.resourceLoader, this.credentialsProvider, this.credentialsRefreshService, - this.connectionFactoryCustomizers); - - } - catch (Exception exception) { - throw new IllegalStateException("Error building connection factory", exception); - } - - return this.ownConnectionFactory; - } - - /* NOTE: This is based on RabbitAutoConfiguration.RabbitConnectionFactoryCreator - * https://github.com/spring-projects/spring-boot/blob/c820ad01a108d419d8548265b8a34ed7c5591f7c/spring-boot-project/spring-boot-autoconfigure/src/main/java/org/springframework/boot/autoconfigure/amqp/RabbitAutoConfiguration.java#L95 - * [UPGRADE_CONSIDERATION] this should stay somewhat in sync w/ the functionality provided by its original source. - */ - private static CachingConnectionFactory rabbitConnectionFactory(RabbitProperties properties, - ResourceLoader resourceLoader, - ObjectProvider credentialsProvider, - ObjectProvider credentialsRefreshService, - ObjectProvider connectionFactoryCustomizers) throws Exception { - - RabbitConnectionFactoryBean connectionFactoryBean = new RabbitConnectionFactoryBean(); - RabbitConnectionFactoryBeanConfigurer connectionFactoryBeanConfigurer = - new RabbitConnectionFactoryBeanConfigurer(resourceLoader, properties); - connectionFactoryBeanConfigurer.setCredentialsProvider(credentialsProvider.getIfUnique()); - connectionFactoryBeanConfigurer.setCredentialsRefreshService(credentialsRefreshService.getIfUnique()); - connectionFactoryBeanConfigurer.configure(connectionFactoryBean); - connectionFactoryBean.afterPropertiesSet(); - - com.rabbitmq.client.ConnectionFactory connectionFactory = connectionFactoryBean.getObject(); - connectionFactoryCustomizers.orderedStream() - .forEach((customizer) -> customizer.customize(connectionFactory)); - - CachingConnectionFactory cachingConnectionFactory = new CachingConnectionFactory(connectionFactory); - CachingConnectionFactoryConfigurer cachingConnectionFactoryConfigurer = - new CachingConnectionFactoryConfigurer(properties); - cachingConnectionFactoryConfigurer.setConnectionNameStrategy(cf -> "rabbit.supplier.own.connection"); - cachingConnectionFactoryConfigurer.configure(cachingConnectionFactory); - cachingConnectionFactory.afterPropertiesSet(); - - return cachingConnectionFactory; - } - -} diff --git a/functions/supplier/rabbit-supplier/src/main/java/org/springframework/cloud/fn/supplier/rabbit/RabbitSupplierProperties.java b/functions/supplier/rabbit-supplier/src/main/java/org/springframework/cloud/fn/supplier/rabbit/RabbitSupplierProperties.java deleted file mode 100644 index 0cb390ca8..000000000 --- a/functions/supplier/rabbit-supplier/src/main/java/org/springframework/cloud/fn/supplier/rabbit/RabbitSupplierProperties.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright 2019-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.rabbit; - -import jakarta.validation.constraints.NotNull; -import jakarta.validation.constraints.Size; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -@ConfigurationProperties("rabbit.supplier") -@Validated -public class RabbitSupplierProperties { - - /** - * Whether rejected messages should be requeued. - */ - private boolean requeue = true; - - /** - * Whether the channel is transacted. - */ - private boolean transacted = false; - - /** - * The queues to which the source will listen for messages. - */ - private String[] queues; - - /** - * Headers that will be mapped. - */ - private String[] mappedRequestHeaders = {"STANDARD_REQUEST_HEADERS"}; - - /** - * Initial retry interval when retry is enabled. - */ - private int initialRetryInterval = 1000; - - /** - * Max retry interval when retry is enabled. - */ - private int maxRetryInterval = 30000; - - /** - * Retry backoff multiplier when retry is enabled. - */ - private double retryMultiplier = 2.0; - - /** - * The maximum delivery attempts when retry is enabled. - */ - private int maxAttempts = 3; - - /** - * true to enable retry. - */ - private boolean enableRetry = false; - - /** - * When true, use a separate connection based on the boot properties. - */ - private boolean ownConnection; - - public boolean getRequeue() { - return requeue; - } - - public void setRequeue(boolean requeue) { - this.requeue = requeue; - } - - public boolean getTransacted() { - return transacted; - } - - public void setTransacted(boolean transacted) { - this.transacted = transacted; - } - - @NotNull(message = "queue(s) are required") - @Size(min = 1, message = "At least one queue is required") - public String[] getQueues() { - return queues; - } - - public void setQueues(String[] queues) { - this.queues = queues; - } - - @NotNull - public String[] getMappedRequestHeaders() { - return mappedRequestHeaders; - } - - public void setMappedRequestHeaders(String[] mappedRequestHeaders) { - this.mappedRequestHeaders = mappedRequestHeaders; - } - - public int getInitialRetryInterval() { - return initialRetryInterval; - } - - public void setInitialRetryInterval(int initialRetryInterval) { - this.initialRetryInterval = initialRetryInterval; - } - - public int getMaxRetryInterval() { - return maxRetryInterval; - } - - public void setMaxRetryInterval(int maxRetryInterval) { - this.maxRetryInterval = maxRetryInterval; - } - - public double getRetryMultiplier() { - return retryMultiplier; - } - - public void setRetryMultiplier(double retryMultiplier) { - this.retryMultiplier = retryMultiplier; - } - - public int getMaxAttempts() { - return maxAttempts; - } - - public void setMaxAttempts(int maxAttempts) { - this.maxAttempts = maxAttempts; - } - - public boolean isEnableRetry() { - return enableRetry; - } - - public void setEnableRetry(boolean enableRetry) { - this.enableRetry = enableRetry; - } - - public boolean isOwnConnection() { - return this.ownConnection; - } - - public void setOwnConnection(boolean ownConnection) { - this.ownConnection = ownConnection; - } -} diff --git a/functions/supplier/s3-supplier/README.adoc b/functions/supplier/s3-supplier/README.adoc deleted file mode 100644 index 760a086e8..000000000 --- a/functions/supplier/s3-supplier/README.adoc +++ /dev/null @@ -1,38 +0,0 @@ -= AWS S3 Supplier - -This module provides an S3 supplier that can be reused and composed in other applications. -The `Supplier` uses the AWS S3 support provided by Spring Integration and spring cloud aws project -`S3Supplier` is implemented as a `java.util.function.Supplier`. -This supplier gives you a reactive stream of files from the provided directory as the supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and receive the data. - -== Beans for injection - -You can import the `AwsS3SupplierConfiguration` in the application and then inject the following bean. - -`s3Supplier` - -You need to inject this as `Supplier>>`. - -You can use `s3Supplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -== Configuration Options - -All configuration properties are prefixed with `s3.supplier`. -There are also properties that need to be used with the prefix `s3.common` and `file.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/s3/AwsS3SupplierProperties.java[AwsS3SupplierProperties], -link:../../common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileConsumerProperties.java[FileConsumerProperties], and -`io.awspring.cloud.autoconfigure.s3.properties.S3Properties` from Spring Cloud AWS auto-configuration.. - -A `ComponentCustomizer` bean can be added in the target project to provide any custom options for the `S3InboundFileSynchronizingMessageSource` configuration used by the `s3Supplier`. - -== Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/s3[test suite] for the various ways, this supplier is used. - -== Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/s3-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes an AWS S3 Source. diff --git a/functions/supplier/s3-supplier/pom.xml b/functions/supplier/s3-supplier/pom.xml deleted file mode 100644 index 53af2be6d..000000000 --- a/functions/supplier/s3-supplier/pom.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - s3-supplier - s3-supplier - s3 supplier - - - - org.springframework.cloud.fn - aws-s3-common - ${project.version} - - - org.springframework.cloud.fn - file-common - ${project.version} - - - org.springframework.cloud.fn - metadata-store-common - ${project.version} - - - - diff --git a/functions/supplier/s3-supplier/src/main/java/org/springframework/cloud/fn/supplier/s3/AwsS3SupplierConfiguration.java b/functions/supplier/s3-supplier/src/main/java/org/springframework/cloud/fn/supplier/s3/AwsS3SupplierConfiguration.java deleted file mode 100644 index be44f4fd4..000000000 --- a/functions/supplier/s3-supplier/src/main/java/org/springframework/cloud/fn/supplier/s3/AwsS3SupplierConfiguration.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.s3; - -import java.util.List; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.S3Object; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.config.ComponentCustomizer; -import org.springframework.cloud.fn.common.file.FileConsumerProperties; -import org.springframework.cloud.fn.common.file.FileUtils; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.aws.inbound.S3InboundFileSynchronizer; -import org.springframework.integration.aws.inbound.S3InboundFileSynchronizingMessageSource; -import org.springframework.integration.aws.support.S3SessionFactory; -import org.springframework.integration.aws.support.filters.S3PersistentAcceptOnceFileListFilter; -import org.springframework.integration.aws.support.filters.S3RegexPatternFileListFilter; -import org.springframework.integration.aws.support.filters.S3SimplePatternFileListFilter; -import org.springframework.integration.core.MessageSource; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.endpoint.ReactiveMessageSourceProducer; -import org.springframework.integration.file.filters.ChainFileListFilter; -import org.springframework.integration.metadata.ConcurrentMetadataStore; -import org.springframework.integration.util.IntegrationReactiveUtils; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; -import org.springframework.util.StringUtils; - -/** - * @author Artem Bilan - * @author David Turanski - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties({AwsS3SupplierProperties.class, FileConsumerProperties.class}) -public class AwsS3SupplierConfiguration { - - protected static final String METADATA_STORE_PREFIX = "s3-metadata-"; - - protected final AwsS3SupplierProperties awsS3SupplierProperties; - - protected final FileConsumerProperties fileConsumerProperties; - - protected final S3SessionFactory s3SessionFactory; - - protected final ConcurrentMetadataStore metadataStore; - - public AwsS3SupplierConfiguration(AwsS3SupplierProperties awsS3SupplierProperties, - FileConsumerProperties fileConsumerProperties, - S3SessionFactory s3SessionFactory, - ConcurrentMetadataStore metadataStore) { - - this.awsS3SupplierProperties = awsS3SupplierProperties; - this.fileConsumerProperties = fileConsumerProperties; - this.s3SessionFactory = s3SessionFactory; - this.metadataStore = metadataStore; - } - - @Configuration - @ConditionalOnProperty(prefix = "s3.supplier", name = "list-only", havingValue = "false", matchIfMissing = true) - static class SynchronizingConfiguration extends AwsS3SupplierConfiguration { - - @Bean - public Supplier>> s3Supplier(Publisher> s3SupplierFlow) { - return () -> Flux.from(s3SupplierFlow); - } - - @Bean - public ChainFileListFilter filter(ConcurrentMetadataStore metadataStore) { - ChainFileListFilter chainFilter = new ChainFileListFilter<>(); - if (StringUtils.hasText(this.awsS3SupplierProperties.getFilenamePattern())) { - chainFilter.addFilter( - new S3SimplePatternFileListFilter(this.awsS3SupplierProperties.getFilenamePattern())); - } - else if (this.awsS3SupplierProperties.getFilenameRegex() != null) { - chainFilter - .addFilter(new S3RegexPatternFileListFilter(this.awsS3SupplierProperties.getFilenameRegex())); - } - - chainFilter.addFilter(new S3PersistentAcceptOnceFileListFilter(metadataStore, METADATA_STORE_PREFIX)); - return chainFilter; - } - - SynchronizingConfiguration(AwsS3SupplierProperties awsS3SupplierProperties, - FileConsumerProperties fileConsumerProperties, - S3SessionFactory s3SessionFactory, - ConcurrentMetadataStore concurrentMetadataStore) { - - super(awsS3SupplierProperties, fileConsumerProperties, s3SessionFactory, concurrentMetadataStore); - } - - @Bean - public Publisher> s3SupplierFlow(S3InboundFileSynchronizingMessageSource s3MessageSource) { - return FileUtils.enhanceFlowForReadingMode( - IntegrationFlow.from( - IntegrationReactiveUtils.messageSourceToFlux(s3MessageSource) - .doOnSubscribe((s) -> s3MessageSource.start())), - fileConsumerProperties) - .toReactivePublisher(true); - } - - @Bean - public S3InboundFileSynchronizer s3InboundFileSynchronizer(ChainFileListFilter filter) { - - S3InboundFileSynchronizer synchronizer = new S3InboundFileSynchronizer(s3SessionFactory); - synchronizer.setDeleteRemoteFiles(this.awsS3SupplierProperties.isDeleteRemoteFiles()); - synchronizer.setPreserveTimestamp(this.awsS3SupplierProperties.isPreserveTimestamp()); - String remoteDir = this.awsS3SupplierProperties.getRemoteDir(); - synchronizer.setRemoteDirectory(remoteDir); - synchronizer.setRemoteFileSeparator(this.awsS3SupplierProperties.getRemoteFileSeparator()); - synchronizer.setTemporaryFileSuffix(this.awsS3SupplierProperties.getTmpFileSuffix()); - synchronizer.setFilter(filter); - - return synchronizer; - } - - @Bean - public S3InboundFileSynchronizingMessageSource s3MessageSource( - S3InboundFileSynchronizer s3InboundFileSynchronizer, - @Nullable ComponentCustomizer s3MessageSourceCustomizer) { - - S3InboundFileSynchronizingMessageSource s3MessageSource = new S3InboundFileSynchronizingMessageSource( - s3InboundFileSynchronizer); - s3MessageSource.setLocalDirectory(this.awsS3SupplierProperties.getLocalDir()); - s3MessageSource.setAutoCreateLocalDirectory(this.awsS3SupplierProperties.isAutoCreateLocalDir()); - s3MessageSource.setUseWatchService(true); - - if (s3MessageSourceCustomizer != null) { - s3MessageSourceCustomizer.customize(s3MessageSource); - } - return s3MessageSource; - } - - } - - @Configuration - @ConditionalOnProperty(prefix = "s3.supplier", name = "list-only", havingValue = "true") - static class ListOnlyConfiguration extends AwsS3SupplierConfiguration { - - ListOnlyConfiguration(AwsS3SupplierProperties awsS3SupplierProperties, - FileConsumerProperties fileConsumerProperties, - S3SessionFactory s3SessionFactory, - ConcurrentMetadataStore metadataStore) { - - super(awsS3SupplierProperties, fileConsumerProperties, s3SessionFactory, metadataStore); - } - - @Bean - public Supplier>> s3Supplier(Publisher> s3SupplierFlow) { - return () -> Flux.from(s3SupplierFlow); - } - - @Bean - public Publisher> s3SupplierFlow(ReactiveMessageSourceProducer s3ListingProducer) { - return IntegrationFlow.from(s3ListingProducer).split().toReactivePublisher(true); - } - - @Bean - Predicate listOnlyFilter(AwsS3SupplierProperties awsS3SupplierProperties) { - Predicate predicate = s -> true; - if (StringUtils.hasText(this.awsS3SupplierProperties.getFilenamePattern())) { - Pattern pattern = Pattern.compile(this.awsS3SupplierProperties.getFilenamePattern()); - predicate = (S3Object summary) -> pattern.matcher(summary.key()).matches(); - } - else if (this.awsS3SupplierProperties.getFilenameRegex() != null) { - predicate = (S3Object summary) -> this.awsS3SupplierProperties.getFilenameRegex() - .matcher(summary.key()).matches(); - } - predicate = predicate.and((S3Object summary) -> { - final String key = METADATA_STORE_PREFIX + awsS3SupplierProperties.getRemoteDir() + "-" + summary.key(); - final String lastModified = String.valueOf(summary.lastModified().toEpochMilli()); - final String storedLastModified = this.metadataStore.get(key); - boolean result = !lastModified.equals(storedLastModified); - if (result) { - metadataStore.put(key, lastModified); - } - return result; - }); - - return predicate; - } - - @Bean - ReactiveMessageSourceProducer s3ListingMessageProducer( - S3Client amazonS3, - ObjectMapper objectMapper, - AwsS3SupplierProperties awsS3SupplierProperties, - Predicate filter - ) { - return new ReactiveMessageSourceProducer( - (MessageSource>) () -> { - List summaryList = - amazonS3.listObjects(ListObjectsRequest.builder() - .bucket(awsS3SupplierProperties.getRemoteDir()) - .build()) - .contents() - .stream() - .filter(filter) - .map(s3Object -> { - try { - return objectMapper.writeValueAsString(s3Object.toBuilder()); - } - catch (JsonProcessingException ex) { - throw new RuntimeException(ex); - } - }) - .collect(Collectors.toList()); - return summaryList.isEmpty() ? null : new GenericMessage<>(summaryList); - }); - } - } -} diff --git a/functions/supplier/s3-supplier/src/main/java/org/springframework/cloud/fn/supplier/s3/AwsS3SupplierProperties.java b/functions/supplier/s3-supplier/src/main/java/org/springframework/cloud/fn/supplier/s3/AwsS3SupplierProperties.java deleted file mode 100644 index 86d3c0989..000000000 --- a/functions/supplier/s3-supplier/src/main/java/org/springframework/cloud/fn/supplier/s3/AwsS3SupplierProperties.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.s3; - -import java.io.File; -import java.util.regex.Pattern; - -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import org.hibernate.validator.constraints.Length; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * @author Artem Bilan - */ -@ConfigurationProperties("s3.supplier") -@Validated -public class AwsS3SupplierProperties { - - /** - * AWS S3 bucket resource. - */ - private String remoteDir = "bucket"; - - /** - * Temporary file suffix. - */ - private String tmpFileSuffix = ".tmp"; - - /** - * Remote File separator. - */ - private String remoteFileSeparator = "/"; - - /** - * Delete or not remote files after processing. - */ - private boolean deleteRemoteFiles = false; - - /** - * The local directory to store files. - */ - private File localDir = new File(System.getProperty("java.io.tmpdir"), "s3-supplier"); - - /** - * Create or not the local directory. - */ - private boolean autoCreateLocalDir = true; - - /** - * The pattern to filter remote files. - */ - private String filenamePattern; - - /** - * The regexp to filter remote files. - */ - private Pattern filenameRegex; - - /** - * To transfer or not the timestamp of the remote file to the local one. - */ - private boolean preserveTimestamp = true; - - /** - * Set to true to return s3 object metadata without copying file to a local directory. - */ - private boolean listOnly = false; - - @Length(min = 3) - public String getRemoteDir() { - return this.remoteDir; - } - - public final void setRemoteDir(String remoteDir) { - this.remoteDir = remoteDir; - } - - @NotBlank - public String getTmpFileSuffix() { - return tmpFileSuffix; - } - - public void setTmpFileSuffix(String tmpFileSuffix) { - this.tmpFileSuffix = tmpFileSuffix; - } - - @NotBlank - public String getRemoteFileSeparator() { - return remoteFileSeparator; - } - - public void setRemoteFileSeparator(String remoteFileSeparator) { - this.remoteFileSeparator = remoteFileSeparator; - } - - public boolean isAutoCreateLocalDir() { - return autoCreateLocalDir; - } - - public void setAutoCreateLocalDir(boolean autoCreateLocalDir) { - this.autoCreateLocalDir = autoCreateLocalDir; - } - - public boolean isDeleteRemoteFiles() { - return deleteRemoteFiles; - } - - public void setDeleteRemoteFiles(boolean deleteRemoteFiles) { - this.deleteRemoteFiles = deleteRemoteFiles; - } - - @NotNull - public File getLocalDir() { - return localDir; - } - - public final void setLocalDir(File localDir) { - this.localDir = localDir; - } - - public String getFilenamePattern() { - return filenamePattern; - } - - public void setFilenamePattern(String filenamePattern) { - this.filenamePattern = filenamePattern; - } - - public Pattern getFilenameRegex() { - return filenameRegex; - } - - public void setFilenameRegex(Pattern filenameRegex) { - this.filenameRegex = filenameRegex; - } - - public boolean isPreserveTimestamp() { - return preserveTimestamp; - } - - public void setPreserveTimestamp(boolean preserveTimestamp) { - this.preserveTimestamp = preserveTimestamp; - } - - @AssertTrue(message = "filenamePattern and filenameRegex are mutually exclusive") - public boolean isExclusivePatterns() { - return !(this.filenamePattern != null && this.filenameRegex != null); - } - - public boolean isListOnly() { - return listOnly; - } - - public void setListOnly(boolean listOnly) { - this.listOnly = listOnly; - } -} diff --git a/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AbstractAwsS3SupplierMockTests.java b/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AbstractAwsS3SupplierMockTests.java deleted file mode 100644 index 23b2ba4c2..000000000 --- a/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AbstractAwsS3SupplierMockTests.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.s3; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Path; -import java.time.Instant; -import java.time.Period; -import java.util.HashMap; -import java.util.Map; -import java.util.function.Supplier; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.io.TempDir; -import reactor.core.publisher.Flux; -import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsResponse; -import software.amazon.awssdk.services.s3.model.S3Object; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Primary; -import org.springframework.integration.dsl.StandardIntegrationFlow; -import org.springframework.integration.test.context.SpringIntegrationTest; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.util.FileCopyUtils; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.BDDMockito.willAnswer; -import static org.mockito.Mockito.mock; - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { - "spring.cloud.aws.credentials.accessKey=" + AbstractAwsS3SupplierMockTests.AWS_ACCESS_KEY, - "spring.cloud.aws.credentials.secretKey=" + AbstractAwsS3SupplierMockTests.AWS_SECRET_KEY, - "spring.cloud.aws.region.static=" + AbstractAwsS3SupplierMockTests.AWS_REGION, - "spring.cloud.aws.s3.endpoint=s3://foo", - "s3.supplier.remoteDir=" + AbstractAwsS3SupplierMockTests.S3_BUCKET - }) -@DirtiesContext -@SpringIntegrationTest(noAutoStartup = "*") -public abstract class AbstractAwsS3SupplierMockTests { - - @TempDir - protected static Path temporaryRemoteFolder; - - protected static final String AWS_ACCESS_KEY = "test.accessKey"; - - protected static final String AWS_SECRET_KEY = "test.secretKey"; - - protected static final String AWS_REGION = "us-gov-west-1"; - - protected static final String S3_BUCKET = "S3_BUCKET"; - - protected static Map S3_OBJECTS; - - @Autowired - Supplier>> s3Supplier; - - @Autowired - protected AwsS3SupplierProperties awsS3SupplierProperties; - - @Autowired - StandardIntegrationFlow standardIntegrationFlow; - - @BeforeAll - public static void setup() throws IOException { - final String remote = temporaryRemoteFolder.toAbsolutePath() + "/remote"; - File f = new File(remote); - f.mkdirs(); - File aFile = new File(f, "1.test"); - FileCopyUtils.copy("Hello".getBytes(), aFile); - File bFile = new File(f, "2.test"); - FileCopyUtils.copy("Bye".getBytes(), bFile); - File otherFile = new File(f, "otherFile"); - FileCopyUtils.copy("Other\nOther2".getBytes(), otherFile); - - S3_OBJECTS = new HashMap<>(); - - Instant instant = Instant.now().plus(Period.ofDays(1)); - - for (File file : f.listFiles()) { - S3Object s3Object = - S3Object.builder() - .key("subdir/" + file.getName()) - .lastModified(instant) - .build(); - - S3_OBJECTS.put(s3Object, new FileInputStream(file)); - } - - final String local = temporaryRemoteFolder.toAbsolutePath() + "/local"; - File f1 = new File(local); - f1.mkdirs(); - System.setProperty("s3.supplier.localDir", f1.getAbsolutePath()); - } - - @AfterAll - public static void tearDown() { - System.clearProperty("s3.supplier.localDir"); - S3_OBJECTS.values() - .forEach(stream -> { - try { - stream.close(); - } - catch (IOException e) { - // Ignore - } - }); - } - - @SpringBootApplication - public static class S3SupplierTestApplication { - - @Bean - @Primary - public S3Client amazonS3Mock() { - S3Client amazonS3 = mock(S3Client.class); - - ListObjectsResponse listObjectsResponse = - ListObjectsResponse.builder().contents(S3_OBJECTS.keySet()).isTruncated(false).build(); - - willAnswer(invocation -> listObjectsResponse) - .given(amazonS3) - .listObjects(any(ListObjectsRequest.class)); - - for (Map.Entry s3Object : S3_OBJECTS.entrySet()) { - willAnswer(invocation -> - new ResponseInputStream<>(GetObjectResponse.builder().build(), s3Object.getValue())) - .given(amazonS3) - .getObject(GetObjectRequest.builder() - .bucket(S3_BUCKET) - .key(s3Object.getKey().key()) - .build()); - } - return amazonS3; - } - - } - -} diff --git a/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AmazonS3FilesTransferredTests.java b/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AmazonS3FilesTransferredTests.java deleted file mode 100644 index f5e4b1972..000000000 --- a/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AmazonS3FilesTransferredTests.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.s3; - -import java.io.File; -import java.time.Duration; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = {"file.consumer.mode=ref", - "s3.supplier.filenameRegex=.*\\\\.test$"}) -public class AmazonS3FilesTransferredTests extends AbstractAwsS3SupplierMockTests { - - @Test - public void test() { - final Flux> messageFlux = s3Supplier.get(); - StepVerifier stepVerifier = - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(new File(message.getPayload().toString().replaceAll("\"", ""))) - .isEqualTo(new File(this.awsS3SupplierProperties.getLocalDir() + File.separator + "subdir" + File.separator + "1.test")); - } - ) - .assertNext((message) -> { - assertThat(new File(message.getPayload().toString().replaceAll("\"", ""))) - .isEqualTo(new File(this.awsS3SupplierProperties.getLocalDir() + File.separator + "subdir" + File.separator + "2.test")); - }) - .thenCancel() - .verifyLater(); - standardIntegrationFlow.start(); - stepVerifier.verify(Duration.ofSeconds(10)); - } - -} diff --git a/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AmazonS3LinesTransferredTests.java b/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AmazonS3LinesTransferredTests.java deleted file mode 100644 index 8f2713d35..000000000 --- a/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AmazonS3LinesTransferredTests.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2016-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.s3; - -import java.io.File; -import java.time.Duration; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.integration.file.FileHeaders; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { - "file.consumer.mode=lines", - "s3.supplier.filenamePattern=*/otherFile", - "file.consumer.with-markers=false"}) -public class AmazonS3LinesTransferredTests extends AbstractAwsS3SupplierMockTests { - - @Test - public void test() { - final Flux> messageFlux = s3Supplier.get(); - StepVerifier stepVerifier = - StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(message.getPayload()).isEqualTo("Other"); - assertThat(message.getHeaders()).containsKey(FileHeaders.ORIGINAL_FILE); - assertThat(message.getHeaders()).containsValue( - new File(this.awsS3SupplierProperties.getLocalDir(), "subdir/otherFile")); - } - ) - .assertNext((message) -> { - assertThat(message.getPayload().toString()).isEqualTo("Other2"); - }) - .thenCancel() - .verifyLater(); - standardIntegrationFlow.start(); - stepVerifier.verify(Duration.ofSeconds(10)); - - assertThat(this.awsS3SupplierProperties.getLocalDir().list()).hasSize(1); - } -} diff --git a/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AmazonS3ListOnlyTests.java b/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AmazonS3ListOnlyTests.java deleted file mode 100644 index bb8d324ca..000000000 --- a/functions/supplier/s3-supplier/src/test/java/org/springframework/cloud/fn/supplier/s3/AmazonS3ListOnlyTests.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright 2020-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.s3; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.time.Duration; -import java.util.HashSet; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.integration.json.JsonPathUtils; -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { - "s3.supplier.list-only=true" -}) -public class AmazonS3ListOnlyTests extends AbstractAwsS3SupplierMockTests { - - @Test - public void test() { - final Flux> messageFlux = s3Supplier.get(); - final HashSet keys = new HashSet<>(); - keys.add("subdir/1.test"); - keys.add("subdir/2.test"); - keys.add("subdir/otherFile"); - StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext(message -> { - String s3Object = (String) message.getPayload(); - String key = jsonPathKey(s3Object); - assertThat(keys).contains(key); - keys.remove(key); - }) - .assertNext(message -> { - String s3Object = (String) message.getPayload(); - String key = jsonPathKey(s3Object); - assertThat(keys).contains(key); - keys.remove(key); - }) - .assertNext(message -> { - String s3Object = (String) message.getPayload(); - String key = jsonPathKey(s3Object); - assertThat(keys).contains(key); - keys.remove(key); - }) - .thenCancel() - .verifyLater(); - standardIntegrationFlow.start(); - stepVerifier.verify(Duration.ofSeconds(10)); - } - - private static String jsonPathKey(String s3Object) { - try { - return JsonPathUtils.evaluate(s3Object, "$.key"); - } - catch (IOException ex) { - throw new UncheckedIOException(ex); - } - } - -} diff --git a/functions/supplier/sftp-supplier/README.adoc b/functions/supplier/sftp-supplier/README.adoc deleted file mode 100644 index b1748b1b7..000000000 --- a/functions/supplier/sftp-supplier/README.adoc +++ /dev/null @@ -1,87 +0,0 @@ -# SFTP Supplier - -This module provides a SFTP supplier that can be reused and composed in other applications. -The `Supplier` uses various `Sftp` inbound adapters from Spring Integration to support a range of modes to consume data from an SFTP server. -These include: - -* Synchronize remote files to a local directory and supplying the File contents, the File reference, or a message per line. -* Stream remote file contents as a byte array directly without copying to a local directory. -* List the remote file names only. - -Messages emitted by the supplier are provided as a byte array by default. However, this can be customized using the `mode` option: - -- *ref* Provides a `java.io.File` reference -- *lines* Will split files line-by-line and emit a new message for each line -- *contents* The default. Provides the contents of a file as a byte array - -NOTE: The `ref` mode (`file.consumer.mode=ref`) is not compatible when streaming results (`sftp.stream=true`). - -NOTE: When using the `lines` mode, you can provide an additional `withMarker` option (via the `file.consumer.with-marker` property). -If set to `true`, the underlying `FileSplitter` will emit additional _start-of-file_ and _end-of-file_ marker messages before and after the actual data. -The payload of these 2 additional marker messages is of type `FileSplitter.FileMarker`. -When not explicitly set, the option defaults to `false`. - -When configuring the `sftp.factory.known-hosts-expression` option, the root object of the evaluation is the application context, an example might be `sftp.factory.known-hosts-expression = @systemProperties['user.home'] + '/.ssh/known_hosts'`. - -## Idempotency - -By default the supplier uses a https://docs.spring.io/spring-integration/api/org/springframework/integration/metadata/SimpleMetadataStore.html[SimpleMetadataStore], storing the last modified time to track files that have already been processed in memory. -If an application using this supplier is restarted, any existing files will be reprocessed. You can inject on of the persistent https://docs.spring.io/spring-integration/reference/html/meta-data-store.html[MetadataStore implementations] provided by Spring Integration, or your own of course, to maintain this state permanently. -See also link:../../common/metadata-store-common/README.adoc[`MetadataStore`] options for possible shared persistent store configuration for the `SftpPersistentAcceptOnceFileListFilter` used in the SFTP Source. - - -## Multiple SFTP Servers -This source supports consuming from multiple SFTP servers. -This requires configuring an SFTP Session Factory for each server. -The labels `one` and `two` shown below can be replaced by any names you want. -The following configuration will rotate between two SFTP servers (this can also be used for multiple directories on the same server), consuming files in a round-robin fashion: - -``` -sftp.supplier.factories.one.host=host1 -sftp.supplier.factories.one.port=1234, -sftp.supplier.factories.one.username = user1, -sftp.supplier.factories.one.password = pass1, -... -sftp.supplier.factories.two.host=host2, -sftp.supplier.factories.two.port=2345, -sftp.supplier.factories.two.username = user2, -sftp.supplier.factories.two.password = pass2, -sftp.supplier.directories=one.sftpSource,two.sftpSecondSource, -sftp.supplier.max-fetch=1, -sftp.supplier.fair=true -``` ---- - - - -`SFtpSupplier` is implemented as a `java.util.function.Supplier`. -This supplier gives you a reactive stream of objects from the provided directory(ies) as the supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and receive the data. - -## Beans for injection - -You can import the `SftpSupplierConfiguration` in the application and then inject the following bean. - -`sftpSupplier` - -You need to inject this as `Supplier>>`. - -You can use `sftpSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `sftp.supplier`. -There are also properties that need to be used with the prefix `file.consumer`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierProperties.java[SftpSupplierProperties]. -Also see link:src/main/java/org/springframework/cloud/fn/supplier/file/FileConsumerProperties.java[FileConsumerProperties]. - -## Examples - -See this link:src/test/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierApplicationTests.java[test suite] for the various ways, this supplier is used. - -## Other usage - -See this link:../../../applications/source/sftp-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application that provides an Sftp Source. diff --git a/functions/supplier/sftp-supplier/pom.xml b/functions/supplier/sftp-supplier/pom.xml deleted file mode 100644 index 8bfe5b7d6..000000000 --- a/functions/supplier/sftp-supplier/pom.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - sftp-supplier - sftp-supplier - sftp supplier - - - - org.springframework.boot - spring-boot-starter-logging - - - org.springframework.integration - spring-integration-sftp - - - org.springframework.cloud.fn - file-common - ${project.version} - - - org.springframework.cloud.fn - ftp-common - ${project.version} - - - org.springframework.cloud.fn - metadata-store-common - ${project.version} - - - org.springframework.cloud.fn - function-test-support - ${project.version} - test - - - - diff --git a/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierConfiguration.java b/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierConfiguration.java deleted file mode 100644 index 9ff6450a4..000000000 --- a/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierConfiguration.java +++ /dev/null @@ -1,458 +0,0 @@ -/* - * Copyright 2018-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.sftp; - -import java.io.IOException; -import java.util.List; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -import org.apache.sshd.sftp.client.SftpClient; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; -import reactor.core.publisher.MonoProcessor; -import reactor.util.context.Context; - -import org.springframework.aop.framework.ProxyFactoryBean; -import org.springframework.aop.support.NameMatchMethodPointcutAdvisor; -import org.springframework.beans.factory.BeanFactory; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.file.FileConsumerProperties; -import org.springframework.cloud.fn.common.file.FileUtils; -import org.springframework.cloud.fn.common.file.remote.RemoteFileDeletingAdvice; -import org.springframework.cloud.fn.common.file.remote.RemoteFileRenamingAdvice; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.http.MediaType; -import org.springframework.integration.aop.ReceiveMessageAdvice; -import org.springframework.integration.channel.QueueChannel; -import org.springframework.integration.core.GenericSelector; -import org.springframework.integration.core.MessageSource; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.dsl.IntegrationFlowBuilder; -import org.springframework.integration.endpoint.MessageProducerSupport; -import org.springframework.integration.file.FileHeaders; -import org.springframework.integration.file.filters.ChainFileListFilter; -import org.springframework.integration.file.filters.FileListFilter; -import org.springframework.integration.file.remote.gateway.AbstractRemoteFileOutboundGateway; -import org.springframework.integration.file.remote.session.SessionFactory; -import org.springframework.integration.handler.MessageProcessor; -import org.springframework.integration.metadata.ConcurrentMetadataStore; -import org.springframework.integration.sftp.dsl.Sftp; -import org.springframework.integration.sftp.dsl.SftpInboundChannelAdapterSpec; -import org.springframework.integration.sftp.dsl.SftpOutboundGatewaySpec; -import org.springframework.integration.sftp.filters.SftpPersistentAcceptOnceFileListFilter; -import org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter; -import org.springframework.integration.sftp.filters.SftpSimplePatternFileListFilter; -import org.springframework.integration.sftp.session.SftpRemoteFileTemplate; -import org.springframework.integration.util.IntegrationReactiveUtils; -import org.springframework.lang.Nullable; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.MessageHandler; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.MessagingException; -import org.springframework.messaging.PollableChannel; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.util.CollectionUtils; -import org.springframework.util.StringUtils; - -/** - * @author Gary Russell - * @author Artem Bilan - * @author Chris Schaefer - * @author Christian Tzolov - * @author David Turanski - * @author Corneil du Plessis - */ - -@Configuration -@EnableConfigurationProperties({ SftpSupplierProperties.class, FileConsumerProperties.class }) -@Import({ SftpSupplierFactoryConfiguration.class }) -public class SftpSupplierConfiguration { - - private static final String METADATA_STORE_PREFIX = "sftpSource/"; - - private static final String FILE_MODIFIED_TIME_HEADER = "FILE_MODIFIED_TIME"; - - @Bean - public MonoProcessor subscriptionBarrier() { - return MonoProcessor.create(); - } - - @Bean - public Supplier>> sftpSupplier(MessageSource sftpMessageSource, - @Nullable Publisher> sftpReadingFlow, - MonoProcessor subscriptionBarrier, - SftpSupplierProperties sftpSupplierProperties) { - - Flux> flux = sftpReadingFlow == null - ? sftpMessageFlux(sftpMessageSource, sftpSupplierProperties, subscriptionBarrier) - : Flux.from(sftpReadingFlow); - - return () -> flux.doOnRequest(l -> subscriptionBarrier.onNext(true)); - } - - @Bean - @Primary - public MessageSource sftpMessageSource( - MessageSource messageSource, - BeanFactory beanFactory, - @Nullable List receiveMessageAdvice) { - - if (CollectionUtils.isEmpty(receiveMessageAdvice)) { - return messageSource; - } - - ProxyFactoryBean proxyFactoryBean = new ProxyFactoryBean(); - proxyFactoryBean.setTarget(messageSource); - proxyFactoryBean.setBeanFactory(beanFactory); - receiveMessageAdvice.stream().map(advice -> { - NameMatchMethodPointcutAdvisor advisor = new NameMatchMethodPointcutAdvisor(advice); - advisor.addMethodName("receive"); - return advisor; - }).forEach(proxyFactoryBean::addAdvisor); - - return (MessageSource) proxyFactoryBean.getObject(); - } - - /* - * Configure the standard filters for SFTP inbound adapters. - */ - @Bean - public FileListFilter chainFilter( - SftpSupplierProperties sftpSupplierProperties, - ConcurrentMetadataStore metadataStore - ) { - - ChainFileListFilter chainFilter = new ChainFileListFilter<>(); - - if (StringUtils.hasText(sftpSupplierProperties.getFilenamePattern())) { - chainFilter.addFilter(new SftpSimplePatternFileListFilter(sftpSupplierProperties.getFilenamePattern())); - } - else if (sftpSupplierProperties.getFilenameRegex() != null) { - chainFilter.addFilter(new SftpRegexPatternFileListFilter(sftpSupplierProperties.getFilenameRegex())); - } - - chainFilter.addFilter(new SftpPersistentAcceptOnceFileListFilter(metadataStore, METADATA_STORE_PREFIX)); - return chainFilter; - } - - /* - * Create a Flux from a MessageSource that will be used by the supplier. - */ - private Flux> sftpMessageFlux(MessageSource sftpMessageSource, - SftpSupplierProperties sftpSupplierProperties, MonoProcessor subscriptionBarrier) { - - return IntegrationReactiveUtils.messageSourceToFlux(sftpMessageSource) - .delaySubscription(subscriptionBarrier) - .contextWrite(Context.of(IntegrationReactiveUtils.DELAY_WHEN_EMPTY_KEY, - sftpSupplierProperties.getDelayWhenEmpty())); - - } - - private static String remoteDirectory(SftpSupplierProperties sftpSupplierProperties) { - return sftpSupplierProperties.isMultiSource() - ? SftpSupplierProperties.keyDirectories(sftpSupplierProperties).get(0).getDirectory() - : sftpSupplierProperties.getRemoteDir(); - } - - @Configuration - @ConditionalOnProperty(prefix = "sftp.supplier", name = "stream") - static class StreamingConfiguration { - - @Bean - public SftpRemoteFileTemplate sftpTemplate(SftpSupplierFactoryConfiguration.DelegatingFactoryWrapper wrapper) { - return new SftpRemoteFileTemplate(wrapper.getFactory()); - } - - /** - * Streaming {@link MessageSource} that provides an InputStream for each remote file. It - * does not synchronize files to a local directory. - * @return a {@link MessageSource}. - */ - @Bean - public MessageSource targetMessageSource(SftpRemoteFileTemplate sftpTemplate, - SftpSupplierProperties sftpSupplierProperties, - FileListFilter fileListFilter) { - - return Sftp.inboundStreamingAdapter(sftpTemplate) - .remoteDirectory(remoteDirectory(sftpSupplierProperties)) - .remoteFileSeparator(sftpSupplierProperties.getRemoteFileSeparator()) - .filter(fileListFilter) - .maxFetchSize(sftpSupplierProperties.getMaxFetch()).get(); - } - - @Bean - public Publisher> sftpReadingFlow( - MessageSource sftpMessageSource, - MonoProcessor subscriptionBarrier, - SftpSupplierProperties sftpSupplierProperties, - FileConsumerProperties fileConsumerProperties) { - - return FileUtils.enhanceStreamFlowForReadingMode(IntegrationFlow - .from(IntegrationReactiveUtils.messageSourceToFlux(sftpMessageSource) - .delaySubscription(subscriptionBarrier) - .contextWrite(Context.of(IntegrationReactiveUtils.DELAY_WHEN_EMPTY_KEY, - sftpSupplierProperties.getDelayWhenEmpty()))), - fileConsumerProperties) - .toReactivePublisher(); - } - - @Bean - @ConditionalOnProperty(prefix = "sftp.supplier", value = "delete-remote-files") - public RemoteFileDeletingAdvice remoteFileDeletingAdvice(SftpRemoteFileTemplate sftpTemplate, - SftpSupplierProperties sftpSupplierProperties) { - - return new RemoteFileDeletingAdvice(sftpTemplate, sftpSupplierProperties.getRemoteFileSeparator()); - } - - @Bean - @ConditionalOnProperty(prefix = "sftp.supplier", value = "rename-remote-files-to") - public RemoteFileRenamingAdvice remoteFileRenamingAdvice(SftpRemoteFileTemplate sftpTemplate, - SftpSupplierProperties sftpSupplierProperties) { - - return new RemoteFileRenamingAdvice(sftpTemplate, sftpSupplierProperties.getRemoteFileSeparator(), - sftpSupplierProperties.getRenameRemoteFilesTo()); - } - } - - @Configuration - @ConditionalOnExpression("environment['sftp.supplier.stream']!='true'") - static class NonStreamingConfiguration { - - /** - * Enrich the flow to provide some standard headers, depending on - * {@link FileConsumerProperties}, when consuming file contents. - * @param sftpMessageSource the {@link MessageSource}. - * @param fileConsumerProperties the {@code FileConsumerProperties}. - * @return a {@code Publisher}. - */ - @Bean - @ConditionalOnExpression("environment['file.consumer.mode']!='ref' && environment['sftp.supplier.list-only']!='true'") - public Publisher> sftpReadingFlow( - MessageSource sftpMessageSource, - MonoProcessor subscriptionBarrier, - SftpSupplierProperties sftpSupplierProperties, - FileConsumerProperties fileConsumerProperties, - @Nullable @Qualifier("renameRemoteFileHandler") MessageHandler renameRemoteFileHandler) { - - IntegrationFlowBuilder flowBuilder = FileUtils.enhanceFlowForReadingMode(IntegrationFlow - .from(IntegrationReactiveUtils.messageSourceToFlux(sftpMessageSource) - .delaySubscription(subscriptionBarrier) - .contextWrite(Context.of(IntegrationReactiveUtils.DELAY_WHEN_EMPTY_KEY, - sftpSupplierProperties.getDelayWhenEmpty()))), - fileConsumerProperties); - - if (renameRemoteFileHandler != null) { - flowBuilder.publishSubscribeChannel(pubsub -> - pubsub.subscribe(subFlow -> subFlow.handle(renameRemoteFileHandler).nullChannel()) - ); - } - - return flowBuilder.toReactivePublisher(); - } - - /** - * A {@link MessageSource} that synchronizes files to a local directory. - * @return the {code MessageSource}. - */ - @ConditionalOnExpression("environment['sftp.supplier.list-only'] != 'true'") - @Bean - public SftpInboundChannelAdapterSpec targetMessageSource(SftpSupplierProperties sftpSupplierProperties, - SftpSupplierFactoryConfiguration.DelegatingFactoryWrapper delegatingFactoryWrapper, - FileListFilter fileListFilter) { - - return Sftp - .inboundAdapter(delegatingFactoryWrapper.getFactory()) - .preserveTimestamp(sftpSupplierProperties.isPreserveTimestamp()) - .autoCreateLocalDirectory(sftpSupplierProperties.isAutoCreateLocalDir()) - .deleteRemoteFiles(sftpSupplierProperties.isDeleteRemoteFiles()) - .localDirectory(sftpSupplierProperties.getLocalDir()) - .remoteDirectory(remoteDirectory(sftpSupplierProperties)) - .remoteFileSeparator(sftpSupplierProperties.getRemoteFileSeparator()) - .temporaryFileSuffix(sftpSupplierProperties.getTmpFileSuffix()) - .metadataStorePrefix(METADATA_STORE_PREFIX) - .maxFetchSize(sftpSupplierProperties.getMaxFetch()) - .filter(fileListFilter); - } - - @Bean - @ConditionalOnProperty(prefix = "sftp.supplier", value = "rename-remote-files-to") - public SftpOutboundGatewaySpec renameRemoteFileHandler( - SftpSupplierFactoryConfiguration.DelegatingFactoryWrapper delegatingFactoryWrapper, - SftpSupplierProperties sftpSupplierProperties) { - - return Sftp.outboundGateway(delegatingFactoryWrapper.getFactory(), - AbstractRemoteFileOutboundGateway.Command.MV.getCommand(), - String.format("headers.get('%s') + '%s' + headers.get('%s')", - FileHeaders.REMOTE_DIRECTORY, - sftpSupplierProperties.getRemoteFileSeparator(), - FileHeaders.REMOTE_FILE)) - .renameExpression(sftpSupplierProperties.getRenameRemoteFilesTo()); - } - } - - /* - * List only configuration - */ - @Configuration - @ConditionalOnProperty(prefix = "sftp.supplier", name = "list-only") - static class ListingOnlyConfiguration { - - @Bean - PollableChannel listingChannel() { - return new QueueChannel(); - } - - @Bean - @SuppressWarnings("unchecked") - public MessageSource targetMessageSource(PollableChannel listingChannel, - SftpListingMessageProducer sftpListingMessageProducer) { - return () -> { - sftpListingMessageProducer.listNames(); - return (Message) listingChannel.receive(); - }; - - } - - @Bean - public SftpListingMessageProducer sftpListingMessageProducer(SftpSupplierProperties sftpSupplierProperties, - SftpSupplierFactoryConfiguration.DelegatingFactoryWrapper delegatingFactoryWrapper) { - - return new SftpListingMessageProducer(delegatingFactoryWrapper.getFactory(), - remoteDirectory(sftpSupplierProperties), - sftpSupplierProperties.getRemoteFileSeparator(), - sftpSupplierProperties.getSortBy() - ); - } - - @Bean - GenericSelector listOnlyFilter(SftpSupplierProperties sftpSupplierProperties) { - Predicate predicate = s -> true; - if (StringUtils.hasText(sftpSupplierProperties.getFilenamePattern())) { - predicate = Pattern.compile(sftpSupplierProperties.getFilenamePattern()).asPredicate(); - } - else if (sftpSupplierProperties.getFilenameRegex() != null) { - predicate = sftpSupplierProperties.getFilenameRegex().asPredicate(); - } - - return predicate::test; - } - - @Bean - public IntegrationFlow listingFlow(MessageProducerSupport listingMessageProducer, - MessageChannel listingChannel, MessageProcessor lsEntryToStringTransformer, - GenericSelector> duplicateFilter, - GenericSelector listOnlyFilter) { - - return IntegrationFlow.from(listingMessageProducer) - .split() - .transform(lsEntryToStringTransformer) - .filter(duplicateFilter) - .filter(listOnlyFilter) - .channel(listingChannel) - .get(); - } - - @Bean - public MessageProcessor> lsEntryToStringTransformer() { - return (Message message) -> { - - SftpClient.DirEntry dirEntry = (SftpClient.DirEntry) message.getPayload(); - - String fileName = message.getHeaders().get(FileHeaders.REMOTE_DIRECTORY) + dirEntry.getFilename(); - - return MessageBuilder.withPayload(fileName) - .copyHeaders(message.getHeaders()) - .setHeader(FILE_MODIFIED_TIME_HEADER, String.valueOf(dirEntry.getAttributes().getModifyTime())) - .setHeader(MessageHeaders.CONTENT_TYPE, MediaType.TEXT_PLAIN) - .build(); - }; - - } - - @Bean - GenericSelector> duplicateFilter(ConcurrentMetadataStore metadataStore) { - return new GenericSelector>() { - @Override - public boolean accept(Message message) { - - String lastModifiedTime = (String) message.getHeaders().get(FILE_MODIFIED_TIME_HEADER); - String storedLastModifiedTime = metadataStore.get(METADATA_STORE_PREFIX + message.getPayload()); - - boolean result = !lastModifiedTime.equals(storedLastModifiedTime); - - if (result) { - metadataStore.put( - METADATA_STORE_PREFIX + message.getPayload(), - message.getHeaders().get(FILE_MODIFIED_TIME_HEADER).toString()); - } - return result; - } - }; - } - - static class SftpListingMessageProducer extends MessageProducerSupport { - - private final String remoteDirectory; - - private final SessionFactory sessionFactory; - - private final String remoteFileSeparator; - - private final SftpSupplierProperties.SortSpec sort; - - SftpListingMessageProducer(SessionFactory sessionFactory, String remoteDirectory, - String remoteFileSeparator, SftpSupplierProperties.SortSpec sort) { - - this.sessionFactory = sessionFactory; - this.remoteDirectory = remoteDirectory; - this.remoteFileSeparator = remoteFileSeparator; - this.sort = sort; - } - - public void listNames() { - Stream stream; - try { - stream = Stream.of(this.sessionFactory.getSession().list(this.remoteDirectory)) - .filter(x -> !(x.getAttributes().isDirectory() || x.getAttributes().isSymbolicLink())); - - if (sort != null) { - stream = stream.sorted(sort.comparator()); - } - } - catch (IOException e) { - throw new MessagingException(e.getMessage(), e); - } - sendMessage(MessageBuilder.withPayload(stream) - .setHeader(FileHeaders.REMOTE_DIRECTORY, this.remoteDirectory + this.remoteFileSeparator) - .build()); - } - - } - - } - -} diff --git a/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierFactoryConfiguration.java b/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierFactoryConfiguration.java deleted file mode 100644 index 012bee25b..000000000 --- a/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierFactoryConfiguration.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2018-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.sftp; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.sshd.sftp.client.SftpClient; - -import org.springframework.beans.factory.DisposableBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.core.io.Resource; -import org.springframework.integration.context.IntegrationContextUtils; -import org.springframework.integration.file.remote.aop.StandardRotationPolicy; -import org.springframework.integration.file.remote.session.CachingSessionFactory; -import org.springframework.integration.file.remote.session.DelegatingSessionFactory; -import org.springframework.integration.file.remote.session.SessionFactory; -import org.springframework.integration.sftp.session.DefaultSftpSessionFactory; -import org.springframework.lang.Nullable; - -/** - * Session factory configuration. - * - * @author Gary Russell - * @author Artem Bilan - * @author David Turanski - * @author Corneil du Plessis - * - */ -public class SftpSupplierFactoryConfiguration { - - @Bean - @ConditionalOnMissingBean - public SessionFactory sftpSessionFactory(SftpSupplierProperties properties, - ApplicationContext applicationContext) { - - return buildFactory(applicationContext, properties.getFactory()); - } - - @Bean - public DelegatingFactoryWrapper delegatingFactoryWrapper(SftpSupplierProperties properties, - SessionFactory defaultFactory, ApplicationContext applicationContext) { - - return new DelegatingFactoryWrapper(properties, defaultFactory, applicationContext); - } - - @Bean - StandardRotationPolicy rotationPolicy(SftpSupplierProperties properties, DelegatingFactoryWrapper factory) { - - return properties.isMultiSource() - ? new StandardRotationPolicy(factory.getFactory(), - SftpSupplierProperties.keyDirectories(properties), properties.isFair()) - : null; - } - - @Bean - public SftpSupplierRotator rotatingAdvice(SftpSupplierProperties properties, - @Nullable StandardRotationPolicy rotationPolicy) { - return properties.isMultiSource() - ? new SftpSupplierRotator(properties, rotationPolicy) - : null; - } - - static SessionFactory buildFactory(ApplicationContext applicationContext, - SftpSupplierProperties.Factory factory) { - - DefaultSftpSessionFactory sftpSessionFactory = new DefaultSftpSessionFactory(); - sftpSessionFactory.setHost(factory.getHost()); - sftpSessionFactory.setPort(factory.getPort()); - sftpSessionFactory.setUser(factory.getUsername()); - sftpSessionFactory.setPassword(factory.getPassword()); - sftpSessionFactory.setPrivateKey(factory.getPrivateKey()); - sftpSessionFactory.setPrivateKeyPassphrase(factory.getPassPhrase()); - sftpSessionFactory.setAllowUnknownKeys(factory.isAllowUnknownKeys()); - if (factory.getKnownHostsExpression() != null) { - String knownHostsLocation = factory.getKnownHostsExpression() - .getValue(IntegrationContextUtils.getEvaluationContext(applicationContext), String.class); - Resource knownHostsResource = applicationContext.getResource(knownHostsLocation); - sftpSessionFactory.setKnownHostsResource(knownHostsResource); - } - - return new CachingSessionFactory<>(sftpSessionFactory); - } - - public final static class DelegatingFactoryWrapper implements DisposableBean { - - private final DelegatingSessionFactory delegatingSessionFactory; - - private final Map> factories = new HashMap<>(); - - DelegatingFactoryWrapper(SftpSupplierProperties properties, SessionFactory defaultFactory, - ApplicationContext applicationContext) { - - properties.getFactories().forEach((key, factory) -> - this.factories.put(key, SftpSupplierFactoryConfiguration.buildFactory(applicationContext, factory))); - this.delegatingSessionFactory = new DelegatingSessionFactory<>(this.factories, defaultFactory); - } - - public DelegatingSessionFactory getFactory() { - return this.delegatingSessionFactory; - } - - @Override - public void destroy() { - this.factories.values().forEach(f -> { - if (f instanceof DisposableBean) { - try { - ((DisposableBean) f).destroy(); - } - catch (Exception e) { - // empty - } - } - }); - } - - } - -} diff --git a/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierProperties.java b/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierProperties.java deleted file mode 100644 index f81f49042..000000000 --- a/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierProperties.java +++ /dev/null @@ -1,517 +0,0 @@ -/* - * Copyright 2018-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.sftp; - -import java.io.File; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -import jakarta.validation.Valid; -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotBlank; -import jakarta.validation.constraints.NotNull; -import org.apache.sshd.sftp.client.SftpClient; -import org.hibernate.validator.constraints.Range; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.core.io.Resource; -import org.springframework.expression.Expression; -import org.springframework.integration.file.remote.aop.RotationPolicy; -import org.springframework.util.Assert; -import org.springframework.validation.annotation.Validated; - -/** - * @author Gary Russell - * @author Artem Bilan - * @author Chris Schaefer - * @author David Turanski - * @author Corneil du Plessis - */ -@ConfigurationProperties("sftp.supplier") -@Validated -public class SftpSupplierProperties { - /** - * Session factory properties. - */ - private final Factory factory = new Factory(); - - /** - * The remote FTP directory. - */ - private String remoteDir = "/"; - - /** - * The suffix to use while the transfer is in progress. - */ - private String tmpFileSuffix = ".tmp"; - - /** - * The remote file separator. - */ - private String remoteFileSeparator = "/"; - - /** - * Set to true to delete remote files after successful transfer. - */ - private boolean deleteRemoteFiles = false; - - /** - * A SpEL expression resolving to the new name remote files must be renamed to after successful transfer. - */ - private Expression renameRemoteFilesTo = null; - - /** - * The local directory to use for file transfers. - */ - private File localDir = new File(System.getProperty("java.io.tmpdir"), "sftp-supplier"); - - /** - * Set to true to create the local directory if it does not exist. - */ - private boolean autoCreateLocalDir = true; - - /** - * A filter pattern to match the names of files to transfer. - */ - private String filenamePattern; - - /** - * A filter regex pattern to match the names of files to transfer. - */ - private Pattern filenameRegex; - - /** - * Set to true to preserve the original timestamp. - */ - private boolean preserveTimestamp = true; - - /** - * Set to true to stream the file rather than copy to a local directory. - */ - private boolean stream = false; - - /** - * Set to true to return file metadata without the entire payload. - */ - private boolean listOnly = false; - - /** - * Duration of delay when no new files are detected. - */ - private Duration delayWhenEmpty = Duration.ofSeconds(1); - - /** - * The maximum number of remote files to fetch per poll; default unlimited. Does not apply - * when listing files or building task launch requests. - */ - private int maxFetch = Integer.MIN_VALUE; - - /** - * True for fair rotation of multiple servers/directories. This is false by default so if - * a source has more than one entry, these will be received before the other sources are - * visited. - */ - private boolean fair; - - /** - * A map of factory names to factories. - */ - private Map factories = Collections.emptyMap(); - - /** - * A list of factory "name.directory" pairs. - */ - private String[] directories; - - /** - * Sorting specification for remote files listings. If null, order of entries is undefined. - * Otherwise, entries are sorted by the specified field and direction, according to the type canonical ordering. - */ - private SortSpec sortBy; - - @NotBlank - public String getRemoteDir() { - return remoteDir; - } - - public void setRemoteDir(String remoteDir) { - this.remoteDir = remoteDir; - } - - @NotBlank - public String getTmpFileSuffix() { - return tmpFileSuffix; - } - - public void setTmpFileSuffix(String tmpFileSuffix) { - this.tmpFileSuffix = tmpFileSuffix; - } - - @NotBlank - public String getRemoteFileSeparator() { - return remoteFileSeparator; - } - - public void setRemoteFileSeparator(String remoteFileSeparator) { - this.remoteFileSeparator = remoteFileSeparator; - } - - public boolean isAutoCreateLocalDir() { - return autoCreateLocalDir; - } - - public void setAutoCreateLocalDir(boolean autoCreateLocalDir) { - this.autoCreateLocalDir = autoCreateLocalDir; - } - - public boolean isDeleteRemoteFiles() { - return deleteRemoteFiles; - } - - public void setDeleteRemoteFiles(boolean deleteRemoteFiles) { - this.deleteRemoteFiles = deleteRemoteFiles; - } - - public Expression getRenameRemoteFilesTo() { - return renameRemoteFilesTo; - } - - public void setRenameRemoteFilesTo(Expression renameRemoteFilesTo) { - this.renameRemoteFilesTo = renameRemoteFilesTo; - } - - @NotNull - public File getLocalDir() { - return localDir; - } - - public final void setLocalDir(File localDir) { - this.localDir = localDir; - } - - public String getFilenamePattern() { - return filenamePattern; - } - - public void setFilenamePattern(String filenamePattern) { - this.filenamePattern = filenamePattern; - } - - public Pattern getFilenameRegex() { - return filenameRegex; - } - - public void setFilenameRegex(Pattern filenameRegex) { - this.filenameRegex = filenameRegex; - } - - public boolean isPreserveTimestamp() { - return preserveTimestamp; - } - - public void setPreserveTimestamp(boolean preserveTimestamp) { - this.preserveTimestamp = preserveTimestamp; - } - - @AssertTrue(message = "filenamePattern and filenameRegex are mutually exclusive") - public boolean isExclusivePatterns() { - return !(this.filenamePattern != null && this.filenameRegex != null); - } - - public boolean isListOnly() { - return listOnly; - } - - public void setListOnly(boolean listOnly) { - this.listOnly = listOnly; - } - - public boolean isMultiSource() { - return this.directories != null && this.directories.length > 0; - } - - public int getMaxFetch() { - return maxFetch; - } - - public void setMaxFetch(int maxFetch) { - this.maxFetch = maxFetch; - } - - public boolean isFair() { - return this.fair; - } - - public void setFair(boolean fair) { - this.fair = fair; - } - - public Map getFactories() { - return this.factories; - } - - public void setFactories(Map factories) { - this.factories = factories; - } - - public String[] getDirectories() { - return this.directories; - } - - public void setDirectories(String[] directories) { - this.directories = directories; - } - - public boolean isStream() { - return stream; - } - - public void setStream(boolean stream) { - this.stream = stream; - } - - public Factory getFactory() { - return factory; - } - - public Duration getDelayWhenEmpty() { - return delayWhenEmpty; - } - - public void setDelayWhenEmpty(Duration delayWhenEmpty) { - this.delayWhenEmpty = delayWhenEmpty; - } - - static List keyDirectories(SftpSupplierProperties properties) { - List keyDirs = new ArrayList<>(); - Assert.isTrue(properties.getDirectories().length > 0, "At least one key.directory required"); - for (String keyDir : properties.getDirectories()) { - String[] split = keyDir.split("\\."); - Assert.isTrue(split.length == 2, () -> "key/directory can only have one '.': " + keyDir); - keyDirs.add(new RotationPolicy.KeyDirectory(split[0], split[1])); - } - return keyDirs; - } - - @Valid - public SftpSupplierProperties.SortSpec getSortBy() { - return sortBy; - } - - public void setSortBy(SortSpec sortBy) { - this.sortBy = sortBy; - } - - @AssertTrue(message = "deleteRemoteFiles must be 'false' when renameRemoteFilesTo is set") - public boolean isRenameRemoteFilesValid() { - return renameRemoteFilesTo == null || !deleteRemoteFiles; - } - - public static class Factory { - - /** - * The host name of the server. - */ - private String host = "localhost"; - - /** - * The username to use to connect to the server. - */ - - private String username; - - /** - * The password to use to connect to the server. - */ - private String password; - - /** - * The port of the server. - */ - private int port = 22; - - /** - * Resource location of user's private key. - */ - private Resource privateKey; - - /** - * Passphrase for user's private key. - */ - private String passPhrase = ""; - - /** - * True to allow an unknown or changed key. - */ - private boolean allowUnknownKeys = false; - - /** - * A SpEL expression resolving to the location of the known hosts file. - */ - private Expression knownHostsExpression = null; - - @NotBlank - public String getHost() { - return this.host; - } - - public void setHost(String host) { - this.host = host; - } - - @NotBlank - public String getUsername() { - return this.username; - } - - public void setUsername(String username) { - this.username = username; - } - - public String getPassword() { - return this.password; - } - - public void setPassword(String password) { - this.password = password; - } - - @Range(min = 0, max = 65535) - public int getPort() { - return this.port; - } - - public void setPort(int port) { - this.port = port; - } - - public Resource getPrivateKey() { - return this.privateKey; - } - - public void setPrivateKey(Resource privateKey) { - this.privateKey = privateKey; - } - - public String getPassPhrase() { - return this.passPhrase; - } - - public void setPassPhrase(String passPhrase) { - this.passPhrase = passPhrase; - } - - public boolean isAllowUnknownKeys() { - return this.allowUnknownKeys; - } - - public void setAllowUnknownKeys(boolean allowUnknownKeys) { - this.allowUnknownKeys = allowUnknownKeys; - } - - public Expression getKnownHostsExpression() { - return this.knownHostsExpression; - } - - public void setKnownHostsExpression(Expression knownHosts) { - this.knownHostsExpression = knownHosts; - } - - } - - public static class SortSpec { - /** - * Attribute of the file listing entry to sort by (FILENAME, ATIME: last access time, MTIME: last modified time). - */ - private Attribute attribute; - - /** - * Sorting direction (ASC or DESC). - */ - private Dir dir = Dir.ASC; - - @NotNull - public Attribute getAttribute() { - return attribute; - } - - public void setAttribute(Attribute attribute) { - this.attribute = attribute; - } - - @NotNull - public Dir getDir() { - return dir; - } - - public void setDir(Dir dir) { - this.dir = dir; - } - - public enum Attribute { - /** - * Filename attribute. - */ - FILENAME, - - /** - * Last access time attribute. - */ - ATIME, - - /** - * Last modified time attribute. - */ - MTIME - } - - public enum Dir { - /** - * Ascending sort direction. - */ - ASC, - - /** - * Descending sort direction. - */ - DESC - } - - private Comparator getAttributeComparator() { - switch (attribute) { - case FILENAME: - return Comparator.comparing(SftpClient.DirEntry::getFilename); - case ATIME: - return Comparator.comparing(x -> x.getAttributes().getAccessTime()); - case MTIME: - return Comparator.comparing(x -> x.getAttributes().getModifyTime()); - } - - throw new UnsupportedOperationException("Unsupported sortBy attribute: " + attribute); - } - - public Comparator comparator() { - Comparator comparator = getAttributeComparator(); - return dir == Dir.ASC ? comparator : comparator.reversed(); - } - } -} diff --git a/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierRotator.java b/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierRotator.java deleted file mode 100644 index 4f3e827af..000000000 --- a/functions/supplier/sftp-supplier/src/main/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierRotator.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2018-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.sftp; - -import org.springframework.integration.core.MessageSource; -import org.springframework.integration.file.remote.aop.RotatingServerAdvice; -import org.springframework.integration.file.remote.aop.StandardRotationPolicy; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; - -/** - * An {@link RotatingServerAdvice} for listing files on multiple directories/servers. - * - * @author Gary Russell - * @author David Turanski - * @since 2.0 - */ -public class SftpSupplierRotator extends RotatingServerAdvice { - - private static String SFTP_SELECTED_SERVER_PROPERTY_KEY = "sftp_selectedServer"; - - private final SftpSupplierProperties properties; - - private final StandardRotationPolicy rotationPolicy; - - public SftpSupplierRotator(SftpSupplierProperties properties, StandardRotationPolicy rotationPolicy) { - super(rotationPolicy); - this.properties = properties; - this.rotationPolicy = rotationPolicy; - } - - public String getCurrentKey() { - return this.rotationPolicy.getCurrent().getKey().toString(); - } - - public String getCurrentDirectory() { - return this.rotationPolicy.getCurrent().getDirectory(); - } - - @Override - public Message afterReceive(Message result, MessageSource source) { - if (result != null) { - result = MessageBuilder.fromMessage(result) - .setHeader(SFTP_SELECTED_SERVER_PROPERTY_KEY, this.getCurrentKey()).build(); - } - this.rotationPolicy.afterReceive(result != null, source); - return result; - } -} diff --git a/functions/supplier/sftp-supplier/src/test/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierApplicationTests.java b/functions/supplier/sftp-supplier/src/test/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierApplicationTests.java deleted file mode 100644 index 5cfd6ffb2..000000000 --- a/functions/supplier/sftp-supplier/src/test/java/org/springframework/cloud/fn/supplier/sftp/SftpSupplierApplicationTests.java +++ /dev/null @@ -1,449 +0,0 @@ -/* - * Copyright 2020-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.sftp; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.assertj.AssertableApplicationContext; -import org.springframework.boot.test.context.runner.ApplicationContextRunner; -import org.springframework.cloud.fn.test.support.sftp.SftpTestSupport; -import org.springframework.http.MediaType; -import org.springframework.integration.file.splitter.FileSplitter; -import org.springframework.integration.json.JsonPathUtils; -import org.springframework.integration.metadata.MetadataStore; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.fail; -import static org.awaitility.Awaitility.await; - -public class SftpSupplierApplicationTests extends SftpTestSupport { - - ApplicationContextRunner defaultApplicationContextRunner; - - @BeforeEach - void setUpDefaultProperties() { - defaultApplicationContextRunner = new ApplicationContextRunner() - .withUserConfiguration(SftpSupplierTestApplication.class) - .withPropertyValues( - "sftp.supplier.factory.host=localhost", - "sftp.supplier.factory.port=${sftp.factory.port}", - "sftp.supplier.factory.username=user", - "sftp.supplier.factory.password=pass", - "sftp.supplier.factory.cache-sessions=true", - "sftp.supplier.factory.allowUnknownKeys=true", - "sftp.supplier.localDir=" + this.targetLocalDirectory.getAbsolutePath(), - "sftp.supplier.remoteDir=sftpSource"); - } - - @Test - void supplierForListOnly() { - defaultApplicationContextRunner - .withPropertyValues("sftp.supplier.listOnly=true") - .run(context -> { - Supplier>> sftpSupplier = context.getBean("sftpSupplier", - Supplier.class); - SftpSupplierProperties properties = context.getBean(SftpSupplierProperties.class); - HashSet fileNames = new HashSet<>(); - fileNames.add(String.join(properties.getRemoteFileSeparator(), properties.getRemoteDir(), - "sftpSource1.txt")); - fileNames.add(String.join(properties.getRemoteFileSeparator(), properties.getRemoteDir(), - "sftpSource2.txt")); - final AtomicReference> expectedFileNames = new AtomicReference<>(fileNames); - StepVerifier.create(sftpSupplier.get()) - .assertNext(message -> { - assertThat(expectedFileNames.get()).contains(message.getPayload()); - expectedFileNames.get().remove(message.getPayload()); - assertThat(message.getHeaders().get(MessageHeaders.CONTENT_TYPE)) - .isEqualTo(MediaType.TEXT_PLAIN); - }) - .assertNext(message -> { - assertThat(expectedFileNames.get()).contains(message.getPayload()); - assertThat(message.getHeaders().get(MessageHeaders.CONTENT_TYPE)) - .isEqualTo(MediaType.TEXT_PLAIN); - }) - .expectTimeout(Duration.ofMillis(1000)) - .verify(Duration.ofSeconds(30)); - - }); - } - - @Test - void supplierForListOnlyWithPatternFilter() { - defaultApplicationContextRunner - .withPropertyValues("sftp.supplier.listOnly=true", "sftp.supplier.file-name-pattern=.*1.txt") - .run(context -> { - Supplier>> sftpSupplier = context.getBean("sftpSupplier", - Supplier.class); - SftpSupplierProperties properties = context.getBean(SftpSupplierProperties.class); - - final AtomicReference expectedFileName = new AtomicReference<>( - properties.getRemoteDir() + "/sftpSource1.txt"); - StepVerifier.create(sftpSupplier.get()) - .assertNext(message -> assertThat(expectedFileName.get()).contains(message.getPayload())) - .expectTimeout(Duration.ofMillis(1000)) - .verify(Duration.ofSeconds(30)); - - }); - } - - @Test - void supplierForListSortedByFilenameAsc() { - defaultApplicationContextRunner - .withPropertyValues("sftp.supplier.listOnly=true", "sftp.supplier.sortBy.attribute=filename", "sftp.supplier.sortBy.dir=asc") - .run(context -> { - Supplier>> sftpSupplier = context.getBean("sftpSupplier", - Supplier.class); - SftpSupplierProperties properties = context.getBean(SftpSupplierProperties.class); - List fileNames = new ArrayList<>(); - fileNames.add(String.join(properties.getRemoteFileSeparator(), properties.getRemoteDir(), - "sftpSource1.txt")); - fileNames.add(String.join(properties.getRemoteFileSeparator(), properties.getRemoteDir(), - "sftpSource2.txt")); - final AtomicReference> expectedFileNames = new AtomicReference<>(fileNames); - StepVerifier.create(sftpSupplier.get()) - .assertNext(message -> { - assertThat(message.getPayload()).isEqualTo(expectedFileNames.get().get(0)); - assertThat(message.getHeaders().get(MessageHeaders.CONTENT_TYPE)) - .isEqualTo(MediaType.TEXT_PLAIN); - }) - .assertNext(message -> { - assertThat(message.getPayload()).isEqualTo(expectedFileNames.get().get(1)); - assertThat(message.getHeaders().get(MessageHeaders.CONTENT_TYPE)) - .isEqualTo(MediaType.TEXT_PLAIN); - }) - .expectTimeout(Duration.ofMillis(1000)) - .verify(Duration.ofSeconds(30)); - - }); - } - - @Test - void supplierForListSortedByFilenameDesc() { - defaultApplicationContextRunner - .withPropertyValues("sftp.supplier.listOnly=true", "sftp.supplier.sortBy.attribute=filename", "sftp.supplier.sortBy.dir=desc") - .run(context -> { - Supplier>> sftpSupplier = context.getBean("sftpSupplier", - Supplier.class); - SftpSupplierProperties properties = context.getBean(SftpSupplierProperties.class); - List fileNames = new ArrayList<>(); - fileNames.add(String.join(properties.getRemoteFileSeparator(), properties.getRemoteDir(), - "sftpSource2.txt")); - fileNames.add(String.join(properties.getRemoteFileSeparator(), properties.getRemoteDir(), - "sftpSource1.txt")); - final AtomicReference> expectedFileNames = new AtomicReference<>(fileNames); - StepVerifier.create(sftpSupplier.get()) - .assertNext(message -> { - assertThat(message.getPayload()).isEqualTo(expectedFileNames.get().get(0)); - assertThat(message.getHeaders().get(MessageHeaders.CONTENT_TYPE)) - .isEqualTo(MediaType.TEXT_PLAIN); - }) - .assertNext(message -> { - assertThat(message.getPayload()).isEqualTo(expectedFileNames.get().get(1)); - assertThat(message.getHeaders().get(MessageHeaders.CONTENT_TYPE)) - .isEqualTo(MediaType.TEXT_PLAIN); - }) - .expectTimeout(Duration.ofMillis(1000)) - .verify(Duration.ofSeconds(30)); - - }); - } - - @Test - void supplierForFileRef() { - defaultApplicationContextRunner - .withPropertyValues( - "sftp.supplier.localDir=" + getTargetLocalDirectory().getAbsolutePath(), - "file.consumer.mode=ref") - .run(context -> { - Supplier>> sftpSupplier = context.getBean("sftpSupplier", - Supplier.class); - SftpSupplierProperties properties = context.getBean(SftpSupplierProperties.class); - MetadataStore metadataStore = context.getBean(MetadataStore.class); - HashSet fileNames = new HashSet<>(); - fileNames.add(properties.getLocalDir() + File.separator + "sftpSource1.txt"); - fileNames.add(properties.getLocalDir() + File.separator + "sftpSource2.txt"); - final AtomicReference> expectedFileNames = new AtomicReference<>(fileNames); - StepVerifier.create(sftpSupplier.get()) - .assertNext(message -> { - File file = message.getPayload(); - assertThat(expectedFileNames.get()).contains(file.getAbsolutePath()); - expectedFileNames.get().remove(file.getAbsolutePath()); - }) - .expectNextMatches( - message -> expectedFileNames.get().contains(message.getPayload().getAbsolutePath())) - .thenCancel() - .verify(Duration.ofSeconds(30)); - - assertThat(metadataStore.get("sftpSource/sftpSource1.txt")).isNotNull(); - assertThat(metadataStore.get("sftpSource/sftpSource2.txt")).isNotNull(); - assertThat(Files.exists(Paths.get(getTargetLocalDirectory().getAbsolutePath(), "sftpSource1.txt"))) - .isTrue(); - assertThat(Files.exists(Paths.get(getTargetLocalDirectory().getAbsolutePath(), "sftpSource2.txt"))) - .isTrue(); - }); - } - - @Test - void deleteRemoteFiles() { - defaultApplicationContextRunner - .withPropertyValues( - "sftp.supplier.stream=true", - "sftp.supplier.delete-remote-files=true") - .run(context -> { - Supplier>> sftpSupplier = context.getBean("sftpSupplier", - Supplier.class); - StepVerifier.create(sftpSupplier.get()) - .expectNextMatches(message -> message.getPayload().length > 0) - .expectNextMatches(message -> message.getPayload().length > 0) - .thenCancel() - .verify(Duration.ofSeconds(30)); - await().atMost(Duration.ofSeconds(30)) - .until(() -> getSourceRemoteDirectory().list().length == 0); - }); - - } - - @Test - void renameRemoteFilesStream() { - defaultApplicationContextRunner - .withPropertyValues( - "sftp.supplier.stream=true", - "sftp.supplier.delete-remote-files=false", - "sftp.supplier.rename-remote-files-to='/sftpTarget/' + headers.file_remoteFile") - .run(this::doTestRenameRemoteFiles); - } - - @Test - void renameRemoteFiles() { - defaultApplicationContextRunner - .withPropertyValues( - "sftp.supplier.stream=false", - "sftp.supplier.delete-remote-files=false", - "sftp.supplier.rename-remote-files-to='/sftpTarget/' + headers.file_remoteFile") - .run(this::doTestRenameRemoteFiles); - } - - private void doTestRenameRemoteFiles(AssertableApplicationContext context) { - Supplier>> sftpSupplier = context.getBean("sftpSupplier", - Supplier.class); - - final Set expectedTargetFiles = Arrays.stream(getSourceRemoteDirectory().list()) - .collect(Collectors.toSet()); - - StepVerifier.create(sftpSupplier.get()) - .expectNextMatches(message -> message.getPayload().length > 0) - .expectNextMatches(message -> message.getPayload().length > 0) - .thenCancel() - .verify(Duration.ofSeconds(30)); - await().atMost(Duration.ofSeconds(30)) - .until(() -> - expectedTargetFiles.equals( - Arrays.stream(getTargetRemoteDirectory().list()) - .collect(Collectors.toSet())) - ); - } - - @Test - public void streamSourceFilesInLineMode() { - defaultApplicationContextRunner - .withPropertyValues( - "sftp.supplier.stream=true", - "sftp.supplier.factory.private-key = classpath:id_rsa_pp", - "sftp.supplier.factory.passphrase = secret", - "sftp.supplier.factory.password = badPassword", // ensure public key was used - "sftp.supplier.delete-remote-files=true", - "file.consumer.mode=lines", - "file.consumer.with-markers=true", - "file.consumer.markers-json=true") - .run(context -> { - Supplier>> sftpSupplier = context.getBean("sftpSupplier", - Supplier.class); - StepVerifier.create(sftpSupplier.get()) - .assertNext(message -> { - final Object evaluate; - try { - evaluate = JsonPathUtils.evaluate(message.getPayload(), "$.mark"); - assertThat(evaluate).isEqualTo(FileSplitter.FileMarker.Mark.START.name()); - } - catch (IOException e) { - fail(e.getMessage()); - } - }) - .expectNextMatches(message -> message.getPayload().startsWith("source")) - .assertNext(message -> { - final Object evaluate; - try { - evaluate = JsonPathUtils.evaluate(message.getPayload(), "$.mark"); - assertThat(evaluate).isEqualTo(FileSplitter.FileMarker.Mark.END.name()); - } - catch (IOException e) { - fail(e.getMessage()); - } - }) - .thenCancel() - .verify(Duration.ofSeconds(30)); - }); - } - - @Test - void supplierWithMultiSourceAndStreamContentsSource3ComesSecond() throws Exception { - Path newSource = createNewRemoteSource( - Paths.get(remoteTemporaryFolder.toString(), "sftpSecondSource", "doesNotMatter.txt"), - "source3"); - try { - new ApplicationContextRunner() - .withUserConfiguration(SftpSupplierTestApplication.class) - .withPropertyValues( - "sftp.supplier.stream=true", - "sftp.supplier.factories.one.host=localhost", - "sftp.supplier.factories.one.port=${sftp.factory.port}", - "sftp.supplier.factories.one.username=user", - "sftp.supplier.factories.one.password=pass", - "sftp.supplier.factories.one.cache-sessions=true", - "sftp.supplier.factories.one.allowUnknownKeys=true", - "sftp.supplier.factories.two.host=localhost", - "sftp.supplier.factories.two.port=${sftp.factory.port}", - "sftp.supplier.factories.two.username = user", - "sftp.supplier.factories.two.password = pass", - "sftp.supplier.factories.two.cache-sessions = true", - "sftp.supplier.factories.two.allowUnknownKeys = true", - "sftp.supplier.directories=one.sftpSource,two.sftpSecondSource", - "sftp.supplier.max-fetch=1", - "sftp.supplier.fair=true") - .run(context -> { - - Supplier>> sftpSupplier = context.getBean("sftpSupplier", Supplier.class); - HashSet contents = new HashSet<>(); - contents.add("source1"); - contents.add("source2"); - final AtomicReference> expectedContentsOfAllFiles = new AtomicReference<>(contents); - - StepVerifier.create(sftpSupplier.get()) - .assertNext(message -> { - String payload = new String(message.getPayload()); - assertThat(expectedContentsOfAllFiles.get()).contains(payload); - expectedContentsOfAllFiles.get().remove(payload); - }) - .expectNextMatches(message -> new String(message.getPayload()).equals("source3")) - .expectNextMatches(message -> expectedContentsOfAllFiles.get() - .contains(new String(message.getPayload()))) - .thenCancel() - .verify(Duration.ofSeconds(30)); - }); - } - finally { - deleteNewSource(newSource); - } - } - - @Test - void supplierMultiSourceRefTestsFor200Alex() throws Exception { - Path newSource = createNewRemoteSource( - Paths.get(remoteTemporaryFolder.toString(), "sftpSecondSource", "sftpSource3.txt"), - "doesNotMatter"); - try { - new ApplicationContextRunner() - .withUserConfiguration(SftpSupplierTestApplication.class) - .withPropertyValues( - "file.consumer.mode = ref", - "sftp.supplier.localDir=" + this.targetLocalDirectory.getAbsolutePath(), - "sftp.supplier.factories.one.host=localhost", - "sftp.supplier.factories.one.port=${sftp.factory.port}", - "sftp.supplier.factories.one.username = user", - "sftp.supplier.factories.one.password = pass", - "sftp.supplier.factories.one.cache-sessions = true", - "sftp.supplier.factories.one.allowUnknownKeys = true", - "sftp.supplier.factories.two.host=localhost", - "sftp.supplier.factories.two.port=${sftp.factory.port}", - "sftp.supplier.factories.two.username = user", - "sftp.supplier.factories.two.password = pass", - "sftp.supplier.factories.two.cache-sessions = true", - "sftp.supplier.factories.two.allowUnknownKeys = true", - "sftp.supplier.factories.empty.host=localhost", - "sftp.supplier.factories.empty.port=${sftp.factory.port}", - "sftp.supplier.factories.empty.username=user", - "sftp.supplier.factories.empty.password=pass", - "sftp.supplier.factories.empty.allowUnknownKeys = true", - "sftp.supplier.directories=one.sftpSource,two.sftpSecondSource,empty.sftpSource", - "sftp.supplier.max-fetch=1", - "sftp.supplier.fair=true") - .run(context -> { - Supplier>> sftpSupplier = context.getBean("sftpSupplier", Supplier.class); - SftpSupplierProperties properties = context.getBean(SftpSupplierProperties.class); - String localDir = properties.getLocalDir().getPath(); - HashSet firstSourceFiles = new HashSet<>(); - firstSourceFiles.add(Paths.get(localDir, "sftpSource1.txt").toString()); - firstSourceFiles.add(Paths.get(localDir, "sftpSource2.txt").toString()); - final AtomicReference> expectedFirstSourcePaths = new AtomicReference<>( - firstSourceFiles); - - StepVerifier.create(sftpSupplier.get()) - .assertNext(message -> { - assertThat(expectedFirstSourcePaths.get()).contains(message.getPayload().getPath()); - expectedFirstSourcePaths.get().remove(message.getPayload().getPath()); - }) - .expectNextMatches(message -> message.getPayload().getPath().equals( - Paths.get(localDir, "sftpSource3.txt").toString())) - .expectNextMatches(message -> expectedFirstSourcePaths.get() - .contains(message.getPayload().getPath())) - .thenCancel() - .verify(Duration.ofSeconds(10)); - }); - } - finally { - deleteNewSource(newSource); - } - } - - private Path createNewRemoteSource(Path remotePath, String contents) throws Exception { - Files.createDirectory(remotePath.getParent()); - Files.write(Files.createFile(remotePath), contents.getBytes()); - return remotePath; - } - - private void deleteNewSource(Path newSource) throws IOException { - Files.delete(newSource); - Files.delete(newSource.getParent()); - } - - @SpringBootApplication - static class SftpSupplierTestApplication { - - } - -} diff --git a/functions/supplier/sftp-supplier/src/test/resources/logback.xml b/functions/supplier/sftp-supplier/src/test/resources/logback.xml deleted file mode 100644 index c85b56f12..000000000 --- a/functions/supplier/sftp-supplier/src/test/resources/logback.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - - - diff --git a/functions/supplier/syslog-supplier/README.adoc b/functions/supplier/syslog-supplier/README.adoc deleted file mode 100644 index 85cd3b0e9..000000000 --- a/functions/supplier/syslog-supplier/README.adoc +++ /dev/null @@ -1,32 +0,0 @@ -# Syslog Supplier - -Syslog supplier that produces both TCP and UDP based syslog events. -The `Supplier` uses the `TcpSyslogReceivingChannelAdapter` and `UdpSyslogReceivingChannelAdapter` from Spring Integration. -This supplier gives you a reactive stream of messages and the supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and receive the data. - -## Beans for injection - -You can import the `SyslogSupplierConfiguration` in the application and then inject the following bean. - -`syslogSupplier` - -You need to inject this as `Supplier>>`. - -You can use `syslogSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `syslog.supplier`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/syslog/SyslogSupplierProperties.java[SyslogSupplierProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/syslog[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/syslog-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a Syslog Source. \ No newline at end of file diff --git a/functions/supplier/syslog-supplier/pom.xml b/functions/supplier/syslog-supplier/pom.xml deleted file mode 100644 index d0e8cbb8e..000000000 --- a/functions/supplier/syslog-supplier/pom.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - syslog-supplier - syslog-supplier - syslog supplier - - - - org.springframework.integration - spring-integration-syslog - - - - diff --git a/functions/supplier/syslog-supplier/src/main/java/org/springframework/cloud/fn/supplier/syslog/SyslogSupplierConfiguration.java b/functions/supplier/syslog-supplier/src/main/java/org/springframework/cloud/fn/supplier/syslog/SyslogSupplierConfiguration.java deleted file mode 100644 index dbd9a093b..000000000 --- a/functions/supplier/syslog-supplier/src/main/java/org/springframework/cloud/fn/supplier/syslog/SyslogSupplierConfiguration.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright 2020-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.syslog; - -import java.util.function.Supplier; - -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.core.serializer.Deserializer; -import org.springframework.integration.channel.FluxMessageChannel; -import org.springframework.integration.ip.tcp.connection.AbstractServerConnectionFactory; -import org.springframework.integration.ip.tcp.connection.TcpNetServerConnectionFactory; -import org.springframework.integration.ip.tcp.connection.TcpNioServerConnectionFactory; -import org.springframework.integration.ip.tcp.serializer.ByteArrayLfSerializer; -import org.springframework.integration.syslog.DefaultMessageConverter; -import org.springframework.integration.syslog.MessageConverter; -import org.springframework.integration.syslog.RFC5424MessageConverter; -import org.springframework.integration.syslog.inbound.RFC6587SyslogDeserializer; -import org.springframework.integration.syslog.inbound.SyslogReceivingChannelAdapterSupport; -import org.springframework.integration.syslog.inbound.TcpSyslogReceivingChannelAdapter; -import org.springframework.integration.syslog.inbound.UdpSyslogReceivingChannelAdapter; -import org.springframework.messaging.Message; - -/** - * Configuration class for SYSLOG Supplier. - * - * @author Soby Chacko - */ -@Configuration -@EnableConfigurationProperties(SyslogSupplierProperties.class) -public class SyslogSupplierConfiguration { - - @Autowired - private SyslogSupplierProperties properties; - - @Bean - public FluxMessageChannel syslogInputChannel() { - return new FluxMessageChannel(); - } - - @Bean - public Supplier>> syslogSupplier(ObjectProvider udpAdapterProvider, - ObjectProvider tcpAdapterProvider) { - return () -> Flux.from(syslogInputChannel()) - .doOnSubscribe(subscription -> { - final UdpSyslogReceivingChannelAdapter udpAdapter = udpAdapterProvider.getIfAvailable(); - final TcpSyslogReceivingChannelAdapter tcpAdapter = tcpAdapterProvider.getIfAvailable(); - if (udpAdapter != null) { - udpAdapter.start(); - } - if (tcpAdapter != null) { - tcpAdapter.start(); - } - }); - } - - @Bean - @ConditionalOnProperty(name = "syslog.supplier.protocol", havingValue = "udp") - public UdpSyslogReceivingChannelAdapter udpAdapter() { - return createUdpAdapter(); - } - - @Bean - @ConditionalOnProperty(name = "syslog.supplier.protocol", havingValue = "both") - public UdpSyslogReceivingChannelAdapter udpBothAdapter() { - return createUdpAdapter(); - } - - private UdpSyslogReceivingChannelAdapter createUdpAdapter() { - UdpSyslogReceivingChannelAdapter adapter = new UdpSyslogReceivingChannelAdapter(); - setAdapterProperties(adapter); - return adapter; - } - - @Bean - @ConditionalOnProperty(name = "syslog.supplier.protocol", havingValue = "tcp", matchIfMissing = true) - public TcpSyslogReceivingChannelAdapter tcpAdapter( - @Qualifier("syslogSupplierConnectionFactory") AbstractServerConnectionFactory connectionFactory) { - return createTcpAdapter(connectionFactory); - } - - @Bean - @ConditionalOnProperty(name = "syslog.supplier.protocol", havingValue = "both") - public TcpSyslogReceivingChannelAdapter tcpBothAdapter( - @Qualifier("syslogSupplierConnectionFactory") AbstractServerConnectionFactory connectionFactory) { - return createTcpAdapter(connectionFactory); - } - - @Bean - public MessageConverter syslogConverter() { - if (this.properties.getRfc().equals("5424")) { - return new RFC5424MessageConverter(); - } - else { - return new DefaultMessageConverter(); - } - } - - private TcpSyslogReceivingChannelAdapter createTcpAdapter(AbstractServerConnectionFactory connectionFactory) { - TcpSyslogReceivingChannelAdapter adapter = new TcpSyslogReceivingChannelAdapter(); - adapter.setConnectionFactory(connectionFactory); - setAdapterProperties(adapter); - return adapter; - } - - private void setAdapterProperties(SyslogReceivingChannelAdapterSupport adapter) { - adapter.setPort(this.properties.getPort()); - adapter.setConverter(syslogConverter()); - adapter.setOutputChannel(syslogInputChannel()); - adapter.setAutoStartup(false); - } - - - @Configuration - @ConditionalOnProperty(name = "syslog.supplier.protocol", havingValue = "tcp", matchIfMissing = true) - protected static class TcpBits { - - @Autowired - private SyslogSupplierProperties properties; - - @Bean - public AbstractServerConnectionFactory syslogSupplierConnectionFactory( - @Qualifier("syslogSupplierDecoder") Deserializer decoder) throws Exception { - AbstractServerConnectionFactory factory; - if (this.properties.isNio()) { - factory = new TcpNioServerConnectionFactory(this.properties.getPort()); - } - else { - factory = new TcpNetServerConnectionFactory(this.properties.getPort()); - } - factory.setLookupHost(this.properties.isReverseLookup()); - factory.setDeserializer(decoder); - factory.setSoTimeout(this.properties.getSocketTimeout()); - return factory; - } - - @Bean - public Deserializer syslogSupplierDecoder() { - ByteArrayLfSerializer decoder = new ByteArrayLfSerializer(); - decoder.setMaxMessageSize(this.properties.getBufferSize()); - if (this.properties.getRfc().equals("5424")) { - return new RFC6587SyslogDeserializer(decoder); - } - else { - return decoder; - } - } - } - - @Configuration - @ConditionalOnProperty(name = "syslog.supplier.protocol", havingValue = "both") - protected static class BothBits extends TcpBits { - - } - -} diff --git a/functions/supplier/syslog-supplier/src/main/java/org/springframework/cloud/fn/supplier/syslog/SyslogSupplierProperties.java b/functions/supplier/syslog-supplier/src/main/java/org/springframework/cloud/fn/supplier/syslog/SyslogSupplierProperties.java deleted file mode 100644 index 8240290e6..000000000 --- a/functions/supplier/syslog-supplier/src/main/java/org/springframework/cloud/fn/supplier/syslog/SyslogSupplierProperties.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.syslog; - -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -@ConfigurationProperties("syslog.supplier") -@Validated -public class SyslogSupplierProperties { - - /** - * the buffer size used when decoding messages; larger messages will be rejected. - */ - private int bufferSize = 2048; - - /** - * Protocol used for SYSLOG (tcp or udp). - */ - private Protocol protocol = Protocol.tcp; - - /** - * The port to listen on. - */ - private int port = 1514; - - /** - * whether or not to use NIO (when supporting a large number of connections). - */ - private boolean nio = false; - - /** - * whether or not to perform a reverse lookup on the incoming socket. - */ - private boolean reverseLookup; - - /** - * the socket timeout. - */ - private int socketTimeout; - - /** - * '5424' or '3164' - the syslog format according to the RFC; 3164 is aka 'BSD' format. - */ - private String rfc = "3164"; - - public int getBufferSize() { - return bufferSize; - } - - public void setBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - } - - public Protocol getProtocol() { - return protocol; - } - - public void setProtocol(Protocol protocol) { - this.protocol = protocol; - } - - public int getPort() { - return port; - } - - public void setPort(int port) { - this.port = port; - } - - public boolean isNio() { - return nio; - } - - public void setNio(boolean nio) { - this.nio = nio; - } - - public boolean isReverseLookup() { - return reverseLookup; - } - - public void setReverseLookup(boolean reverseLookup) { - this.reverseLookup = reverseLookup; - } - - public int getSocketTimeout() { - return socketTimeout; - } - - public void setSocketTimeout(int socketTimeout) { - this.socketTimeout = socketTimeout; - } - - @NotNull - public String getRfc() { - return rfc; - } - - public void setRfc(String rfc) { - this.rfc = rfc; - } - - @AssertTrue(message = "rfc must be 5424 or 3164") - public boolean isSupportedRfc() { - return "5424".equals(this.rfc) || "3164".equals(this.rfc); - } - - public enum Protocol { - /** - * TCP protocol. - */ - tcp, - - /** - * UDP protocol. - */ - udp, - - /** - * Represents both TCP and UDP. - */ - both; - } -} diff --git a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/AbstractSyslogSupplierTests.java b/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/AbstractSyslogSupplierTests.java deleted file mode 100644 index 599f15817..000000000 --- a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/AbstractSyslogSupplierTests.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.syslog; - -import java.net.DatagramPacket; -import java.net.DatagramSocket; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.util.function.Supplier; - -import javax.net.SocketFactory; - -import reactor.core.publisher.Flux; - -import org.springframework.beans.DirectFieldAccessor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.integration.ip.tcp.connection.AbstractServerConnectionFactory; -import org.springframework.integration.ip.udp.UnicastReceivingChannelAdapter; -import org.springframework.integration.syslog.inbound.UdpSyslogReceivingChannelAdapter; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - - -@SpringBootTest(properties = "syslog.supplier.port = 0") -@DirtiesContext -public class AbstractSyslogSupplierTests { - - protected static final String RFC3164_PACKET = "<157>JUL 26 22:08:35 WEBERN TESTING[70729]: TEST SYSLOG MESSAGE"; - - protected static final String RFC5424_PACKET = - "<14>1 2014-06-20T09:14:07+00:00 loggregator d0602076-b14a-4c55-852a-981e7afeed38 DEA - " - + "[exampleSDID@32473 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"1011\\\"]" - + "[exampleSDID@32473 iut=\\\"3\\\" eventSource=\\\"Application\\\" eventID=\\\"1011\\\"] " - + "Removing instance"; - - @Autowired - Supplier>> syslogSupplier; - - @Autowired - protected SyslogSupplierProperties properties; - - @Autowired(required = false) - protected AbstractServerConnectionFactory connectionFactory; - - @Autowired(required = false) - protected UdpSyslogReceivingChannelAdapter udpAdapter; - - protected void sendTcp(String syslog) throws Exception { - int port = getPort(); - Socket socket = SocketFactory.getDefault().createSocket("localhost", port); - socket.getOutputStream().write(syslog.getBytes()); - socket.close(); - } - - private int getPort() throws Exception { - int n = 0; - while (n++ < 100 && !this.connectionFactory.isListening()) { - Thread.sleep(100); - } - assertThat(this.connectionFactory.isListening()).isTrue(); - int port = this.connectionFactory.getPort(); - assertThat(port > 0).isTrue(); - return port; - } - - protected void sendUdp(String syslog) throws Exception { - int port = waitUdp(); - DatagramSocket socket = new DatagramSocket(); - DatagramPacket packet = new DatagramPacket(syslog.getBytes(), syslog.length()); - packet.setSocketAddress(new InetSocketAddress("localhost", port)); - socket.send(packet); - socket.close(); - } - - private int waitUdp() throws Exception { - int n = 0; - DirectFieldAccessor dfa = new DirectFieldAccessor(this.udpAdapter); - while (n++ < 100 && !((UnicastReceivingChannelAdapter) dfa.getPropertyValue("udpAdapter")).isListening()) { - Thread.sleep(100); - } - return ((UnicastReceivingChannelAdapter) dfa.getPropertyValue("udpAdapter")).getPort(); - } - - @SpringBootApplication - public static class SyslogSupplierTestApplication { - - } - -} diff --git a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/NotNioTests.java b/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/NotNioTests.java deleted file mode 100644 index fd485d967..000000000 --- a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/NotNioTests.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.syslog; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.connection.TcpNetServerConnectionFactory; -import org.springframework.integration.test.util.TestUtils; - -import static org.assertj.core.api.Assertions.assertThat; - -public class NotNioTests extends AbstractSyslogSupplierTests { - - @Test - public void test() throws Exception { - assertThat(this.connectionFactory).isInstanceOf(TcpNetServerConnectionFactory.class); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "lookupHost", Boolean.class)).isFalse(); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "soTimeout")).isEqualTo(0); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "deserializer.maxMessageSize")).isEqualTo(2048); - } -} diff --git a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/PropertiesPopulatedTests.java b/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/PropertiesPopulatedTests.java deleted file mode 100644 index 5e2c7dbb5..000000000 --- a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/PropertiesPopulatedTests.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.syslog; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.connection.TcpNioServerConnectionFactory; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { "syslog.supplier.port = 0", "syslog.supplier.nio = true", "syslog.supplier.reverseLookup = true", - "syslog.supplier.socketTimeout = 123", "syslog.supplier.bufferSize = 5" }) -public class PropertiesPopulatedTests extends AbstractSyslogSupplierTests { - - @Test - public void test() throws Exception { - assertThat(this.connectionFactory).isInstanceOf(TcpNioServerConnectionFactory.class); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "lookupHost", Boolean.class)).isTrue(); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "soTimeout")).isEqualTo(123); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "deserializer.maxMessageSize")).isEqualTo(5); - } -} diff --git a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Tcp3164Tests.java b/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Tcp3164Tests.java deleted file mode 100644 index 648a4130b..000000000 --- a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Tcp3164Tests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.syslog; - -import java.util.Map; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; - -import static org.assertj.core.api.Assertions.assertThat; - -public class Tcp3164Tests extends AbstractSyslogSupplierTests { - - @Test - public void test() throws Exception { - final Flux> messageFlux = syslogSupplier.get(); - - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(((Map) message.getPayload()).get("HOST")).isEqualTo("WEBERN"); - } - ) - .thenCancel() - .verifyLater(); - - sendTcp(AbstractSyslogSupplierTests.RFC3164_PACKET + "\n"); - - stepVerifier.verify(); - } -} diff --git a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Tcp5424Tests.java b/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Tcp5424Tests.java deleted file mode 100644 index a0934956b..000000000 --- a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Tcp5424Tests.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.syslog; - -import java.util.Map; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { "syslog.supplier.port = 0", "syslog.supplier.rfc = 5424" }) -public class Tcp5424Tests extends AbstractSyslogSupplierTests { - - @Test - public void test() throws Exception { - final Flux> messageFlux = syslogSupplier.get(); - - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(((Map) message.getPayload()).get("syslog_HOST")).isEqualTo("loggregator"); - } - ) - .thenCancel() - .verifyLater(); - - sendTcp("253 " + RFC5424_PACKET); - - stepVerifier.verify(); - } -} diff --git a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/TcpAndUdp3164Tests.java b/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/TcpAndUdp3164Tests.java deleted file mode 100644 index da19ce977..000000000 --- a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/TcpAndUdp3164Tests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.syslog; - -import java.util.Map; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = "syslog.supplier.protocol = both") -public class TcpAndUdp3164Tests extends AbstractSyslogSupplierTests { - - @Test - public void test() throws Exception { - final Flux> messageFlux = syslogSupplier.get(); - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(((Map) message.getPayload()).get("HOST")).isEqualTo("WEBERN"); - } - ) - .assertNext((message) -> { - assertThat(((Map) message.getPayload()).get("HOST")).isEqualTo("WEBERN"); - } - ) - .thenCancel() - .verifyLater(); - - sendTcp(RFC3164_PACKET + "\n"); - sendUdp(RFC3164_PACKET); - stepVerifier.verify(); - } -} diff --git a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/TcpAndUdp5424Tests.java b/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/TcpAndUdp5424Tests.java deleted file mode 100644 index 394ce58cb..000000000 --- a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/TcpAndUdp5424Tests.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.syslog; - -import java.util.Map; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { "syslog.supplier.protocol = both", "syslog.supplier.rfc = 5424" }) -public class TcpAndUdp5424Tests extends AbstractSyslogSupplierTests { - - @Test - public void test() throws Exception { - final Flux> messageFlux = syslogSupplier.get(); - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(((Map) message.getPayload()).get("syslog_HOST")).isEqualTo("loggregator"); - } - ) - .assertNext((message) -> { - assertThat(((Map) message.getPayload()).get("syslog_HOST")).isEqualTo("loggregator"); - } - ) - .thenCancel() - .verifyLater(); - - sendTcp("253 " + RFC5424_PACKET); - sendUdp(RFC5424_PACKET); - stepVerifier.verify(); - } - -} diff --git a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Udp3164Tests.java b/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Udp3164Tests.java deleted file mode 100644 index ab1ccf6af..000000000 --- a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Udp3164Tests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.syslog; - -import java.util.Map; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = "syslog.supplier.protocol = udp") -public class Udp3164Tests extends AbstractSyslogSupplierTests { - - @Test - public void test() throws Exception { - final Flux> messageFlux = syslogSupplier.get(); - - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(((Map) message.getPayload()).get("HOST")).isEqualTo("WEBERN"); - } - ) - .thenCancel() - .verifyLater(); - - sendUdp(RFC3164_PACKET); - - stepVerifier.verify(); - - - } - -} diff --git a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Udp5424Tests.java b/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Udp5424Tests.java deleted file mode 100644 index c64cf3071..000000000 --- a/functions/supplier/syslog-supplier/src/test/java/org/springframework/cloud/fn/supplier/syslog/Udp5424Tests.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.syslog; - -import java.util.Map; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.messaging.Message; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -@TestPropertySource(properties = { "syslog.supplier.protocol = udp", "syslog.supplier.rfc = 5424" }) -public class Udp5424Tests extends AbstractSyslogSupplierTests { - - @Test - public void test() throws Exception { - final Flux> messageFlux = syslogSupplier.get(); - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(((Map) message.getPayload()).get("syslog_HOST")).isEqualTo("loggregator"); - } - ) - .thenCancel() - .verifyLater(); - - sendUdp(RFC5424_PACKET); - stepVerifier.verify(); - } -} diff --git a/functions/supplier/tcp-supplier/README.adoc b/functions/supplier/tcp-supplier/README.adoc deleted file mode 100644 index a957d3506..000000000 --- a/functions/supplier/tcp-supplier/README.adoc +++ /dev/null @@ -1,33 +0,0 @@ -# TCP Supplier - -This module provides a TCP supplier that can be reused and composed in other applications. -The `Supplier` uses the `TcpReceivingChannelAdapter` from Spring Integration. -A `tcpSupplier` bean is implemented as a `java.util.function.Supplier`. -This supplier gives you a reactive stream from TCP sources. -The supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and then receive the data. - -## Beans for injection - -You can import the `TcpSupplierConfiguration` in the application and then inject the following bean: `tcpSupplier`. - -You need to inject this as `Supplier>>`. - -You can use `tcpSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `tcp.supplier` and `tcp`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/tcp/TcpSupplierProperties.java[TcpSupplierProperties]. -In addition to this, there is also link:../../common/tcp-common/src/main/java/org/springframework/cloud/fn/common/tcp/TcpConnectionFactoryProperties.java[this set of properties] to consider. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/tcp[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/tcp-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes an TCP Source. \ No newline at end of file diff --git a/functions/supplier/tcp-supplier/pom.xml b/functions/supplier/tcp-supplier/pom.xml deleted file mode 100644 index 0e22d4ae6..000000000 --- a/functions/supplier/tcp-supplier/pom.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - tcp-supplier - tcp-supplier - tcp supplier - - - - org.springframework.cloud.fn - tcp-common - ${project.version} - - - - diff --git a/functions/supplier/tcp-supplier/src/main/java/org/springframework/cloud/fn/supplier/tcp/TcpSupplierConfiguration.java b/functions/supplier/tcp-supplier/src/main/java/org/springframework/cloud/fn/supplier/tcp/TcpSupplierConfiguration.java deleted file mode 100644 index d5370ab0e..000000000 --- a/functions/supplier/tcp-supplier/src/main/java/org/springframework/cloud/fn/supplier/tcp/TcpSupplierConfiguration.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2015-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import java.util.function.Supplier; - -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.tcp.EncoderDecoderFactoryBean; -import org.springframework.cloud.fn.common.tcp.TcpConnectionFactoryProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.ip.IpHeaders; -import org.springframework.integration.ip.config.TcpConnectionFactoryFactoryBean; -import org.springframework.integration.ip.tcp.TcpReceivingChannelAdapter; -import org.springframework.integration.ip.tcp.connection.AbstractConnectionFactory; -import org.springframework.integration.ip.tcp.serializer.AbstractByteArraySerializer; -import org.springframework.messaging.Message; - -/** - * A source module that receives data over TCP. - * - * @author Gary Russell - * @author Christian Tzolov - * @author Soby Chacko - * @author Artem Bilan - */ -@Configuration(proxyBeanMethods = false) -@EnableConfigurationProperties({ TcpSupplierProperties.class, TcpConnectionFactoryProperties.class }) -public class TcpSupplierConfiguration { - - @Bean - public EncoderDecoderFactoryBean tcpSourceDecoder(TcpSupplierProperties properties) { - EncoderDecoderFactoryBean factoryBean = new EncoderDecoderFactoryBean(properties.getDecoder()); - factoryBean.setMaxMessageSize(properties.getBufferSize()); - return factoryBean; - } - - @Bean - public TcpConnectionFactoryFactoryBean tcpSourceConnectionFactory( - TcpConnectionFactoryProperties tcpConnectionProperties, - @Qualifier("tcpSourceDecoder") AbstractByteArraySerializer decoder) { - - TcpConnectionFactoryFactoryBean factoryBean = new TcpConnectionFactoryFactoryBean(); - factoryBean.setType("server"); - factoryBean.setPort(tcpConnectionProperties.getPort()); - factoryBean.setUsingNio(tcpConnectionProperties.isNio()); - factoryBean.setUsingDirectBuffers(tcpConnectionProperties.isUseDirectBuffers()); - factoryBean.setLookupHost(tcpConnectionProperties.isReverseLookup()); - factoryBean.setDeserializer(decoder); - factoryBean.setSoTimeout(tcpConnectionProperties.getSocketTimeout()); - return factoryBean; - } - - @Bean - public TcpReceivingChannelAdapter adapter( - @Qualifier("tcpSourceConnectionFactory") AbstractConnectionFactory connectionFactory) { - - TcpReceivingChannelAdapter adapter = new TcpReceivingChannelAdapter(); - adapter.setConnectionFactory(connectionFactory); - adapter.setAutoStartup(false); - return adapter; - } - - @Bean - public Publisher> tcpSupplierFlow(TcpReceivingChannelAdapter adapter) { - return IntegrationFlow.from(adapter) - .headerFilter(IpHeaders.LOCAL_ADDRESS) - .toReactivePublisher(); - } - - @Bean - public Supplier>> tcpSupplier( - Publisher> tcpSupplierFlow, - TcpReceivingChannelAdapter tcpReceivingChannelAdapter) { - - return () -> Flux.from(tcpSupplierFlow) - .doOnSubscribe(subscription -> tcpReceivingChannelAdapter.start()); - } - -} diff --git a/functions/supplier/tcp-supplier/src/main/java/org/springframework/cloud/fn/supplier/tcp/TcpSupplierProperties.java b/functions/supplier/tcp-supplier/src/main/java/org/springframework/cloud/fn/supplier/tcp/TcpSupplierProperties.java deleted file mode 100644 index d1dafe407..000000000 --- a/functions/supplier/tcp-supplier/src/main/java/org/springframework/cloud/fn/supplier/tcp/TcpSupplierProperties.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import jakarta.validation.constraints.NotNull; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.cloud.fn.common.tcp.Encoding; -import org.springframework.validation.annotation.Validated; - -/** - * Properties for the TCP Source. - * - * @author Gary Russell - * @author Christian Tzolov - * - */ -@ConfigurationProperties("tcp.supplier") -@Validated -public class TcpSupplierProperties { - - /** - * The decoder to use when receiving messages. - */ - private Encoding decoder = Encoding.CRLF; - - /** - * The buffer size used when decoding messages; larger messages will be rejected. - */ - private int bufferSize = 2048; - - @NotNull - public Encoding getDecoder() { - return this.decoder; - } - - public void setDecoder(Encoding decoder) { - this.decoder = decoder; - } - - public int getBufferSize() { - return bufferSize; - } - - public void setBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - } - -} diff --git a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/AbstractTcpSupplierTests.java b/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/AbstractTcpSupplierTests.java deleted file mode 100644 index d3ab09952..000000000 --- a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/AbstractTcpSupplierTests.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2015-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import java.net.Socket; -import java.util.function.Supplier; - -import javax.net.SocketFactory; - -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.integration.ip.tcp.connection.AbstractServerConnectionFactory; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * Tests for TCP Supplier. - * - * @author Gary Russell - * @author Soby Chacko - * @author Artem Bilan - */ -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.NONE, properties = "tcp.port = 0") -@DirtiesContext -public class AbstractTcpSupplierTests { - - @Autowired - Supplier>> tcpSupplier; - - @Autowired - protected AbstractServerConnectionFactory connectionFactory; - - @Autowired - protected TcpSupplierProperties properties; - - /* - * Sends two messages with and asserts the - * payload is received on the other side. - */ - protected void doTest(String prefix, String payload, String suffix) throws Exception { - - final Flux> messageFlux = tcpSupplier.get(); - - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> { - assertThat(message.getPayload()) - .isEqualTo(payload.getBytes()); - } - ) - .assertNext((message) -> { - assertThat(message.getPayload()) - .isEqualTo(payload.getBytes()); - } - ) - .thenCancel() - .verifyLater(); - - - int port = getPort(); - Socket socket = SocketFactory.getDefault().createSocket("localhost", port); - socket.getOutputStream().write((prefix + payload + suffix).getBytes()); - if (prefix.length() == 0 && suffix.length() == 0) { - socket.close(); // RAW - for the others, close AFTER the messages are decoded. - socket = SocketFactory.getDefault().createSocket("localhost", port); - } - - socket.getOutputStream().write((prefix + payload + suffix).getBytes()); - if (prefix.length() == 0 && suffix.length() == 0) { - socket.close(); // RAW - for the others, close AFTER the messages are decoded. - } - socket.close(); - - stepVerifier.verify(); - } - - private int getPort() throws Exception { - int n = 0; - while (n++ < 100 && !this.connectionFactory.isListening()) { - Thread.sleep(100); - } - assertThat(this.connectionFactory.isListening()).isTrue(); - int port = this.connectionFactory.getPort(); - assertThat(port > 0).isTrue(); - return port; - } - - @SpringBootApplication - public static class TcpSupplierTestApplication { - - } -} diff --git a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/CRLFTests.java b/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/CRLFTests.java deleted file mode 100644 index a19543cb8..000000000 --- a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/CRLFTests.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import org.junit.jupiter.api.Test; - -/** - * @author Gary Russell - */ -public class CRLFTests extends AbstractTcpSupplierTests { - - @Test - public void test() throws Exception { - doTest("", "foo", "\r\n"); - } -} diff --git a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/L1Tests.java b/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/L1Tests.java deleted file mode 100644 index 7aec2c436..000000000 --- a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/L1Tests.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = { "tcp.supplier.decoder = L1" }) -public class L1Tests extends AbstractTcpSupplierTests { - - @Test - public void test() throws Exception { - doTest("\u0003", "foo", ""); - } -} diff --git a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/L2Tests.java b/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/L2Tests.java deleted file mode 100644 index f2d7144c2..000000000 --- a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/L2Tests.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = { "tcp.supplier.decoder = L2" }) -public class L2Tests extends AbstractTcpSupplierTests { - - @Test - public void test() throws Exception { - doTest("\u0000\u0003", "foo", ""); - } -} diff --git a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/L4Tests.java b/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/L4Tests.java deleted file mode 100644 index 1d0210a1e..000000000 --- a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/L4Tests.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = { "tcp.supplier.decoder = L4" }) -public class L4Tests extends AbstractTcpSupplierTests { - - @Test - public void test() throws Exception { - doTest("\u0000\u0000\u0000\u0003", "foo", ""); - } -} diff --git a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/LFTests.java b/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/LFTests.java deleted file mode 100644 index cbb898881..000000000 --- a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/LFTests.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = { "tcp.supplier.decoder = LF" }) -public class LFTests extends AbstractTcpSupplierTests { - - @Test - public void test() throws Exception { - doTest("", "foo", "\n"); - } -} diff --git a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/NULLTests.java b/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/NULLTests.java deleted file mode 100644 index 9aac7d4a2..000000000 --- a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/NULLTests.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = { "tcp.supplier.decoder = NULL" }) -public class NULLTests extends AbstractTcpSupplierTests { - - @Test - public void test() throws Exception { - doTest("", "foo", "\u0000"); - } -} diff --git a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/NotNioTests.java b/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/NotNioTests.java deleted file mode 100644 index 2d7e039dc..000000000 --- a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/NotNioTests.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.connection.TcpNetServerConnectionFactory; -import org.springframework.integration.test.util.TestUtils; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Gary Russell - */ -public class NotNioTests extends AbstractTcpSupplierTests { - - @Test - public void test() { - assertThat(this.connectionFactory).isInstanceOf(TcpNetServerConnectionFactory.class); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "lookupHost", Boolean.class)).isFalse(); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "soTimeout")).isEqualTo(120000); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "deserializer.maxMessageSize")).isEqualTo(2048); - } -} diff --git a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/PropertiesPopulatedTests.java b/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/PropertiesPopulatedTests.java deleted file mode 100644 index d3aff90a1..000000000 --- a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/PropertiesPopulatedTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.integration.ip.tcp.connection.TcpNioServerConnectionFactory; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.test.context.TestPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = {"tcp.nio = true", "tcp.reverseLookup = true", - "tcp.useDirectBuffers = true", "tcp.socketTimeout = 123", "tcp.supplier.bufferSize = 5"}) -public class PropertiesPopulatedTests extends AbstractTcpSupplierTests { - - @Test - public void test() { - assertThat(this.connectionFactory).isInstanceOf(TcpNioServerConnectionFactory.class); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "lookupHost", Boolean.class)).isTrue(); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "usingDirectBuffers", Boolean.class)).isTrue(); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "soTimeout")).isEqualTo(123); - assertThat(TestUtils.getPropertyValue(this.connectionFactory, "deserializer.maxMessageSize")).isEqualTo(5); - } - -} diff --git a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/RAWTests.java b/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/RAWTests.java deleted file mode 100644 index 6ab80d622..000000000 --- a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/RAWTests.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = { "tcp.supplier.decoder = RAW" }) -public class RAWTests extends AbstractTcpSupplierTests { - - @Test - public void test() throws Exception { - doTest("", "foo", ""); - } -} diff --git a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/STXETXTests.java b/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/STXETXTests.java deleted file mode 100644 index 1f3111a30..000000000 --- a/functions/supplier/tcp-supplier/src/test/java/org/springframework/cloud/fn/supplier/tcp/STXETXTests.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2015-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.tcp; - -import org.junit.jupiter.api.Test; - -import org.springframework.test.context.TestPropertySource; - -/** - * @author Gary Russell - */ -@TestPropertySource(properties = { "tcp.supplier.decoder = STXETX" }) -public class STXETXTests extends AbstractTcpSupplierTests { - - @Test - public void test() throws Exception { - doTest("\u0002", "foo", "\u0003"); - } -} diff --git a/functions/supplier/time-supplier/README.adoc b/functions/supplier/time-supplier/README.adoc deleted file mode 100644 index 66b2657c4..000000000 --- a/functions/supplier/time-supplier/README.adoc +++ /dev/null @@ -1,31 +0,0 @@ -# Time Supplier - -This module provides a Time supplier that can be reused and composed in other applications. -The `Supplier` uses the `FastDateFormat` from Apache Commons library. -`timeSupplier` is implemented as a `java.util.function.Supplier`. - -## Beans for injection - -You can import the `TimeSupplierConfiguration` in the application and then inject the following bean. - -`timeSupplier` - -You need to inject this as `Supplier`. - -You can use `timeSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it. - -## Configuration Options - -All configuration properties are prefixed with `time`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierProperties.java[TimeSupplierProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/time[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/time-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes an Time Source. \ No newline at end of file diff --git a/functions/supplier/time-supplier/pom.xml b/functions/supplier/time-supplier/pom.xml deleted file mode 100644 index 045884919..000000000 --- a/functions/supplier/time-supplier/pom.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - time-supplier - time-supplier - time supplier - - - - org.apache.commons - commons-lang3 - - - - diff --git a/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/DateFormat.java b/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/DateFormat.java deleted file mode 100644 index 7e94ef584..000000000 --- a/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/DateFormat.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.time; - -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.text.SimpleDateFormat; - -import jakarta.validation.Constraint; -import jakarta.validation.ConstraintValidator; -import jakarta.validation.ConstraintValidatorContext; -import jakarta.validation.Payload; - -/** - * The annotated String must be a valid {@link java.text.SimpleDateFormat} pattern. - * - * @author Eric Bottard - * @author Soby Chacko - */ -@Target({ElementType.METHOD, ElementType.FIELD, ElementType.ANNOTATION_TYPE, ElementType.CONSTRUCTOR, - ElementType.PARAMETER}) -@Retention(RetentionPolicy.RUNTIME) -@Documented -@Constraint(validatedBy = {DateFormat.DateFormatValidator.class}) -public @interface DateFormat { - - /** - * Default message for validation. - */ - String DEFAULT_MESSAGE = ""; - - String message() default DEFAULT_MESSAGE; - - Class[] groups() default {}; - - Class[] payload() default {}; - - class DateFormatValidator implements ConstraintValidator { - - private String message; - - @Override - public void initialize(DateFormat constraintAnnotation) { - this.message = constraintAnnotation.message(); - } - - @Override - public boolean isValid(CharSequence value, ConstraintValidatorContext context) { - if (value == null) { - return true; - } - try { - new SimpleDateFormat(value.toString()); - } - catch (IllegalArgumentException e) { - if (DEFAULT_MESSAGE.equals(this.message)) { - context.disableDefaultConstraintViolation(); - context.buildConstraintViolationWithTemplate(e.getMessage()).addConstraintViolation(); - } - return false; - } - return true; - } - - } - -} diff --git a/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierConfiguration.java b/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierConfiguration.java deleted file mode 100644 index b58b8f1ba..000000000 --- a/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierConfiguration.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.time; - -import java.util.Date; -import java.util.function.Supplier; - -import org.apache.commons.lang3.time.FastDateFormat; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * @author Soby Chacko - */ -@Configuration -@EnableConfigurationProperties(TimeSupplierProperties.class) -public class TimeSupplierConfiguration { - - @Bean - public Supplier timeSupplier(TimeSupplierProperties timeSupplierProperties) { - FastDateFormat fastDateFormat = FastDateFormat.getInstance(timeSupplierProperties.getDateFormat()); - return () -> fastDateFormat.format(new Date()); - } - -} diff --git a/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierProperties.java b/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierProperties.java deleted file mode 100644 index 5c6df97c5..000000000 --- a/functions/supplier/time-supplier/src/main/java/org/springframework/cloud/fn/supplier/time/TimeSupplierProperties.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.time; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * @author Soby Chacko - */ -@ConfigurationProperties("time") -@Validated -public class TimeSupplierProperties { - - /** - * Format for the date value. - */ - private String dateFormat = "MM/dd/yy HH:mm:ss"; - - @DateFormat - public String getDateFormat() { - return this.dateFormat; - } - - public void setDateFormat(String dateFormat) { - this.dateFormat = dateFormat; - } - -} diff --git a/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/SimpleTimeSupplierTests.java b/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/SimpleTimeSupplierTests.java deleted file mode 100644 index 6777c8ad8..000000000 --- a/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/SimpleTimeSupplierTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.time; - -import java.text.SimpleDateFormat; -import java.util.Date; - -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; - -/** - * @author Soby Chacko - * @author Artem Bilan - */ -public class SimpleTimeSupplierTests extends TimeSupplierApplicationTests { - - @Test - public void testTimeSupplier() { - final String time = timeSupplier.get(); - SimpleDateFormat dateFormat = new SimpleDateFormat(new TimeSupplierProperties().getDateFormat()); - assertThatCode(() -> { - Date date = dateFormat.parse(time); - assertThat(date).isNotNull(); - }).doesNotThrowAnyException(); - } - -} diff --git a/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/TimeSupplierApplicationTests.java b/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/TimeSupplierApplicationTests.java deleted file mode 100644 index 2812f4925..000000000 --- a/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/TimeSupplierApplicationTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.time; - -import java.util.function.Supplier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; - -/** - * @author Soby Chacko - * @author Artem Bilan - */ -@SpringBootTest -public abstract class TimeSupplierApplicationTests { - - @Autowired - Supplier timeSupplier; - - @Autowired - TimeSupplierProperties timeSupplierProperties; - - protected abstract void testTimeSupplier(); - - @SpringBootApplication - static class TimeSupplierTestApplication { - } -} diff --git a/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/VariationToSimpleTests.java b/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/VariationToSimpleTests.java deleted file mode 100644 index c33edb351..000000000 --- a/functions/supplier/time-supplier/src/test/java/org/springframework/cloud/fn/supplier/time/VariationToSimpleTests.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.time; - -import java.text.SimpleDateFormat; -import java.util.Date; - -import org.junit.jupiter.api.Test; - -import org.springframework.boot.test.context.SpringBootTest; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; -import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException; - -/** - * @author Soby Chacko - * @author Artem Bilan - */ -@SpringBootTest({"time.dateFormat=MMddyyyy HH:mm:ss"}) -public class VariationToSimpleTests extends TimeSupplierApplicationTests { - - @Test - public void testTimeSupplier() { - final String time = timeSupplier.get(); - SimpleDateFormat dateFormat = new SimpleDateFormat(timeSupplierProperties.getDateFormat()); - assertThatCode(() -> { - Date date = dateFormat.parse(time); - assertThat(date).isNotNull(); - }).doesNotThrowAnyException(); - } - - @Test - public void testInvalidDateFormat() { - TimeSupplierProperties timeSupplierProperties = new TimeSupplierProperties(); - timeSupplierProperties.setDateFormat("AA/dd/yyyy HH:mm:ss"); - assertThatIllegalArgumentException().isThrownBy(() -> new SimpleDateFormat(timeSupplierProperties.getDateFormat())); - } - -} diff --git a/functions/supplier/twitter-supplier/README.adoc b/functions/supplier/twitter-supplier/README.adoc deleted file mode 100644 index 9c7955869..000000000 --- a/functions/supplier/twitter-supplier/README.adoc +++ /dev/null @@ -1,143 +0,0 @@ -# Twitter Suppliers - -This module provides a Twitter Status, Message, Friendship suppliers that can be reused and composed in various applications. - -`java.util.function.Supplier` - - -## 1. Twitter Status Search - -The Twitter's https://developer.twitter.com/en/docs/tweets/search/api-reference/get-search-tweets.html[Standard search API] (search/tweets) allows simple queries against the indices of recent or popular Tweets. This `Source` provides continuous searches against a sampling of recent Tweets published in the past 7 days. Part of the 'public' set of APIs. - -Returns a collection of relevant Tweets matching a specified query. - -### 1.1 Beans for injection - -You can import the `TwitterSearchSupplierConfiguration` in the application and then inject the following bean. - -`twitterSearchSupplier` - -You need to inject this as `Supplier>`. - -You can use `twitterSearchSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it. - -### 1.2 Configuration Options - -The configuration properties prefixed with `twitter.search`. -There are also properties that need to be used with the prefix `twitter.connection`. - -The `spring.cloud.stream.poller` properties control the interval between consecutive search requests. Rate Limit - 180 requests per 30 min. window (e.g. ~6 r/m, ~ 1 req / 10 sec.) - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/twitter/search/stream/TwitterSearchSupplierProperties.java[TwitterSearchSupplierProperties]. -See link:src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionProperties.java[TwitterConnectionProperties] and https://github.com/spring-cloud/spring-cloud-stream/blob/master/spring-cloud-stream/src/main/java/org/springframework/cloud/stream/config/DefaultPollerProperties.java[DefaultPollerProperties] - -### 1.3 Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/twitter-search-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a Twitter Search Source. - -## 2. Twitter Status Real-time Retrieval - -Provides real-time, Tweet streaming based on the https://developer.twitter.com/en/docs/tweets/filter-realtime/api-reference/post-statuses-filter.html[Filter] and https://developer.twitter.com/en/docs/tweets/sample-realtime/overview/GET_statuse_sample[Sample] APIs. -The `Filter API` flavor returns public statuses that match one or more filter predicates. -The `Sample API` flavor returns a small random sample of all public statuses. - -This supplier gives you a reactive stream of tweets from the configured connection as the supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and receive the data. - -The default access level allows up to 400 track keywords, 5,000 follow user Ids and 25 0.1-360 degree location boxes. - -### 2.1 Beans for injection - -You can import the `TwitterStreamSupplierConfiguration` in the application and then inject the following bean. - -`twitterStreamSupplier` - -You need to inject this as `Supplier>>`. - -You can use `twitterStreamSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -### 2.2 Configuration Options - -All configuration properties are prefixed with `twitter.stream`. -There are also properties that need to be used with the prefix `twitter.connection`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/twitter/status/stream/TwitterStreamSupplierProperties.java[TwitterStreamSupplierProperties]. -See link:src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionProperties.java[TwitterConnectionProperties] also. - - -### 2.3 Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/twitter-stream-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a Twitter Stream Source. - -## 3. Twitter Direct Message Supplier - -Repeatedly retrieves the direct messages (both sent and received) within the last 30 days, sorted in reverse-chronological order. -The relieved messages are cached (in a `MetadataStore` cache) to prevent duplications. -By default an in-memory `SimpleMetadataStore` is used. - -The `twitter.message.source.count` controls the number or returned messages. - -The `spring.cloud.stream.poller` properties control the message poll interval. -Must be aligned with used APIs rate limit - -### 3.1 Beans for injection - -You can import the `TwitterMessageSupplierConfiguration` in the application and then inject the following bean. - -`twitterMessageSupplier` - -You need to inject this as `Supplier>`. - -You can use `twitterMessageSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it. - -### 3.2 Configuration Options - -The configuration properties prefixed with `twitter.search`. -There are also properties that need to be used with the prefix `twitter.connection`. - -The `spring.cloud.stream.poller` properties control the interval between consecutive search requests. Rate Limit - 180 requests per 30 min. window (e.g. ~6 r/m, ~ 1 req / 10 sec.) - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/twitter/message/TwitterMessageSupplierProperties.java[TwitterMessageSupplierProperties]. -See link:src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionProperties.java[TwitterConnectionProperties] and https://github.com/spring-cloud/spring-cloud-stream/blob/master/spring-cloud-stream/src/main/java/org/springframework/cloud/stream/config/DefaultPollerProperties.java[DefaultPollerProperties] - -### 3.3 Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/twitter-message-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a Twitter Message Source. - -## 4. Twitter Friendships Supplier - -Returns a cursored collection of user objects either for the https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-followers-list[users following the specified user] (`followers`) or for https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friends-list[every user the specified user is following] (`friends`). - -The `twitter.friendships.source.type` property allow to select between both types. - -TIP: Rate limit: 15 requests per 30 min window. ~ 1 req/ 2 min - -### 4.1 Beans for injection - -You can import the `TwitterFriendshipsSupplierConfiguration` in the application and then inject one the following beans. - -- `followersSupplier` (only if `twitter.friendships.source.type=followers` ) - retrieves the https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-followers-list[users following the specified user] (`followers`) - -- `friendsSupplier` (only if `twitter.friendships.source.type=friends`) - retrieves the for https://developer.twitter.com/en/docs/accounts-and-users/follow-search-get-users/api-reference/get-friends-list[every user the specified user is following] (`friends`). - -Both suppliers expose `Supplier>`. - -- `deduplicatedFriendsJsonSupplier` - retrieves either the followers, or the friends collection (controlled by the `twitter.friendships.source.type`) property, . -encoded as JSON `Message` payloads. You need to inject this as `Supplier>`. - - -### 4.2 Configuration Options - -The configuration properties prefixed with `twitter.friendships.source`. -There are also properties that need to be used with the prefix `twitter.connection`. - -The `spring.cloud.stream.poller` properties control the interval between consecutive search requests. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/twitter/friendships/TwitterFriendshipsSupplierProperties.java[TwitterFriendshipsSupplierProperties]. -See link:src/main/java/org/springframework/cloud/fn/common/twitter/TwitterConnectionProperties.java[TwitterConnectionProperties] and https://github.com/spring-cloud/spring-cloud-stream/blob/master/spring-cloud-stream/src/main/java/org/springframework/cloud/stream/config/DefaultPollerProperties.java[DefaultPollerProperties] diff --git a/functions/supplier/twitter-supplier/pom.xml b/functions/supplier/twitter-supplier/pom.xml deleted file mode 100644 index 4d33b853e..000000000 --- a/functions/supplier/twitter-supplier/pom.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - twitter-supplier - twitter-supplier - twitter suppliers - - - - org.springframework.cloud.fn - twitter-common - ${project.version} - - - org.springframework.integration - spring-integration-jms - - - jakarta.jms - jakarta.jms-api - provided - - - org.mock-server - mockserver-netty - ${mockserver.version} - test - - - org.mock-server - mockserver-client-java - ${mockserver.version} - test - - - - diff --git a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/friendships/TwitterFriendshipsSupplierConfiguration.java b/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/friendships/TwitterFriendshipsSupplierConfiguration.java deleted file mode 100644 index 58587056b..000000000 --- a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/friendships/TwitterFriendshipsSupplierConfiguration.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.twitter.friendships; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; -import java.util.function.Supplier; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import twitter4j.PagableResponseList; -import twitter4j.Twitter; -import twitter4j.TwitterException; -import twitter4j.User; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.twitter.Cursor; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.integration.metadata.MetadataStore; -import org.springframework.integration.metadata.SimpleMetadataStore; -import org.springframework.messaging.Message; - -/** - * - * @author Christian Tzolov - */ -@Configuration -@EnableConfigurationProperties(TwitterFriendshipsSupplierProperties.class) -@Import(TwitterConnectionConfiguration.class) -public class TwitterFriendshipsSupplierConfiguration { - - private static final Log logger = LogFactory.getLog(TwitterFriendshipsSupplierConfiguration.class); - - @Bean - @ConditionalOnMissingBean - public MetadataStore metadataStore() { - return new SimpleMetadataStore(); - } - - @Bean - @ConditionalOnMissingBean - public Cursor cursor() { - return new Cursor(); - } - - - @Bean - @ConditionalOnProperty(name = "twitter.friendships.source.type", havingValue = "followers") - public Supplier> followersSupplier(TwitterFriendshipsSupplierProperties properties, - Twitter twitter, Cursor cursorState) { - return () -> { - try { - PagableResponseList users; - if (properties.getUserId() != null) { - users = twitter.getFollowersList(properties.getUserId(), cursorState.getCursor(), - properties.getCount(), properties.isSkipStatus(), properties.isIncludeUserEntities()); - } - else { // by ScreenName - users = twitter.getFollowersList(properties.getScreenName(), cursorState.getCursor(), - properties.getCount(), properties.isSkipStatus(), properties.isIncludeUserEntities()); - } - - if (users != null) { - cursorState.updateCursor(users.getNextCursor()); - return users; - } - - logger.error(String.format("NULL users response for properties: %s and cursor: %s!", properties, cursorState)); - cursorState.updateCursor(-1); - } - catch (TwitterException e) { - logger.error("Twitter API error:", e); - } - - return new ArrayList<>(); - }; - } - - @Bean - @ConditionalOnProperty(name = "twitter.friendships.source.type", havingValue = "friends") - public Supplier> friendsSupplier(TwitterFriendshipsSupplierProperties properties, - Twitter twitter, Cursor cursorState) { - return () -> { - try { - PagableResponseList users; - if (properties.getUserId() != null) { - users = twitter.getFriendsList(properties.getUserId(), cursorState.getCursor(), - properties.getCount(), properties.isSkipStatus(), properties.isIncludeUserEntities()); - } - else { // by ScreenName - users = twitter.getFriendsList(properties.getScreenName(), cursorState.getCursor(), - properties.getCount(), properties.isSkipStatus(), properties.isIncludeUserEntities()); - } - - if (users != null) { - cursorState.updateCursor(users.getNextCursor()); - return users; - } - - logger.error(String.format("NULL users response for properties: %s and cursor: %s!", properties, cursorState)); - cursorState.updateCursor(-1); - } - catch (TwitterException e) { - logger.error("Twitter API error:", e); - } - - return new ArrayList<>(); - }; - } - - @Bean - public Function, List> userDeduplicate(MetadataStore metadataStore) { - return users -> { - List uniqueUsers = new ArrayList<>(); - for (User user : users) { - if (metadataStore.get(user.getId() + "") == null) { - metadataStore.put(user.getId() + "", user.getName()); - uniqueUsers.add(user); - } - } - return uniqueUsers; - }; - } - - @Bean - public Supplier> deduplicatedFriendsJsonSupplier(Function, List> userDeduplication, - Supplier> userRetriever, Function> managedJson) { - return () -> userDeduplication.andThen(managedJson).apply(userRetriever.get()); - } -} diff --git a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/friendships/TwitterFriendshipsSupplierProperties.java b/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/friendships/TwitterFriendshipsSupplierProperties.java deleted file mode 100644 index 4031e5c33..000000000 --- a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/friendships/TwitterFriendshipsSupplierProperties.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.twitter.friendships; - -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.Max; -import jakarta.validation.constraints.NotNull; -import jakarta.validation.constraints.Positive; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("twitter.friendships.source") -@Validated -public class TwitterFriendshipsSupplierProperties { - - public enum FriendshipsRequestType { - /** Friendship query types. */ - followers, friends - } - - /** - * Selects between followers or friends APIs. - */ - @NotNull - private TwitterFriendshipsSupplierProperties.FriendshipsRequestType type = FriendshipsRequestType.followers; - - /** - * The screen name of the user for whom to return results. - */ - private String screenName; - - /** - * The ID of the user for whom to return results. - */ - private Long userId; - - /** - * The number of users to return per page, up to a maximum of 200. Defaults to 20. - */ - @Positive - @Max(200) - private int count = 200; - - /** - * When set to true, statuses will not be included in the returned user objects. - */ - private boolean skipStatus = false; - - /** - * The user object entities node will be included when set to false. - */ - private boolean includeUserEntities = true; - - /** - * API request poll interval in milliseconds. Must be aligned with used APIs rate limits (~ 1 req/ 2 min). - */ - private int pollInterval = 121000; - - public FriendshipsRequestType getType() { - return type; - } - - public void setType(FriendshipsRequestType type) { - this.type = type; - } - - public String getScreenName() { - return screenName; - } - - public void setScreenName(String screenName) { - this.screenName = screenName; - } - - public Long getUserId() { - return userId; - } - - public void setUserId(Long userId) { - this.userId = userId; - } - - public int getCount() { - return count; - } - - public void setCount(int count) { - this.count = count; - } - - public boolean isSkipStatus() { - return skipStatus; - } - - public void setSkipStatus(boolean skipStatus) { - this.skipStatus = skipStatus; - } - - public boolean isIncludeUserEntities() { - return includeUserEntities; - } - - public void setIncludeUserEntities(boolean includeUserEntities) { - this.includeUserEntities = includeUserEntities; - } - - public int getPollInterval() { - return pollInterval; - } - - public void setPollInterval(int pollInterval) { - this.pollInterval = pollInterval; - } - - @AssertTrue(message = "Either userId or screenName must be provided") - public boolean isUserProvided() { - return this.userId != null || this.screenName != null; - } - - @Override - public String toString() { - return "TwitterFriendshipsSourceProperties{" + - "type=" + type + - ", screenName='" + screenName + '\'' + - ", userId=" + userId + - ", count=" + count + - ", skipStatus=" + skipStatus + - ", includeUserEntities=" + includeUserEntities + - ", pollInterval=" + pollInterval + - '}'; - } -} diff --git a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/message/TwitterMessageSupplierConfiguration.java b/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/message/TwitterMessageSupplierConfiguration.java deleted file mode 100644 index cc921f004..000000000 --- a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/message/TwitterMessageSupplierConfiguration.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.twitter.message; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Function; -import java.util.function.Supplier; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import twitter4j.DirectMessage; -import twitter4j.DirectMessageList; -import twitter4j.Twitter; -import twitter4j.TwitterException; - -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.integration.metadata.MetadataStore; -import org.springframework.integration.metadata.SimpleMetadataStore; -import org.springframework.messaging.Message; - -/** - * - * @author Christian Tzolov - */ -@EnableConfigurationProperties({ TwitterMessageSupplierProperties.class }) -@Import(TwitterConnectionConfiguration.class) -public class TwitterMessageSupplierConfiguration { - - private static final Log logger = LogFactory.getLog(TwitterMessageSupplierConfiguration.class); - - @Bean - @ConditionalOnMissingBean - public MetadataStore metadataStore() { - return new SimpleMetadataStore(); - } - - @Bean - @ConditionalOnMissingBean - public MessageCursor cursor() { - return new MessageCursor(); - } - - @Bean - public Supplier> directMessagesSupplier(TwitterMessageSupplierProperties properties, - Twitter twitter, MessageCursor cursorState) { - return () -> { - try { - String cs = cursorState.getCursor(); - DirectMessageList messages = (cursorState.getCursor() == null) ? - twitter.getDirectMessages(properties.getCount()) : - twitter.getDirectMessages(properties.getCount(), cursorState.getCursor()); - - if (messages != null) { - cursorState.updateCursor(messages.getNextCursor()); - return messages; - } - - logger.error(String.format("NULL messages response for properties: %s and cursor: %s!", properties, cursorState)); - cursorState.updateCursor(null); - } - catch (TwitterException e) { - logger.error("Twitter API error:", e); - } - - return new ArrayList<>(); - }; - } - - @Bean - public Function, List> messageDeduplicate(MetadataStore metadataStore) { - return messages -> { - List uniqueMessages = new ArrayList<>(); - for (DirectMessage message : messages) { - if (metadataStore.get(message.getId() + "") == null) { - metadataStore.put(message.getId() + "", message.getCreatedAt() + ""); - uniqueMessages.add(message); - } - } - return uniqueMessages; - }; - } - - @Bean - public Supplier> twitterMessageSupplier(Function, List> messageDeduplicate, - Function> managedJson, Supplier> directMessagesSupplier) { - return () -> messageDeduplicate.andThen(managedJson).apply(directMessagesSupplier.get()); - } - - public static class MessageCursor { - private String cursor = null; - - public String getCursor() { - return cursor; - } - - public void updateCursor(String newCursor) { - this.cursor = newCursor; - } - - @Override - public String toString() { - return "Cursor{" + - "cursor=" + cursor + - '}'; - } - } -} diff --git a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/message/TwitterMessageSupplierProperties.java b/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/message/TwitterMessageSupplierProperties.java deleted file mode 100644 index dedee2fd1..000000000 --- a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/message/TwitterMessageSupplierProperties.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.twitter.message; - -import jakarta.validation.constraints.Max; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("twitter.message.source") -@Validated -public class TwitterMessageSupplierProperties { - - /** - * Max number of events to be returned. 20 default. 50 max. - */ - @Max(50) - private int count = 20; - - public int getCount() { - return count; - } - - public void setCount(int count) { - this.count = count; - } -} diff --git a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/search/SearchPagination.java b/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/search/SearchPagination.java deleted file mode 100644 index c13a15a24..000000000 --- a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/search/SearchPagination.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.twitter.status.search; - -import java.util.List; - -import twitter4j.Status; - -import org.springframework.util.Assert; - -/** - * Searched tweets are ordered from top to bottom by their IDs. The higher the ID, more recent the tweet is. - * - * The search goes backwards - from most recent to the oldest tweets and it uses the sinceId and the maxId to retrieve - * only tweets with IDs in the [sinceId, maxId) range. The -1 stands for unbounded sinceId or maxId. - * - * The first `pageCount` number requests are performed backwards, leaving the bottom boundary (sinceId) unbounded and - * adjusting the upper boundary (maxId) to the lowest tweet ID received. This ensures that no already processed - * tweets are returned. - * - * The pageCounter is used the to count the number of pages retrieved in the pageCount range. Is starts from pageCount - * and goes backward until 0. On 0 the pageCounter is reset back to pageCount. - * - * After performing pageCount number requests (e.g. pageCounter = 0), we start new iteration of searches from the top, - * most recent tweets but now the bottom boundary (sinceId) is adjusted to the max ID received so far. That means that - * only the newly added tweets will be processed - * - * Search pagination with max_id and since_id: https://developer.twitter.com/en/docs/tweets/timelines/guides/working-with-timelines.html - * - * @author Christian Tzolov - */ -public class SearchPagination { - - /** - * Wildcard used as refer as infinity. - */ - public static final long UNBOUNDED = -1; - - /** - * Number of pages to search in history before start form the top again. - * - * Note that the search goes backwards - from most recent to the oldest tweets. - * (eg. maxId == To max ID , sinceId == From min ID) - */ - private final int pageCount; - - /** - * Min tweet ID (e.g. from ID) to be included in the search result - */ - private long sinceId; - - /** - * Keep the max tweet ID in the last pageCount search requests. - */ - private long pageMaxId; - - /** - * Max tweet ID (e.g. To (ID - 1) )to be included in the search result - */ - private long maxId; - - /** - * Current page. Goes backwards from (pageCount-1) to 0. - */ - private int pageCounter; - - /** - * When set it. - */ - boolean searchBackwardsUntilEmptyResponse = false; - - public SearchPagination(int pageCount, boolean searchBackwardsUntilEmptyResponse) { - - Assert.isTrue(pageCount > 0, "At least one page needs to be set but was: " + pageCount); - - this.searchBackwardsUntilEmptyResponse = searchBackwardsUntilEmptyResponse; - this.pageCount = pageCount; - this.sinceId = UNBOUNDED; // == From ID - this.pageMaxId = UNBOUNDED; // == From ID - this.maxId = UNBOUNDED; // == To (ID - 1) - this.pageCounter = pageCount - 1; - } - - public long getSinceId() { - return sinceId; - } - - public long getMaxId() { - return maxId; - } - - public long getPageMaxId() { - return pageMaxId; - } - - public int getPageCounter() { - return pageCounter; - } - - public void update(List tweets) { - - tweets.stream().mapToLong(t -> t.getId()).min() - .ifPresent(tweetsMinId -> { - this.maxId = tweetsMinId - 1; - }); - - tweets.stream().mapToLong(t -> t.getId()).max() - .ifPresent(tweetsMaxId -> { - Assert.isTrue(this.sinceId <= tweetsMaxId, - String.format("MAX_ID (%s) must be bigger then current SINCE_ID(%s)", - tweetsMaxId, this.sinceId)); - this.pageMaxId = Math.max(this.pageMaxId, tweetsMaxId); - }); - - this.countDown(tweets.size()); - } - - private void countDown(int responseSize) { - - if (this.sinceId == UNBOUNDED) { // == first pass before reset - if (this.pageCounter <= 0) { - this.restartSearchFromMostRecent(); - } - } - else { - if (this.searchBackwardsUntilEmptyResponse) { - if (responseSize == 0) { - this.restartSearchFromMostRecent(); - } - } - else if (this.pageCounter <= 0) { - this.restartSearchFromMostRecent(); - } - } - - this.pageCounter--; - } - - private void restartSearchFromMostRecent() { - this.pageCounter = this.pageCount; - - this.sinceId = Math.max(this.sinceId, Math.max(this.maxId + 1, this.pageMaxId)); - - this.maxId = UNBOUNDED; - this.pageMaxId = UNBOUNDED; - } - - public String status() { - return String.format("MaxId: %s, SinceId: %s, Page Counter# %s, pageMaxId: %s", - this.maxId, this.sinceId, this.pageCounter, this.pageMaxId); - } -} diff --git a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/search/TwitterSearchSupplierConfiguration.java b/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/search/TwitterSearchSupplierConfiguration.java deleted file mode 100644 index 8b04df48f..000000000 --- a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/search/TwitterSearchSupplierConfiguration.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright 2020-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.twitter.status.search; - -import java.util.List; -import java.util.function.Function; -import java.util.function.Supplier; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import twitter4j.GeoLocation; -import twitter4j.Query; -import twitter4j.QueryResult; -import twitter4j.Status; -import twitter4j.Twitter; -import twitter4j.TwitterException; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.messaging.Message; -import org.springframework.util.Assert; -import org.springframework.util.StringUtils; - -/** - * Search pagination with max_id and since_id: https://developer.twitter.com/en/docs/tweets/timelines/guides/working-with-timelines.html . - * - * @author Christian Tzolov - * @author Chris Bono - */ -@EnableConfigurationProperties({ TwitterSearchSupplierProperties.class }) -@Import(TwitterConnectionConfiguration.class) -public class TwitterSearchSupplierConfiguration { - - private static final Log logger = LogFactory.getLog(TwitterSearchSupplierConfiguration.class); - - @Autowired - private TwitterSearchSupplierProperties searchProperties; - - @Autowired - private Twitter twitter; - - @Autowired - private Function> json; - - @Bean - public SearchPagination searchPage() { - return new SearchPagination( - this.searchProperties.getPage(), - this.searchProperties.isRestartFromMostRecentOnEmptyResponse()); - } - - - @Bean - public Supplier> twitterSearchSupplier(SearchPagination searchPage) { - return () -> { - try { - Query query = toQuery(this.searchProperties, searchPage); - - QueryResult result = this.twitter.search(query); - - List tweets = result.getTweets(); - - logger.info(String.format("%s, size: %s", searchPage.status(), tweets.size())); - - searchPage.update(tweets); - - return this.json.apply(tweets); - } - catch (TwitterException e) { - logger.error("Twitter error", e); - } - - return null; - }; - } - - private Query toQuery(TwitterSearchSupplierProperties searchProperties, SearchPagination pagination) { - - Query query = new Query(); - if (searchProperties.getCount() > 0) { - query.count(searchProperties.getCount()); - } - if (StringUtils.hasText(searchProperties.getQuery())) { - query.setQuery(searchProperties.getQuery()); - } - if (StringUtils.hasText(searchProperties.getLang())) { - query.setLang(searchProperties.getLang()); - } - if (StringUtils.hasText(searchProperties.getSince())) { - query.setSince(searchProperties.getSince()); - } - if (searchProperties.getGeocode().isValid()) { - query.setGeoCode( - new GeoLocation( - searchProperties.getGeocode().getLatitude(), - searchProperties.getGeocode().getLongitude()), - searchProperties.getGeocode().getRadius(), - Query.KILOMETERS); - } - - if (searchProperties.getResultType() != Query.ResultType.mixed) { - query.setResultType(searchProperties.getResultType()); - } - - if (pagination.getSinceId() > 0) { - query.setSinceId(pagination.getSinceId()); - } - if (pagination.getMaxId() > 0) { - query.setMaxId(pagination.getMaxId()); - - Assert.isTrue(pagination.getMaxId() >= (pagination.getSinceId() - 1), - String.format("For non empty MAX_ID, The MAX_ID (%s) must always be bigger than [SINCE_ID -1](%s)", - pagination.getMaxId(), (pagination.getSinceId() - 1))); - } - - return query; - } -} diff --git a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/search/TwitterSearchSupplierProperties.java b/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/search/TwitterSearchSupplierProperties.java deleted file mode 100644 index 2cc9d7266..000000000 --- a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/search/TwitterSearchSupplierProperties.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.twitter.status.search; - -import jakarta.validation.constraints.Max; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; -import jakarta.validation.constraints.Pattern; -import jakarta.validation.constraints.Positive; -import twitter4j.Query; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("twitter.search") -@Validated -public class TwitterSearchSupplierProperties { - - /** - * Search tweets by search query string. - */ - @NotNull - @NotEmpty - private String query; - - /** - * Number of pages (e.g. requests) to search backwards (from most recent to the oldest tweets) before start - * the search from the most recent tweets again. - * The total amount of tweets searched backwards is (page * count) - */ - @Positive - private int page = 3; - - /** - * Number of tweets to return per page (e.g. per single request), up to a max of 100. - */ - @Positive - @Max(100) - private int count = 100; - - /** - * Restricts searched tweets to the given language, given by an http://en.wikipedia.org/wiki/ISO_639-1 . - */ - private String lang = null; - - /** - * If specified, returns tweets with since the given date. Date should be formatted as YYYY-MM-DD. - */ - @Pattern(regexp = "^\\d{4}-\\d{2}-\\d{2}$") - private String since = null; - - /** - * If specified, returns tweets by users located within a given radius (in Km) of the given latitude/longitude, - * where the user's location is taken from their Twitter profile. - * Should be formatted as - */ - private Geocode geocode = new Geocode(); - - /** - * Specifies what type of search results you would prefer to receive. - * The current default is "mixed." Valid values include: - * mixed : Include both popular and real time results in the response. - * recent : return only the most recent results in the response - * popular : return only the most popular results in the response - */ - @NotNull - private Query.ResultType resultType = Query.ResultType.mixed; - - /** - * Restart search from the most recent tweets on empty response. - * Applied only after the first restart (e.g. when since_id != UNBOUNDED) - */ - private boolean restartFromMostRecentOnEmptyResponse = false; - - public String getQuery() { - return query; - } - - public void setQuery(String query) { - this.query = query; - } - - public String getLang() { - return lang; - } - - public void setLang(String lang) { - this.lang = lang; - } - - public int getPage() { - return page; - } - - public void setPage(int page) { - this.page = page; - } - - public int getCount() { - return count; - } - - public void setCount(int count) { - this.count = count; - } - - public String getSince() { - return since; - } - - public void setSince(String since) { - this.since = since; - } - - public Geocode getGeocode() { - return geocode; - } - - public Query.ResultType getResultType() { - return resultType; - } - - public void setResultType(Query.ResultType resultType) { - this.resultType = resultType; - } - - public boolean isRestartFromMostRecentOnEmptyResponse() { - return restartFromMostRecentOnEmptyResponse; - } - - public void setRestartFromMostRecentOnEmptyResponse(boolean restartFromMostRecentOnEmptyResponse) { - this.restartFromMostRecentOnEmptyResponse = restartFromMostRecentOnEmptyResponse; - } - - public static class Geocode { - - /** - * User's latitude. - */ - private double latitude = -1; - - /** - * User's longitude. - */ - private double longitude = -1; - - /** - * Radius (in kilometers) around the (latitude, longitude) point. - */ - private double radius = -1; - - public double getLatitude() { - return latitude; - } - - public void setLatitude(double latitude) { - this.latitude = latitude; - } - - public double getLongitude() { - return longitude; - } - - public void setLongitude(double longitude) { - this.longitude = longitude; - } - - public double getRadius() { - return radius; - } - - public void setRadius(double radius) { - this.radius = radius; - } - - public boolean isValid() { - return this.radius > 0; - } - } -} diff --git a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/stream/TwitterStreamSupplierConfiguration.java b/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/stream/TwitterStreamSupplierConfiguration.java deleted file mode 100644 index 9f9ee2399..000000000 --- a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/stream/TwitterStreamSupplierConfiguration.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2020-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.twitter.status.stream; - -import java.util.function.Supplier; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import reactor.core.publisher.Flux; -import twitter4j.StallWarning; -import twitter4j.Status; -import twitter4j.StatusDeletionNotice; -import twitter4j.StatusListener; -import twitter4j.TwitterStream; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionConfiguration; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.integration.channel.FluxMessageChannel; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHeaders; -import org.springframework.messaging.support.MessageBuilder; -import org.springframework.util.MimeTypeUtils; - -/** - * - * @author Christian Tzolov - */ - -@EnableConfigurationProperties({ TwitterStreamSupplierProperties.class, TwitterConnectionProperties.class }) -@Import(TwitterConnectionConfiguration.class) -public class TwitterStreamSupplierConfiguration { - - private static final Log logger = LogFactory.getLog(TwitterStreamSupplierConfiguration.class); - - @Bean - public FluxMessageChannel twitterStatusInputChannel() { - return new FluxMessageChannel(); - } - - @Bean - public StatusListener twitterStatusListener(FluxMessageChannel twitterStatusInputChannel, TwitterStream twitterStream, - ObjectMapper objectMapper) { - - StatusListener statusListener = new StatusListener() { - - @Override - public void onException(Exception e) { - logger.error("Status Error: ", e); - throw new RuntimeException("Status Error: ", e); - } - - @Override - public void onDeletionNotice(StatusDeletionNotice arg) { - logger.info("StatusDeletionNotice: " + arg); - } - - @Override - public void onScrubGeo(long userId, long upToStatusId) { - logger.info("onScrubGeo: " + userId + ", " + upToStatusId); - } - - @Override - public void onStallWarning(StallWarning warning) { - logger.warn("Stall Warning: " + warning); - throw new RuntimeException("Stall Warning: " + warning); - } - - @Override - public void onStatus(Status status) { - - try { - String json = objectMapper.writeValueAsString(status); - Message message = MessageBuilder.withPayload(json.getBytes()) - .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON_VALUE) - .build(); - twitterStatusInputChannel.send(message); - } - catch (JsonProcessingException e) { - logger.error("Status to JSON conversion error!", e); - throw new RuntimeException("Status to JSON conversion error!", e); - } - } - - @Override - public void onTrackLimitationNotice(int numberOfLimitedStatuses) { - logger.warn("Track Limitation Notice: " + numberOfLimitedStatuses); - } - }; - - twitterStream.addListener(statusListener); - - return statusListener; - } - - @Bean - public Supplier>> twitterStreamSupplier(TwitterStream twitterStream, - FluxMessageChannel twitterStatusInputChannel, TwitterStreamSupplierProperties streamProperties) { - - return () -> Flux.from(twitterStatusInputChannel) - .doOnSubscribe(subscription -> { - try { - switch (streamProperties.getType()) { - - case filter: - twitterStream.filter(streamProperties.getFilter().toFilterQuery()); - return; - - case sample: - twitterStream.sample(); - return; - - case firehose: - twitterStream.firehose(streamProperties.getFilter().getCount()); - return; - - case link: - twitterStream.links(streamProperties.getFilter().getCount()); - return; - default: - throw new IllegalArgumentException("Unknown stream type:" + streamProperties.getType()); - } - } - catch (Exception e) { - this.logger.error("Filter is not property set"); - } - }) - .doAfterTerminate(() -> { - this.logger.info("Proactive cancel for twitter stream"); - twitterStream.shutdown(); - }) - .doOnError(throwable -> { - this.logger.error(throwable.getMessage(), throwable); - }); - } -} diff --git a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/stream/TwitterStreamSupplierProperties.java b/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/stream/TwitterStreamSupplierProperties.java deleted file mode 100644 index 6de5b11ef..000000000 --- a/functions/supplier/twitter-supplier/src/main/java/org/springframework/cloud/fn/supplier/twitter/status/stream/TwitterStreamSupplierProperties.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.twitter.status.stream; - -import java.util.ArrayList; -import java.util.List; - -import twitter4j.FilterQuery; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.util.CollectionUtils; -import org.springframework.validation.annotation.Validated; - -/** - * @author Christian Tzolov - */ -@ConfigurationProperties("twitter.stream") -@Validated -public class TwitterStreamSupplierProperties { - - public enum StreamType { - /** Starts listening on random sample of all public statuses. The default access level provides a small - * proportion of the Firehose. */ - sample, - - /** Start consuming public statuses that match one or more filter predicates. At least one predicate parameter, - * follow, locations, or track must be specified. Multiple parameters may be specified which allows most - * clients to use a single connection to the Streaming API. Placing long parameters in the URL may cause the - * request to be rejected for excessive URL length.
- * The default access level allows up to 200 track keywords, 400 follow userids and 10 1-degree location boxes. - * Increased access levels allow 80,000 follow userids ('shadow' role), 400,000 follow userids ('birddog' role), - * 10,000 track keywords ('restricted track' role), 200,000 track keywords ('partner track' role), - * and 200 10-degree location boxes ('locRestricted' role). Increased track access levels also pass a higher - * proportion of statuses before limiting the stream.*/ - filter, - - /** tarts listening on all public statuses. Available only to approved parties and requires a signed agreement - * to access. */ - firehose, - - /** Starts listening on all public statuses containing links. - * Available only to approved parties and requires a signed agreement to access.*/ - link - } - - private StreamType type = StreamType.sample; - - private Filter filter = new Filter(); - - public Filter getFilter() { - return filter; - } - - public StreamType getType() { - return type; - } - - public void setType(StreamType type) { - this.type = type; - } - - public static class Filter { - - public enum FilterLevel { - /** filter level. */ - all, none, low, medium - } - - /** - * Indicates the number of previous statuses to stream before transitioning to the live stream. - */ - private int count = 0; - - /** - * Specifies the users, by ID, to receive public tweets from. - */ - private List follow; - - /** - * Specifies keywords to track. - */ - private List track; - - - /** - * Locations to track. Internally represented as 2D array. - * Bounding box is invalid: 52.38, 4.90, 51.51, -0.12. The first pair must be the SW corner of the box - */ - private List locations = new ArrayList<>(); - - - /** - * Specifies the tweets language of the stream. - */ - private List language; - - /** - * The filter level limits what tweets appear in the stream to those with a minimum filterLevel attribute value. - * One of either none, low, or medium. - */ - private FilterLevel filterLevel = FilterLevel.all; - - public int getCount() { - return count; - } - - public void setCount(int count) { - this.count = count; - } - - public List getFollow() { - return follow; - } - - public void setFollow(List follow) { - this.follow = follow; - } - - public List getTrack() { - return track; - } - - public void setTrack(List track) { - this.track = track; - } - - public List getLanguage() { - return language; - } - - public void setLanguage(List language) { - this.language = language; - } - - public List getLocations() { - return locations; - } - - public FilterLevel getFilterLevel() { - return filterLevel; - } - - public void setFilterLevel(FilterLevel filterLevel) { - this.filterLevel = filterLevel; - } - - public FilterQuery toFilterQuery() { - - FilterQuery filterQuery = new FilterQuery(); - - filterQuery.count(this.count); - - if (!CollectionUtils.isEmpty(this.track)) { - filterQuery.track(String.join(",", this.track)); - } - - if (!CollectionUtils.isEmpty(this.follow)) { - long[] followIds = new long[this.follow.size()]; - for (int i = 0; i < this.follow.size(); i++) { - followIds[i] = this.follow.get(i); - } - filterQuery.follow(followIds); - } - - if (!CollectionUtils.isEmpty(this.language)) { - filterQuery.language(this.language.toArray(new String[this.language.size()])); - } - - if (!CollectionUtils.isEmpty(this.locations)) { - double[][] bboxLocations = new double[this.locations.size() * 2][2]; - for (int i = 0; i < this.locations.size(); i = i + 2) { - //SW lat, lon - bboxLocations[i][0] = this.locations.get(i).getSw().getLat(); - bboxLocations[i][1] = this.locations.get(i).getSw().getLon(); - //NE lat, lon - bboxLocations[i + 1][0] = this.locations.get(i).getNe().getLat(); - bboxLocations[i + 1][1] = this.locations.get(i).getNe().getLon(); - } - filterQuery.locations(bboxLocations); - } - - if (this.filterLevel != FilterLevel.all) { - filterQuery.filterLevel(this.filterLevel.name()); - } - - return filterQuery; - } - - public boolean isValid() { - return count > 0 || !CollectionUtils.isEmpty(this.track) || !CollectionUtils.isEmpty(this.follow) - || !CollectionUtils.isEmpty(this.language) || this.filterLevel != FilterLevel.all; - } - - public static class BoundingBox { - - /** - * Bounding Box's South-West point (e.g. bottom-left). - */ - private Geocode sw; - - /** - * Bounding Box's North-East point (e.g. top-right). - */ - private Geocode ne; - - public Geocode getSw() { - return sw; - } - - public void setSw(Geocode sw) { - this.sw = sw; - } - - public Geocode getNe() { - return ne; - } - - public void setNe(Geocode ne) { - this.ne = ne; - } - } - - public static class Geocode { - - /** - * latitude. - */ - private double lat = -1; - - /** - * longitude. - */ - private double lon = -1; - - - public double getLat() { - return lat; - } - - public void setLat(double lat) { - this.lat = lat; - } - - public double getLon() { - return lon; - } - - public void setLon(double lon) { - this.lon = lon; - } - } - } -} diff --git a/functions/supplier/twitter-supplier/src/test/java/org/springframework/cloud/fn/supplier/twitter/status/search/SearchPaginationTests.java b/functions/supplier/twitter-supplier/src/test/java/org/springframework/cloud/fn/supplier/twitter/status/search/SearchPaginationTests.java deleted file mode 100644 index 988822ea1..000000000 --- a/functions/supplier/twitter-supplier/src/test/java/org/springframework/cloud/fn/supplier/twitter/status/search/SearchPaginationTests.java +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.twitter.status.search; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import org.junit.jupiter.api.Test; -import twitter4j.GeoLocation; -import twitter4j.HashtagEntity; -import twitter4j.MediaEntity; -import twitter4j.Place; -import twitter4j.RateLimitStatus; -import twitter4j.Scopes; -import twitter4j.Status; -import twitter4j.SymbolEntity; -import twitter4j.URLEntity; -import twitter4j.User; -import twitter4j.UserMentionEntity; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.springframework.cloud.fn.supplier.twitter.status.search.SearchPagination.UNBOUNDED; - -/** - * @author Christian Tzolov - */ -public class SearchPaginationTests { - - @Test - public void tests1() { - - int maxCountPerRequest = 5; - - int count = 10; - - int pageCount = count / maxCountPerRequest; - - assertThat(pageCount).isEqualTo(2); - - SearchPagination pagination = new SearchPagination(pageCount, false); - - assertThat(pagination.getSinceId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageCounter()).isEqualTo(pageCount - 1); - - pagination.update(tweets(10, 8, 1)); - - assertThat(pagination.getSinceId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getMaxId()).isEqualTo(8L - 1); // = min - 1 - assertThat(pagination.getPageMaxId()).isEqualTo(10L); - assertThat(pagination.getPageCounter()).isEqualTo(pageCount - 2); - - pagination.update(tweets(7, 4, 1)); - - assertThat(pagination.getSinceId()).isEqualTo(10L); - assertThat(pagination.getMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageCounter()).isEqualTo(pageCount - 1); - - pagination.update(tweets(20, 14, 1)); - - assertThat(pagination.getSinceId()).isEqualTo(10L); - assertThat(pagination.getMaxId()).isEqualTo(14L - 1); - assertThat(pagination.getPageMaxId()).isEqualTo(20L); - assertThat(pagination.getPageCounter()).isEqualTo(pageCount - 2); - - pagination.update(tweets(13, 8, 1)); - - assertThat(pagination.getSinceId()).isEqualTo(20L); - assertThat(pagination.getMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageCounter()).isEqualTo(pageCount - 1); - } - - @Test - public void tests2() { - - int maxCountPerRequest = 5; - - int count = 10; - - int pageCount = count / maxCountPerRequest; - - assertThat(pageCount).isEqualTo(2); - - SearchPagination pagination = new SearchPagination(pageCount, true); - - assertThat(pagination.getSinceId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageCounter()).isEqualTo(pageCount - 1); - - pagination.update(tweets(10, 8, 1)); - - assertThat(pagination.getSinceId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getMaxId()).isEqualTo(8L - 1); // = min - 1 - assertThat(pagination.getPageMaxId()).isEqualTo(10L); - assertThat(pagination.getPageCounter()).isEqualTo(pageCount - 2); - - pagination.update(tweets(7, 4, 1)); - - // Restart from Most Recent due to pageCounter == 0 while no reset have been performed so ar - assertThat(pagination.getSinceId()).isEqualTo(10L); - assertThat(pagination.getMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageCounter()).isEqualTo(pageCount - 1); - - pagination.update(tweets(20, 14, 1)); - - assertThat(pagination.getSinceId()).isEqualTo(10L); - assertThat(pagination.getMaxId()).isEqualTo(14L - 1); - assertThat(pagination.getPageMaxId()).isEqualTo(20L); - assertThat(pagination.getPageCounter()).isEqualTo(pageCount - 2); - - pagination.update(tweets(13, 8, 1)); - - assertThat(pagination.getSinceId()).isEqualTo(10L); - assertThat(pagination.getMaxId()).isEqualTo(8L - 1); - assertThat(pagination.getPageMaxId()).isEqualTo(20L); - assertThat(pagination.getPageCounter()).isEqualTo(pageCount - 3); - - pagination.update(new ArrayList<>()); // EMPTY - - // Restart from Most Recent on Empty Response - assertThat(pagination.getSinceId()).isEqualTo(20L); - assertThat(pagination.getMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageMaxId()).isEqualTo(UNBOUNDED); - assertThat(pagination.getPageCounter()).isEqualTo(pageCount - 1); - } - - public List tweets(int to, int from, int step) { - List tweets = new ArrayList<>(to - from); - for (int i = to; i >= from; i = i - step) { - tweets.add(new MyStatus(i)); - } - - return tweets; - } - - public static class MyStatus implements Status { - - private long id; - - public MyStatus(long id) { - this.id = id; - } - - @Override - public Date getCreatedAt() { - return null; - } - - @Override - public long getId() { - return this.id; - } - - @Override - public String getText() { - return null; - } - - @Override - public int getDisplayTextRangeStart() { - return 0; - } - - @Override - public int getDisplayTextRangeEnd() { - return 0; - } - - @Override - public String getSource() { - return null; - } - - @Override - public boolean isTruncated() { - return false; - } - - @Override - public long getInReplyToStatusId() { - return 0; - } - - @Override - public long getInReplyToUserId() { - return 0; - } - - @Override - public String getInReplyToScreenName() { - return null; - } - - @Override - public GeoLocation getGeoLocation() { - return null; - } - - @Override - public Place getPlace() { - return null; - } - - @Override - public boolean isFavorited() { - return false; - } - - @Override - public boolean isRetweeted() { - return false; - } - - @Override - public int getFavoriteCount() { - return 0; - } - - @Override - public User getUser() { - return null; - } - - @Override - public boolean isRetweet() { - return false; - } - - @Override - public Status getRetweetedStatus() { - return null; - } - - @Override - public long[] getContributors() { - return new long[0]; - } - - @Override - public int getRetweetCount() { - return 0; - } - - @Override - public boolean isRetweetedByMe() { - return false; - } - - @Override - public long getCurrentUserRetweetId() { - return 0; - } - - @Override - public boolean isPossiblySensitive() { - return false; - } - - @Override - public String getLang() { - return null; - } - - @Override - public Scopes getScopes() { - return null; - } - - @Override - public String[] getWithheldInCountries() { - return new String[0]; - } - - @Override - public long getQuotedStatusId() { - return 0; - } - - @Override - public Status getQuotedStatus() { - return null; - } - - @Override - public URLEntity getQuotedStatusPermalink() { - return null; - } - - @Override - public int compareTo(Status o) { - return 0; - } - - @Override - public UserMentionEntity[] getUserMentionEntities() { - return new UserMentionEntity[0]; - } - - @Override - public URLEntity[] getURLEntities() { - return new URLEntity[0]; - } - - @Override - public HashtagEntity[] getHashtagEntities() { - return new HashtagEntity[0]; - } - - @Override - public MediaEntity[] getMediaEntities() { - return new MediaEntity[0]; - } - - @Override - public SymbolEntity[] getSymbolEntities() { - return new SymbolEntity[0]; - } - - @Override - public RateLimitStatus getRateLimitStatus() { - return null; - } - - @Override - public int getAccessLevel() { - return 0; - } - } -} diff --git a/functions/supplier/twitter-supplier/src/test/java/org/springframework/cloud/fn/supplier/twitter/status/stream/TwitterStreamSupplierTests.java b/functions/supplier/twitter-supplier/src/test/java/org/springframework/cloud/fn/supplier/twitter/status/stream/TwitterStreamSupplierTests.java deleted file mode 100644 index d6bc2b550..000000000 --- a/functions/supplier/twitter-supplier/src/test/java/org/springframework/cloud/fn/supplier/twitter/status/stream/TwitterStreamSupplierTests.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright 2020-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.twitter.status.stream; - -import java.util.concurrent.TimeUnit; -import java.util.function.Function; -import java.util.function.Supplier; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.mockserver.client.MockServerClient; -import org.mockserver.integration.ClientAndServer; -import org.mockserver.model.Header; -import org.mockserver.model.HttpRequest; -import org.mockserver.model.StringBody; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; -import twitter4j.conf.ConfigurationBuilder; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.common.twitter.TwitterConnectionProperties; -import org.springframework.cloud.fn.common.twitter.util.TwitterTestUtils; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Import; -import org.springframework.context.annotation.Primary; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.util.TestSocketUtils; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.mockserver.matchers.Times.exactly; -import static org.mockserver.model.HttpRequest.request; -import static org.mockserver.model.HttpResponse.response; -import static org.mockserver.verify.VerificationTimes.once; - -/** - * @author Christian Tzolov - */ -@SpringBootTest( - webEnvironment = SpringBootTest.WebEnvironment.NONE, - properties = { - "twitter.connection.consumerKey=consumerKey666", - "twitter.connection.consumerSecret=consumerSecret666", - "twitter.connection.accessToken=accessToken666", - "twitter.connection.accessTokenSecret=accessTokenSecret666" - }) -@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) -public abstract class TwitterStreamSupplierTests { - - private static final String MOCK_SERVER_IP = "127.0.0.1"; - - private static final Integer MOCK_SERVER_PORT = TestSocketUtils.findAvailableTcpPort(); - - private static ClientAndServer mockServer; - - private static MockServerClient mockClient; - private static HttpRequest streamFilterRequest; - private static HttpRequest streamSampleRequest; - private static HttpRequest streamFirehoseRequest; - - @Autowired - protected Supplier>> twitterStreamSupplier; - - @BeforeAll - public static void startServer() { - - mockServer = ClientAndServer.startClientAndServer(MOCK_SERVER_PORT); - - mockClient = new MockServerClient(MOCK_SERVER_IP, MOCK_SERVER_PORT); - - streamFilterRequest = mockClientRecordRequest(request() - .withMethod("POST") - .withPath("/stream/statuses/filter.json") - .withBody(new StringBody("count=0&track=Java%2CPython&stall_warnings=true"))); - - streamSampleRequest = mockClientRecordRequest(request() - .withMethod("GET") - .withPath("/stream/statuses/sample.json")); - - streamFirehoseRequest = mockClientRecordRequest(request() - .withMethod("POST") - .withPath("/stream/statuses/links.json") - .withBody(new StringBody("count=0&stall_warnings=true"))); - - streamFirehoseRequest = mockClientRecordRequest(request() - .withMethod("POST") - .withPath("/stream/statuses/firehose.json") - .withBody(new StringBody("count=0&stall_warnings=true"))); - } - - @AfterAll - public static void stopServer() { - mockServer.stop(); - } - - private static HttpRequest mockClientRecordRequest(HttpRequest request) { - mockClient.when(request, /*unlimited())*/ exactly(1)) - .respond( - response() - .withStatusCode(200) - .withHeaders( - new Header("Content-Type", "application/json; charset=utf-8"), - new Header("Cache-Control", "public, max-age=86400")) - .withBody(TwitterTestUtils.asString("classpath:/response/stream_test_1.json")) - .withDelay(TimeUnit.SECONDS, 10)); - return request; - } - - @TestPropertySource(properties = { - "twitter.stream.type=sample" - }) - public static class TwitterStreamSampleTests extends TwitterStreamSupplierTests { - - @Test - public void testOne() { - final Flux> messageFlux = twitterStreamSupplier.get(); - - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> assertThat(new String((byte[]) message.getPayload())) - .contains("\"id\":1075751718749659136")) - .thenCancel() - .verifyLater(); - - stepVerifier.verify(); - - mockClient.verify(streamSampleRequest, once()); - } - } - - @TestPropertySource(properties = { - "twitter.stream.type=filter", - "twitter.stream.filter.track=Java,Python" - }) - public static class TwitterStreamFilterTests extends TwitterStreamSupplierTests { - - @Test - public void testOne() { - final Flux> messageFlux = twitterStreamSupplier.get(); - - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> assertThat(new String((byte[]) message.getPayload())) - .contains("\"id\":1075751718749659136")) - .thenCancel() - .verifyLater(); - - stepVerifier.verify(); - - mockClient.verify(streamFilterRequest, once()); - } - } - - @TestPropertySource(properties = { - "twitter.stream.type=firehose" - }) - public static class TwitterStreamFirehoseTests extends TwitterStreamSupplierTests { - - @Test - public void testOne() { - final Flux> messageFlux = twitterStreamSupplier.get(); - - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> assertThat(new String((byte[]) message.getPayload())) - .contains("\"id\":1075751718749659136")) - .thenCancel() - .verifyLater(); - - stepVerifier.verify(); - - mockClient.verify(streamFirehoseRequest, once()); - } - } - - @SpringBootConfiguration - @EnableAutoConfiguration - @Import(TwitterStreamSupplierConfiguration.class) - public static class TwitterStreamSupplierTestApplication { - - @Bean - @Primary - public twitter4j.conf.Configuration twitterConfiguration2(TwitterConnectionProperties properties, - Function toConfigurationBuilder) { - - Function mockedConfiguration = - toConfigurationBuilder.andThen( - new TwitterTestUtils().mockTwitterUrls( - String.format("http://%s:%s", MOCK_SERVER_IP, MOCK_SERVER_PORT))); - - return mockedConfiguration.apply(properties).build(); - } - } -} diff --git a/functions/supplier/twitter-supplier/src/test/resources/logback.xml b/functions/supplier/twitter-supplier/src/test/resources/logback.xml deleted file mode 100644 index f7e48fcff..000000000 --- a/functions/supplier/twitter-supplier/src/test/resources/logback.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - - - - - - - - diff --git a/functions/supplier/twitter-supplier/src/test/resources/response/stream_test_1.json b/functions/supplier/twitter-supplier/src/test/resources/response/stream_test_1.json deleted file mode 100644 index bb8312063..000000000 --- a/functions/supplier/twitter-supplier/src/test/resources/response/stream_test_1.json +++ /dev/null @@ -1,2 +0,0 @@ -{"created_at":"Thu Dec 20 13:56:10 +0000 2018","id":1075751718749659136,"id_str":"1075751718749659136","text":"Codementor: Kubernetes for Python Developers: Part 1\n#100daysofcode #python https:\/\/t.co\/VviTgpFpge","source":"\u003ca href=\"https:\/\/ifttt.com\" rel=\"nofollow\"\u003eIFTTT\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":859252650512072704,"id_str":"859252650512072704","name":"Freelancing|WebDev","screen_name":"FreelanceForBTC","location":"THE NET","url":"http:\/\/bit.ly\/BTCFREELANCING","description":"This twitter is designed to give the best information from web development to Blockchain.\n|YouTube's |Jobs |learning contented","translator_type":"none","protected":false,"verified":false,"followers_count":146,"friends_count":118,"listed_count":4,"favourites_count":445,"statuses_count":7528,"created_at":"Tue May 02 03:46:10 +0000 2017","utc_offset":null,"time_zone":null,"geo_enabled":false,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"F5F8FA","profile_background_image_url":"","profile_background_image_url_https":"","profile_background_tile":false,"profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/1071607798771793920\/UegrCs84_normal.jpg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/1071607798771793920\/UegrCs84_normal.jpg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/859252650512072704\/1544326203","default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"quote_count":0,"reply_count":0,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"100daysofcode","indices":[53,67]},{"text":"python","indices":[68,75]}],"urls":[{"url":"https:\/\/t.co\/VviTgpFpge","expanded_url":"https:\/\/ift.tt\/2GxdMZF","display_url":"ift.tt\/2GxdMZF","indices":[76,99]}],"user_mentions":[],"symbols":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"low","lang":"ca","timestamp_ms":"1545314170907"} - diff --git a/functions/supplier/websocket-supplier/README.adoc b/functions/supplier/websocket-supplier/README.adoc deleted file mode 100644 index 33f546d35..000000000 --- a/functions/supplier/websocket-supplier/README.adoc +++ /dev/null @@ -1,32 +0,0 @@ -# Websocket Supplier - -A basic websocket supplier that produced messages through web socket. -The `Supplier` uses the `WebsocketInboundChannelAdapter` from Spring Integration. -This supplier gives you a reactive stream of messages and the supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and receive the data. - -## Beans for injection - -You can import the `WebsocketSupplierConfiguration` in the application and then inject the following bean. - -`websocketSupplier` - -You need to inject this as `Supplier>>`. - -You can use `websocketSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `websocket.supplier`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/websocket/WebsocketSupplierProperties.java[WebsocketSupplierProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/websocket/WebsocketSupplierTests.java[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/websocket-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a File Source. \ No newline at end of file diff --git a/functions/supplier/websocket-supplier/pom.xml b/functions/supplier/websocket-supplier/pom.xml deleted file mode 100644 index 12380b262..000000000 --- a/functions/supplier/websocket-supplier/pom.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - websocket-supplier - websocket-supplier - websocket supplier - - - - org.springframework.integration - spring-integration-websocket - - - org.springframework.boot - spring-boot-starter-web - test - - - org.springframework.boot - spring-boot-starter-security - test - - - org.springframework.cloud.fn - function-test-support - ${project.version} - test - - - - diff --git a/functions/supplier/websocket-supplier/src/main/java/org/springframework/cloud/fn/supplier/websocket/WebsocketSupplierConfiguration.java b/functions/supplier/websocket-supplier/src/main/java/org/springframework/cloud/fn/supplier/websocket/WebsocketSupplierConfiguration.java deleted file mode 100644 index 60fd80a17..000000000 --- a/functions/supplier/websocket-supplier/src/main/java/org/springframework/cloud/fn/supplier/websocket/WebsocketSupplierConfiguration.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2018-2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.websocket; - -import java.util.function.Supplier; - -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.dsl.IntegrationFlow; -import org.springframework.integration.websocket.IntegrationWebSocketContainer; -import org.springframework.integration.websocket.ServerWebSocketContainer; -import org.springframework.integration.websocket.inbound.WebSocketInboundChannelAdapter; -import org.springframework.messaging.Message; - -/** - * A supplier that receives data over WebSocket. - * - * @author Krishnaprasad A S - * @author Artem Bilan - * - */ -@Configuration -@EnableConfigurationProperties(WebsocketSupplierProperties.class) -public class WebsocketSupplierConfiguration { - - @Autowired - WebsocketSupplierProperties properties; - - @Bean - public Supplier>> websocketSupplier(Publisher> websocketPublisher, - WebSocketInboundChannelAdapter webSocketInboundChannelAdapter) { - return () -> Flux.from(websocketPublisher) - .doOnSubscribe(subscription -> webSocketInboundChannelAdapter.start()) - .doOnTerminate(webSocketInboundChannelAdapter::stop); - } - - @Bean - public Publisher> websocketPublisher(IntegrationWebSocketContainer serverWebSocketContainer) { - return IntegrationFlow.from( - webSocketInboundChannelAdapter(serverWebSocketContainer)) - .toReactivePublisher(); - } - - private WebSocketInboundChannelAdapter webSocketInboundChannelAdapter( - IntegrationWebSocketContainer serverWebSocketContainer) { - WebSocketInboundChannelAdapter webSocketInboundChannelAdapter = - new WebSocketInboundChannelAdapter(serverWebSocketContainer); - webSocketInboundChannelAdapter.setAutoStartup(false); - return webSocketInboundChannelAdapter; - } - - @Bean - @ConditionalOnMissingBean - @ConditionalOnProperty(prefix = "websocket.sockJs", name = "enable", havingValue = "true") - public ServerWebSocketContainer.SockJsServiceOptions sockJsServiceOptions() { - // TODO Expose SockJsServiceOptions as configuration properties - return new ServerWebSocketContainer.SockJsServiceOptions(); - } - - @Bean - public IntegrationWebSocketContainer serverWebSocketContainer( - ObjectProvider sockJsServiceOptions) { - return new ServerWebSocketContainer(properties.getPath()) - .setAllowedOrigins(properties.getAllowedOrigins()) - .withSockJs(sockJsServiceOptions.getIfAvailable()); - } - -} diff --git a/functions/supplier/websocket-supplier/src/main/java/org/springframework/cloud/fn/supplier/websocket/WebsocketSupplierProperties.java b/functions/supplier/websocket-supplier/src/main/java/org/springframework/cloud/fn/supplier/websocket/WebsocketSupplierProperties.java deleted file mode 100644 index 3d23343c3..000000000 --- a/functions/supplier/websocket-supplier/src/main/java/org/springframework/cloud/fn/supplier/websocket/WebsocketSupplierProperties.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright 2018-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.websocket; - -import org.springframework.boot.context.properties.ConfigurationProperties; - -/** - * Configuration properties available for Websocket Supplier. - * - * @author Krishnaprasad A S - * @author Artem Bilan - * - */ -@ConfigurationProperties("websocket.supplier") -public class WebsocketSupplierProperties { - - /** - * Default path. - */ - public static final String DEFAULT_PATH = "/websocket"; - - /** - * Default allowed origins. - */ - public static final String DEFAULT_ALLOWED_ORIGINS = "*"; - - /** - * The path on which server WebSocket handler is exposed. - */ - private String path = DEFAULT_PATH; - - /** - * The allowed origins. - */ - private String allowedOrigins = DEFAULT_ALLOWED_ORIGINS; - - /** - * The SockJS options. - */ - private SockJs sockJs = new SockJs(); - - public String getPath() { - return this.path; - } - - public void setPath(String path) { - this.path = path; - } - - public String getAllowedOrigins() { - return this.allowedOrigins; - } - - public void setAllowedOrigins(String allowedOrigins) { - this.allowedOrigins = allowedOrigins; - } - - public SockJs getSockJs() { - return this.sockJs; - } - - public void setSockJs(SockJs sockJs) { - this.sockJs = sockJs; - } - - public static class SockJs { - - /** - * Enable SockJS service on the server. Default is 'false' - */ - private boolean enable; - - public boolean getEnable() { - return this.enable; - } - - public void setEnable(boolean enable) { - this.enable = enable; - } - - } -} diff --git a/functions/supplier/websocket-supplier/src/test/java/org/springframework/cloud/fn/supplier/websocket/WebsocketSupplierTests.java b/functions/supplier/websocket-supplier/src/test/java/org/springframework/cloud/fn/supplier/websocket/WebsocketSupplierTests.java deleted file mode 100644 index 1e40e644b..000000000 --- a/functions/supplier/websocket-supplier/src/test/java/org/springframework/cloud/fn/supplier/websocket/WebsocketSupplierTests.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2018-2021 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.websocket; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.function.Supplier; - -import org.junit.jupiter.api.Test; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.security.SecurityProperties; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.web.server.LocalServerPort; -import org.springframework.http.HttpHeaders; -import org.springframework.integration.websocket.ClientWebSocketContainer; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; -import org.springframework.util.Base64Utils; -import org.springframework.web.socket.TextMessage; -import org.springframework.web.socket.WebSocketSession; -import org.springframework.web.socket.client.standard.StandardWebSocketClient; - -import static org.assertj.core.api.Assertions.assertThat; - -@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, - properties = "websocket.supplier.path=some_websocket_path") -@DirtiesContext -public class WebsocketSupplierTests { - - @Autowired - private Supplier>> websocketSupplier; - - @LocalServerPort - private int port; - - @Autowired - private WebsocketSupplierProperties properties; - - @Autowired - private SecurityProperties securityProperties; - - private final String messageString = "foo"; - - @Test - public void checkCmdlineArgs() { - assertThat(this.properties.getPath()).isEqualTo("some_websocket_path"); - assertThat(this.properties.getAllowedOrigins()).isEqualTo("*"); - } - - @Test - public void testBasicFlow() throws IOException { - final Flux> messageFlux = websocketSupplier.get(); - final StepVerifier stepVerifier = StepVerifier.create(messageFlux) - .assertNext((message) -> - assertThat(message.getPayload()) - .isEqualTo(messageString) - ) - .thenCancel() - .verifyLater(); - StandardWebSocketClient webSocketClient = new StandardWebSocketClient(); - ClientWebSocketContainer clientWebSocketContainer = - new ClientWebSocketContainer(webSocketClient, "ws://localhost:{port}/{path}", - this.port, - this.properties.getPath()); - - HttpHeaders httpHeaders = new HttpHeaders(); - String token = Base64Utils.encodeToString( - (this.securityProperties.getUser().getName() + ":" + this.securityProperties.getUser().getPassword()) - .getBytes(StandardCharsets.UTF_8)); - httpHeaders.set(HttpHeaders.AUTHORIZATION, "Basic " + token); - clientWebSocketContainer.setHeaders(httpHeaders); - clientWebSocketContainer.start(); - WebSocketSession session = clientWebSocketContainer.getSession(null); - session.sendMessage(new TextMessage(this.messageString)); - session.close(); - - stepVerifier.verify(); - } - - @SpringBootApplication - static class WebsocketSupplierTestApplication { - - } - -} diff --git a/functions/supplier/xmpp-supplier/README.adoc b/functions/supplier/xmpp-supplier/README.adoc deleted file mode 100644 index ba4bf9f23..000000000 --- a/functions/supplier/xmpp-supplier/README.adoc +++ /dev/null @@ -1,30 +0,0 @@ -# XMPP Supplier - -A supplier that allows you to receive messages through a XMPP server. - -## Beans for injection - -You can import the `XmppSupplierConfiguration` in the application and then inject the following bean. - -`Supplier xmppSupplier` - -You need to inject this as `Supplier xmppSupplier`. - -You can use `xmppSupplier` as a qualifier when injecting. - -**NOTE:** This is a functional endpoint. One will need to subscribe to this endpoint in order to start accepting data -on it. - -## Configuration Options - -All configuration properties are prefixed with `xmpp.supplier`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/xmpp/XmppSupplierProperties.java[XmppSupplierProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/consumer/xmpp/[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/xmpp-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a XMPP Source. diff --git a/functions/supplier/xmpp-supplier/pom.xml b/functions/supplier/xmpp-supplier/pom.xml deleted file mode 100644 index 8c2d6277e..000000000 --- a/functions/supplier/xmpp-supplier/pom.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - xmpp-supplier - xmpp-supplier - XMPP supplier - - - - org.springframework.cloud.fn - xmpp-common - ${project.version} - - - - - org.springframework.cloud.fn - function-test-support - ${project.version} - test - - - - diff --git a/functions/supplier/xmpp-supplier/src/main/java/org/springframework/cloud/fn/supplier/xmpp/XmppSupplierConfiguration.java b/functions/supplier/xmpp-supplier/src/main/java/org/springframework/cloud/fn/supplier/xmpp/XmppSupplierConfiguration.java deleted file mode 100644 index 2fa79356e..000000000 --- a/functions/supplier/xmpp-supplier/src/main/java/org/springframework/cloud/fn/supplier/xmpp/XmppSupplierConfiguration.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.xmpp; - -import java.util.function.Supplier; - -import org.jivesoftware.smack.XMPPConnection; -import reactor.core.publisher.Flux; - -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.fn.common.xmpp.XmppConnectionFactoryConfiguration; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; -import org.springframework.integration.channel.FluxMessageChannel; -import org.springframework.integration.xmpp.inbound.ChatMessageListeningEndpoint; -import org.springframework.messaging.Message; - -/** - * A source module that receives data from ZeroMQ. - * - * @author Daniel Frey - * @since 4.0.0 - */ -@Configuration -@EnableConfigurationProperties(XmppSupplierProperties.class) -@Import(XmppConnectionFactoryConfiguration.class) -public class XmppSupplierConfiguration { - - private FluxMessageChannel output = new FluxMessageChannel(); - - @Bean - public ChatMessageListeningEndpoint chatMessageListeningEndpoint(XMPPConnection xmppConnection, XmppSupplierProperties properties) { - - var chatMessageListeningEndpoint = new ChatMessageListeningEndpoint(xmppConnection); - - if (properties.getPayloadExpression() != null) { - chatMessageListeningEndpoint.setPayloadExpression(properties.getPayloadExpression()); - } - - chatMessageListeningEndpoint.setStanzaFilter(properties.getStanzaFilter()); - chatMessageListeningEndpoint.setOutputChannel(output); - chatMessageListeningEndpoint.setAutoStartup(false); - - return chatMessageListeningEndpoint; - } - - @Bean - public Supplier>> xmppSupplier(ChatMessageListeningEndpoint chatMessageListeningEndpoint) { - return () -> Flux.from(output).doOnSubscribe(subscription -> chatMessageListeningEndpoint.start()); - } - -} diff --git a/functions/supplier/xmpp-supplier/src/main/java/org/springframework/cloud/fn/supplier/xmpp/XmppSupplierProperties.java b/functions/supplier/xmpp-supplier/src/main/java/org/springframework/cloud/fn/supplier/xmpp/XmppSupplierProperties.java deleted file mode 100644 index 7e8c4a265..000000000 --- a/functions/supplier/xmpp-supplier/src/main/java/org/springframework/cloud/fn/supplier/xmpp/XmppSupplierProperties.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2014-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.xmpp; - -import org.jivesoftware.smack.filter.StanzaFilter; -import org.jivesoftware.smack.filter.StanzaTypeFilter; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.expression.Expression; -import org.springframework.validation.annotation.Validated; - -/** - * - * @author Daniel Frey - * @since 4.0.0 - */ -@ConfigurationProperties("xmpp.supplier") -@Validated -public class XmppSupplierProperties { - - private StanzaFilter stanzaFilter = StanzaTypeFilter.MESSAGE; - - private Expression payloadExpression; - - public void setStanzaFilter(StanzaFilter stanzaFilter) { - this.stanzaFilter = stanzaFilter; - } - - public StanzaFilter getStanzaFilter() { - return stanzaFilter; - } - - public void setPayloadExpression(Expression payloadExpression) { - this.payloadExpression = payloadExpression; - } - public Expression getPayloadExpression() { - return payloadExpression; - } - -} diff --git a/functions/supplier/xmpp-supplier/src/test/java/org/springframework/cloud/fn/supplier/xmpp/XmppSupplierConfigurationTests.java b/functions/supplier/xmpp-supplier/src/test/java/org/springframework/cloud/fn/supplier/xmpp/XmppSupplierConfigurationTests.java deleted file mode 100644 index 48acd9eb0..000000000 --- a/functions/supplier/xmpp-supplier/src/test/java/org/springframework/cloud/fn/supplier/xmpp/XmppSupplierConfigurationTests.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright 2014-2022 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.xmpp; - -import java.io.IOException; -import java.time.Duration; -import java.util.function.Supplier; - -import org.assertj.core.api.InstanceOfAssertFactories; -import org.jivesoftware.smack.ConnectionConfiguration; -import org.jivesoftware.smack.SmackException; -import org.jivesoftware.smack.XMPPException; -import org.jivesoftware.smack.chat2.ChatManager; -import org.jivesoftware.smack.tcp.XMPPTCPConnection; -import org.jivesoftware.smack.tcp.XMPPTCPConnectionConfiguration; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.jxmpp.jid.impl.JidCreate; -import org.jxmpp.stringprep.XmppStringprepException; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.SpringBootConfiguration; -import org.springframework.boot.autoconfigure.EnableAutoConfiguration; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.cloud.fn.test.support.xmpp.XmppTestContainerSupport; -import org.springframework.context.annotation.Import; -import org.springframework.integration.xmpp.XmppHeaders; -import org.springframework.messaging.Message; -import org.springframework.test.context.DynamicPropertyRegistry; -import org.springframework.test.context.DynamicPropertySource; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.springframework.cloud.fn.test.support.xmpp.XmppTestContainerSupport.JANE_USER; -import static org.springframework.cloud.fn.test.support.xmpp.XmppTestContainerSupport.SERVICE_NAME; -import static org.springframework.cloud.fn.test.support.xmpp.XmppTestContainerSupport.USER_PW; - -/** - * @author Daniel Frey - */ -@SpringBootTest( - properties = { - "xmpp.factory.user=" + JANE_USER, // Connect as user intended to listen for messages on behalf of - "xmpp.factory.password=" + USER_PW, - "xmpp.factory.service-name=" + SERVICE_NAME, - "xmpp.factory.security-mode=disabled" - } -) -public class XmppSupplierConfigurationTests implements XmppTestContainerSupport { - - @DynamicPropertySource - static void registerConfigurationProperties(DynamicPropertyRegistry registry) { - registry.add("xmpp.factory.host", () -> XmppTestContainerSupport.getXmppHost()); - registry.add("xmpp.factory.port", () -> XmppTestContainerSupport.getXmppMappedPort()); - } - - @Autowired - Supplier>> subject; - - // A client source connection is needed to send the message to the xmpp server - private XMPPTCPConnection sourceConnection; - - @BeforeEach - void setup() throws IOException, SmackException, XMPPException, InterruptedException { - - var builder = XMPPTCPConnectionConfiguration.builder(); - builder.setSecurityMode(ConnectionConfiguration.SecurityMode.disabled); - builder.setHost(XmppTestContainerSupport.getXmppHost()); - builder.setPort(XmppTestContainerSupport.getXmppMappedPort()); - builder.setResource(SERVICE_NAME); - builder.setUsernameAndPassword(JOHN_USER, USER_PW) // Connect as user intended to send messages from - .setXmppDomain(SERVICE_NAME); - this.sourceConnection = new XMPPTCPConnection(builder.build()); - this.sourceConnection.connect(); - this.sourceConnection.login(); - - } - - @AfterEach - void teardown() { - this.sourceConnection.instantShutdown(); - } - - @Test - void testSubscriptionConfiguration() throws XmppStringprepException, SmackException.NotConnectedException, InterruptedException { - - var payload = "test"; - - var stepVerifier = - StepVerifier.create(subject.get()) - .assertNext((message) -> { - - assertThat(message.getPayload()) - .asInstanceOf(InstanceOfAssertFactories.type(String.class)) - .isEqualTo(payload); - - assertThat(message.getHeaders().containsKey(XmppHeaders.TO)).isTrue(); - assertThat(message.getHeaders().get(XmppHeaders.TO, String.class)).isEqualTo(JANE_USER + "@" + SERVICE_NAME); - - }) - .thenCancel() - .verifyLater(); - - var chatManager = ChatManager.getInstanceFor(this.sourceConnection); - var jid = JidCreate.entityBareFrom(JANE_USER + "@" + SERVICE_NAME); - var chat = chatManager.chatWith(jid); - chat.send(payload); - - stepVerifier.verify(Duration.ofSeconds(10)); - - } - - @SpringBootConfiguration - @EnableAutoConfiguration - @Import(XmppSupplierConfiguration.class) - static class XmppSupplierTestApplication { } - -} diff --git a/functions/supplier/zeromq-supplier/README.adoc b/functions/supplier/zeromq-supplier/README.adoc deleted file mode 100644 index bacc875ab..000000000 --- a/functions/supplier/zeromq-supplier/README.adoc +++ /dev/null @@ -1,32 +0,0 @@ -# ZeroMQ Supplier - -A basic ZeroMQ supplier that produced messages through TCP connection. -The `Supplier` uses the `ZeroMqMessageProducer` from Spring Integration. -This supplier gives you a reactive stream of messages and the supplier has a signature of `Supplier>>`. -Users have to subscribe to this `Flux` and receive the data. - -## Beans for injection - -You can import the `ZeroMqSupplierConfiguration` in the application and then inject the following bean. - -`zeromqSupplier` - -You need to inject this as `Supplier>>`. - -You can use `zeromqSupplier` as a qualifier when injecting. - -Once injected, you can use the `get` method of the `Supplier` to invoke it and then subscribe to the returned `Flux`. - -## Configuration Options - -All configuration properties are prefixed with `zeromq.supplier`. - -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/zeromq/ZeroMqSupplierProperties.java[ZeroMqSupplierProperties]. - -## Tests - -See this link:src/test/java/org/springframework/cloud/fn/supplier/zeromq/ZeroMqSupplierTests.java[test suite] for the various ways, this supplier is used. - -## Other usage - -See this https://github.com/spring-cloud/stream-applications/blob/master/applications/source/zeromq-source/README.adoc[README] where this supplier is used to create a Spring Cloud Stream application where it makes a ZeroMQ Source. \ No newline at end of file diff --git a/functions/supplier/zeromq-supplier/pom.xml b/functions/supplier/zeromq-supplier/pom.xml deleted file mode 100644 index 408e4fad2..000000000 --- a/functions/supplier/zeromq-supplier/pom.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - 4.0.0 - - - org.springframework.cloud.fn - spring-functions-parent - 5.0.0-SNAPSHOT - ../../spring-functions-parent/pom.xml - - - zeromq-supplier - zeromq-supplier - ZeroMQ supplier - - - - org.springframework.integration - spring-integration-zeromq - - - - diff --git a/functions/supplier/zeromq-supplier/src/main/java/org/springframework/cloud/fn/supplier/zeromq/ZeroMqSupplierConfiguration.java b/functions/supplier/zeromq-supplier/src/main/java/org/springframework/cloud/fn/supplier/zeromq/ZeroMqSupplierConfiguration.java deleted file mode 100644 index c55bc0085..000000000 --- a/functions/supplier/zeromq-supplier/src/main/java/org/springframework/cloud/fn/supplier/zeromq/ZeroMqSupplierConfiguration.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.zeromq; - -import java.util.function.Consumer; -import java.util.function.Supplier; - -import org.zeromq.SocketType; -import org.zeromq.ZContext; -import org.zeromq.ZMQ; -import reactor.core.publisher.Flux; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.integration.channel.FluxMessageChannel; -import org.springframework.integration.zeromq.inbound.ZeroMqMessageProducer; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; - -/** - * A source module that receives data from ZeroMQ. - * - * @author Daniel Frey - * @since 3.1.0 - */ -@Configuration -@EnableConfigurationProperties(ZeroMqSupplierProperties.class) -public class ZeroMqSupplierConfiguration { - - private FluxMessageChannel output = new FluxMessageChannel(); - - @Bean - public ZContext zContext() { - return new ZContext(); - } - - @Bean - public ZeroMqMessageProducer adapter(ZeroMqSupplierProperties properties, ZContext zContext, - @Autowired(required = false) Consumer socketConfigurer) { - - ZeroMqMessageProducer zeroMqMessageProducer = new ZeroMqMessageProducer(zContext, properties.getSocketType()); - - if (properties.getConnectUrl() != null) { - zeroMqMessageProducer.setConnectUrl(properties.getConnectUrl()); - } - else if (properties.getBindPort() > 0) { - zeroMqMessageProducer.setBindPort(properties.getBindPort()); - } - zeroMqMessageProducer.setConsumeDelay(properties.getConsumeDelay()); - if (SocketType.SUB.equals(properties.getSocketType())) { - zeroMqMessageProducer.setTopics(properties.getTopics()); - } - zeroMqMessageProducer.setMessageMapper(GenericMessage::new); - if (socketConfigurer != null) { - zeroMqMessageProducer.setSocketConfigurer(socketConfigurer); - } - zeroMqMessageProducer.setOutputChannel(output); - zeroMqMessageProducer.setAutoStartup(false); - - return zeroMqMessageProducer; - } - - @Bean - public Supplier>> zeromqSupplier(ZeroMqMessageProducer adapter) { - return () -> Flux.from(output).doOnSubscribe(subscription -> adapter.start()); - } - -} diff --git a/functions/supplier/zeromq-supplier/src/main/java/org/springframework/cloud/fn/supplier/zeromq/ZeroMqSupplierProperties.java b/functions/supplier/zeromq-supplier/src/main/java/org/springframework/cloud/fn/supplier/zeromq/ZeroMqSupplierProperties.java deleted file mode 100644 index 08651c57b..000000000 --- a/functions/supplier/zeromq-supplier/src/main/java/org/springframework/cloud/fn/supplier/zeromq/ZeroMqSupplierProperties.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.zeromq; - -import java.time.Duration; - -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; -import org.hibernate.validator.constraints.Range; -import org.zeromq.SocketType; - -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.validation.annotation.Validated; - -/** - * - * @author Daniel Frey - * @since 3.1.0 - */ -@ConfigurationProperties("zeromq.supplier") -@Validated -public class ZeroMqSupplierProperties { - - /** - * The Socket Type the connection should make. - */ - private SocketType socketType = SocketType.SUB; - - /** - * Connection URL for to the ZeroMQ Socket. - */ - private String connectUrl; - - /** - * Bind Port for creating a ZeroMQ Socket; 0 selects a random port. - */ - private int bindPort; - - /** - * The delay to consume from the ZeroMQ Socket when no data received. - */ - private Duration consumeDelay = Duration.ofSeconds(1); - - /** - * The Topics to subscribe to. - */ - private String[] topics = {""}; - - /** - * @param socketType the {@link SocketType} to establish. - */ - public void setSocketType(SocketType socketType) { - this.socketType = socketType; - } - - @NotNull(message = "'socketType' is required") - public SocketType getSocketType() { - return socketType; - } - - @NotEmpty(message = "connectUrl is required like tcp://server:port") - public String getConnectUrl() { - return connectUrl; - } - - /** - * - * @param connectUrl The ZeroMQ server connect url - * - * @see org.springframework.integration.zeromq.inbound.ZeroMqMessageProducer#setConnectUrl(String) - */ - public void setConnectUrl(String connectUrl) { - this.connectUrl = connectUrl; - } - - @Range(min = 0, message = "'bindPort' must not be negative") - public int getBindPort() { - return bindPort; - } - - /** - * @param bindPort The Port to bind to on all interfaces - * - * @see org.springframework.integration.zeromq.inbound.ZeroMqMessageProducer#setBindPort(int) - */ - public void setBindPort(int bindPort) { - this.bindPort = bindPort; - } - - @NotNull(message = "'consumeDelay' is required") - public Duration getConsumeDelay() { - return consumeDelay; - } - - /** - * Specify a {@link Duration} to delay consumption when no data received. - * @param consumeDelay the {@link Duration} to delay consumption when empty. - */ - public void setConsumeDelay(Duration consumeDelay) { - this.consumeDelay = consumeDelay; - } - - public String[] getTopics() { - return topics; - } - - /** - * - * @param topics The ZeroMQ Topics to subscribe to - * - * @see org.springframework.integration.zeromq.inbound.ZeroMqMessageProducer#setTopics(String...) - */ - public void setTopics(String... topics) { - this.topics = topics; - } - -} diff --git a/functions/supplier/zeromq-supplier/src/test/java/org/springframework/cloud/fn/supplier/zeromq/ZeroMqSupplierConfigurationTests.java b/functions/supplier/zeromq-supplier/src/test/java/org/springframework/cloud/fn/supplier/zeromq/ZeroMqSupplierConfigurationTests.java deleted file mode 100644 index 7239fab8c..000000000 --- a/functions/supplier/zeromq-supplier/src/test/java/org/springframework/cloud/fn/supplier/zeromq/ZeroMqSupplierConfigurationTests.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2016-2020 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.cloud.fn.supplier.zeromq; - -import java.time.Duration; -import java.util.function.Supplier; - -import org.assertj.core.api.InstanceOfAssertFactories; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.zeromq.SocketType; -import org.zeromq.ZContext; -import org.zeromq.ZFrame; -import org.zeromq.ZMQ; -import org.zeromq.ZMsg; -import reactor.core.publisher.Flux; -import reactor.test.StepVerifier; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.messaging.Message; -import org.springframework.test.annotation.DirtiesContext; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * @author Daniel Frey - * since 3.1.0 - */ -@SpringBootTest(properties = {"zeromq.supplier.topics=test-topic"}) -@DirtiesContext -public class ZeroMqSupplierConfigurationTests { - - private static final ZContext CONTEXT = new ZContext(); - private static ZMQ.Socket socket; - - @Autowired - Supplier>> subject; - - @BeforeAll - static void setup() { - - String socketAddress = "tcp://*"; - socket = CONTEXT.createSocket(SocketType.PUB); - int bindPort = socket.bindToRandomPort(socketAddress); - - System.setProperty("zeromq.supplier.connectUrl", "tcp://localhost:" + bindPort); - - } - - @AfterAll - static void tearDown() { - socket.close(); - CONTEXT.close(); - } - - @Test - void testSubscriptionConfiguration() throws InterruptedException { - - StepVerifier stepVerifier = - StepVerifier.create(subject.get()) - .assertNext((message) -> - assertThat(message.getPayload()) - .asInstanceOf(InstanceOfAssertFactories.type(byte[].class)) - .isEqualTo("test".getBytes(ZMQ.CHARSET)) - ) - .thenCancel() - .verifyLater(); - - Thread.sleep(2000); - - ZMsg msg = ZMsg.newStringMsg("test"); - msg.wrap(new ZFrame("test-topic")); - msg.send(socket); - - stepVerifier.verify(Duration.ofSeconds(10)); - - } - - @SpringBootApplication - public static class ZeroMqSourceTestApplication { } - -} diff --git a/pom.xml b/pom.xml index 18ffbc843..7a593f60b 100644 --- a/pom.xml +++ b/pom.xml @@ -7,12 +7,11 @@ stream-applications 5.0.0-SNAPSHOT stream-applications - Functions and Infrastructure for stream applications + Infrastructure for Stream Applications pom stream-applications-build - functions applications stream-applications-release-train diff --git a/stream-applications-build/pom.xml b/stream-applications-build/pom.xml index 05bf37955..8b3b5adba 100644 --- a/stream-applications-build/pom.xml +++ b/stream-applications-build/pom.xml @@ -7,7 +7,7 @@ stream-applications-build 5.0.0-SNAPSHOT stream-applications-build - Common Parent for Functions and Applications + Common Parent for Stream Applications pom @@ -36,26 +36,24 @@ true 0.0.35 - 3.1.8 - 6.0.16 + 3.2.3 + 6.1.4 - 3.0.13 - 3.0.10 - 2022.0.4 + 2023.0.0 4.0.5 - 4.0.6 - 4.0.5 + 4.1.0 + 4.1.0 - 1.18.3 - 5.13.2 + 1.19.5 + 5.15.0 4.0.14 2.5.2 3.1.0 - 3.0.2 - 3.0.2 + 3.0.4 + 3.0.5 5.5.0 1 diff --git a/update-project-version.sh b/update-project-version.sh index f3ee1064a..e840b5550 100755 --- a/update-project-version.sh +++ b/update-project-version.sh @@ -39,8 +39,6 @@ $SCDIR/mvnw versions:set-property -pl :stream-applications-release-train \ -DskipResolution=true -s .settings.xml -DgenerateBackupPoms=false -B $VERBOSE \ -Dproperty=apps.version -DnewVersion="$VERSION" -$SCDIR/mvnw install -pl :function-dependencies -am -DskipTests -T 1C - OLD_RT_VERSION=$($SCDIR/mvnw help:evaluate -Dexpression=project.version -q -DforceStdout -f ./stream-applications-release-train 2> /dev/null) OLD_RT_VERSION=$(find_version "$OLD_RT_VERSION") echo "Release Train Version: [$OLD_RT_VERSION] -> [$RELEASE_TRAIN_VERSION]"