From 6260c15b179a4f5715022deee5e85d448760a499 Mon Sep 17 00:00:00 2001 From: RayDNoper Date: Wed, 16 Oct 2024 15:15:22 +0300 Subject: [PATCH 1/3] #318 Added functionality to do non-blocking HTTP requests --- .../ruuter/domain/steps/http/HttpGetStep.java | 2 +- .../domain/steps/http/HttpPostStep.java | 2 +- .../ruuter/domain/steps/http/HttpStep.java | 2 +- .../helper/ExternalForwardingHelper.java | 4 +- .../buerokratt/ruuter/helper/HttpHelper.java | 38 +++++++++++-------- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpGetStep.java b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpGetStep.java index f781c357..e59dda19 100644 --- a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpGetStep.java +++ b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpGetStep.java @@ -22,7 +22,7 @@ public ResponseEntity getRequestResponse(DslInstance di) { Map evaluatedQuery = di.getScriptingHelper().evaluateScripts(args.getQuery(), di.getContext(), di.getRequestBody(), di.getRequestQuery(), di.getRequestHeaders()); Map evaluatedHeaders = di.getScriptingHelper().evaluateScripts(args.getHeaders(), di.getContext(), di.getRequestBody(), di.getRequestQuery(), di.getRequestHeaders()); Map mappedHeaders = di.getMappingHelper().convertMapObjectValuesToString(evaluatedHeaders); - return di.getHttpHelper().doMethod(HttpMethod.GET, evaluatedURL, evaluatedQuery, null, mappedHeaders, args.getContentType() , null, getLimit(), di, args.isDynamicParameters()); + return di.getHttpHelper().doMethod(HttpMethod.GET, evaluatedURL, evaluatedQuery, null, mappedHeaders, args.getContentType() , null, getLimit(), di, args.isDynamicParameters(), resultName != null && !resultName.isEmpty() ); } @Override diff --git a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpPostStep.java b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpPostStep.java index 7e4a49ae..d0efbec6 100644 --- a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpPostStep.java +++ b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpPostStep.java @@ -32,7 +32,7 @@ protected ResponseEntity getRequestResponse(DslInstance di) { args.getContentType(), "plaintext".equals(args.getContentType()) ? args.getPlaintext() : null, getLimit(), di, - args.isDynamicParameters()); + args.isDynamicParameters(),resultName != null && !resultName.isEmpty() ); } @Override diff --git a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpStep.java b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpStep.java index 4dbb62b1..0d3fc116 100644 --- a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpStep.java +++ b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpStep.java @@ -77,7 +77,7 @@ protected void executeStepAction(DslInstance di) { @Override public void handleFailedResult(DslInstance di) { super.handleFailedResult(di); - HttpStepResult stepResult = (HttpStepResult) di.getContext().get(resultName); + HttpStepResult stepResult = (HttpStepResult) di.getContext().get( resultName); if (stepResult != null && !isAllowedHttpStatusCode(di, stepResult.getResponse().getStatusCodeValue())) { DefaultHttpDsl globalHttpExceptionDsl = di.getProperties().getDefaultDslInCaseOfException(); if (localHttpExceptionDslExists()) { diff --git a/src/main/java/ee/buerokratt/ruuter/helper/ExternalForwardingHelper.java b/src/main/java/ee/buerokratt/ruuter/helper/ExternalForwardingHelper.java index bcce507d..5c299449 100644 --- a/src/main/java/ee/buerokratt/ruuter/helper/ExternalForwardingHelper.java +++ b/src/main/java/ee/buerokratt/ruuter/helper/ExternalForwardingHelper.java @@ -58,10 +58,10 @@ public ResponseEntity forwardRequest(String dsl, Map req .toUriString(); if (methodType.equals(HttpMethod.POST.name())) { - return httpHelper.doMethod(HttpMethod.POST,forwardingUrl, query, body, headers, contentType, null, null, di, false); + return httpHelper.doMethod(HttpMethod.POST,forwardingUrl, query, body, headers, contentType, null, null, di, false, true ); } if (methodType.equals(HttpMethod.GET.name())) { - return httpHelper.doMethod(HttpMethod.GET, forwardingUrl, query,null, headers, null, null, null, di, false); + return httpHelper.doMethod(HttpMethod.GET, forwardingUrl, query,null, headers, null, null, null, di, false, true ); } throw new InvalidHttpMethodTypeException(methodType); } diff --git a/src/main/java/ee/buerokratt/ruuter/helper/HttpHelper.java b/src/main/java/ee/buerokratt/ruuter/helper/HttpHelper.java index 526b889e..f83ba944 100644 --- a/src/main/java/ee/buerokratt/ruuter/helper/HttpHelper.java +++ b/src/main/java/ee/buerokratt/ruuter/helper/HttpHelper.java @@ -2,7 +2,6 @@ import ee.buerokratt.ruuter.configuration.ApplicationProperties; import ee.buerokratt.ruuter.domain.DslInstance; -import ee.buerokratt.ruuter.util.LoggingUtils; import io.netty.channel.ChannelOption; import io.netty.handler.timeout.ReadTimeoutHandler; import io.netty.handler.timeout.WriteTimeoutHandler; @@ -22,6 +21,7 @@ import org.springframework.web.reactive.function.client.WebClient; import org.springframework.web.reactive.function.client.WebClientResponseException; +import reactor.core.Disposable; import reactor.core.publisher.Mono; import reactor.netty.http.client.HttpClient; @@ -46,24 +46,25 @@ public class HttpHelper { public ResponseEntity doPost(String url, Map body, Map query, Map headers, DslInstance di, boolean dynamicBody) { return doPost(url, body, query, headers, this.getClass().getName(), di, dynamicBody); } + public ResponseEntity doPost(String url, Map body, Map query, Map headers, String contentType, DslInstance di, boolean dynamicBody) { - return doMethod(POST, url, query, body,headers, contentType, null, null, di, dynamicBody); + return doMethod(POST, url, query, body,headers, contentType, null, null, di, dynamicBody, true); } public ResponseEntity doPostPlaintext(String url, Map body, Map query, Map headers, String plaintext, DslInstance di) { - return doMethod(POST, url, body, query, headers, "plaintext", plaintext, null, di, false); + return doMethod(POST, url, body, query, headers, "plaintext", plaintext, null, di, false, true); } public ResponseEntity doGet(String url, Map query, Map headers, DslInstance di) { - return doMethod(HttpMethod.GET, url, query, null, headers, null, null, null, di, false); + return doMethod(HttpMethod.GET, url, query, null, headers, null, null, null, di, false, true ); } - public ResponseEntity doPut(String url, Map body, Map query, Map headers, String contentType, DslInstance di, boolean dynamicBody) { - return doMethod(HttpMethod.PUT, url, query, body,headers, contentType, null, null, di, dynamicBody); + public ResponseEntity doPut(String url, Map body, Map query, Map headers, String contentType, DslInstance di, boolean dynamicBody, boolean blockResult) { + return doMethod(HttpMethod.PUT, url, query, body,headers, contentType, null, null, di, dynamicBody, true); } public ResponseEntity doDelete(String url, Map body, Map query, Map headers, String contentType, DslInstance di) { - return doMethod(HttpMethod.DELETE, url, query, body, headers, contentType, null, null, di, false); + return doMethod(HttpMethod.DELETE, url, query, body, headers, contentType, null, null, di, false, true); } public ResponseEntity doMethod(HttpMethod method, @@ -75,7 +76,7 @@ public ResponseEntity doMethod(HttpMethod method, String plaintextValue, Integer limit, DslInstance instance, - boolean dynamicBody) { + boolean dynamicBody, boolean blockResult) { try { MultiValueMap qp = new LinkedMultiValueMap<>( query.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e-> Arrays.asList(e.getValue().toString())))); @@ -126,15 +127,15 @@ public ResponseEntity doMethod(HttpMethod method, } Integer finalLimit = limit == null ? properties.getHttpResponseSizeLimit() : limit; - return WebClient.builder() + Mono> retrieve = WebClient.builder() .filter((request, next) -> !("json_override".equals(contentType)) ? next.exchange(request) - :next.exchange(request) - .flatMap(response -> Mono.just(response.mutate() - .headers(httpHeaders -> httpHeaders.set(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)) - .build()))) + : next.exchange(request) + .flatMap(response -> Mono.just(response.mutate() + .headers(httpHeaders -> httpHeaders.set(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)) + .build()))) .exchangeStrategies( ExchangeStrategies.builder().codecs( - configurer -> configurer.defaultCodecs().maxInMemorySize(finalLimit * 1024 )).build()) + configurer -> configurer.defaultCodecs().maxInMemorySize(finalLimit * 1024)).build()) .clientConnector(new ReactorClientHttpConnector(getHttpClient())).build() .method(method) .uri(url, uriBuilder -> uriBuilder.queryParams(qp).build()) @@ -142,8 +143,13 @@ public ResponseEntity doMethod(HttpMethod method, .body(bodyValue) .header(HttpHeaders.CONTENT_TYPE, mediaType) .retrieve() - .toEntity(Object.class) - .block(); + .toEntity(Object.class); + if (blockResult) + return retrieve.block(); + else { + Disposable dis = retrieve.subscribe(); + return ResponseEntity.ok(null); + } } catch (WebClientResponseException e) { log.error("Failed HTTP request: ", e); return new ResponseEntity<>(e.getStatusText(), e.getStatusCode()); From a50fabeaee88387e0bd83a96d068d74de6d250e1 Mon Sep 17 00:00:00 2001 From: RayDNoper Date: Thu, 24 Oct 2024 21:13:55 +0300 Subject: [PATCH 2/3] Add HTTP timeout parameter to HTTPStep and application properties --- .../configuration/ApplicationProperties.java | 2 + .../domain/steps/conditional/SwitchStep.java | 9 +++++ .../ruuter/domain/steps/http/HttpGetStep.java | 2 +- .../domain/steps/http/HttpPostStep.java | 3 +- .../ruuter/domain/steps/http/HttpStep.java | 1 + .../helper/ExternalForwardingHelper.java | 4 +- .../buerokratt/ruuter/helper/HttpHelper.java | 40 +++++++++++-------- 7 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/main/java/ee/buerokratt/ruuter/configuration/ApplicationProperties.java b/src/main/java/ee/buerokratt/ruuter/configuration/ApplicationProperties.java index b254e833..78e259cb 100644 --- a/src/main/java/ee/buerokratt/ruuter/configuration/ApplicationProperties.java +++ b/src/main/java/ee/buerokratt/ruuter/configuration/ApplicationProperties.java @@ -32,6 +32,8 @@ public class ApplicationProperties { private Boolean allowDuplicateRequestKeys; + private Integer httpRequestTimeout; + @Setter @Getter public static class FinalResponse { diff --git a/src/main/java/ee/buerokratt/ruuter/domain/steps/conditional/SwitchStep.java b/src/main/java/ee/buerokratt/ruuter/domain/steps/conditional/SwitchStep.java index 262a0f2e..8163efc0 100644 --- a/src/main/java/ee/buerokratt/ruuter/domain/steps/conditional/SwitchStep.java +++ b/src/main/java/ee/buerokratt/ruuter/domain/steps/conditional/SwitchStep.java @@ -25,10 +25,19 @@ public class SwitchStep extends DslStep { @Override protected void executeStepAction(DslInstance di) { +/* + Optional correctStatement = + conditions.stream() + .filter(condition -> condition.getConditionStatement().equals(di.getRequestQuery().get("metric"))) + .findFirst(); + + System.out.println("METRIC: " + di.getRequestQuery().get("metric")); +*/ ScriptingHelper scriptingHelper = di.getScriptingHelper(); Optional correctStatement = conditions.stream() .filter(condition -> Boolean.TRUE.equals(scriptingHelper.evaluateScripts(condition.getConditionStatement(), di.getContext(), di.getRequestBody(), di.getRequestQuery(), di.getRequestHeaders()))) .findFirst(); + correctStatement.ifPresentOrElse(condition -> this.setNextStepName(condition.getNextStepName()), () -> this.setNextStepName(elseNextStepName)); } diff --git a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpGetStep.java b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpGetStep.java index f781c357..11fbac13 100644 --- a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpGetStep.java +++ b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpGetStep.java @@ -22,7 +22,7 @@ public ResponseEntity getRequestResponse(DslInstance di) { Map evaluatedQuery = di.getScriptingHelper().evaluateScripts(args.getQuery(), di.getContext(), di.getRequestBody(), di.getRequestQuery(), di.getRequestHeaders()); Map evaluatedHeaders = di.getScriptingHelper().evaluateScripts(args.getHeaders(), di.getContext(), di.getRequestBody(), di.getRequestQuery(), di.getRequestHeaders()); Map mappedHeaders = di.getMappingHelper().convertMapObjectValuesToString(evaluatedHeaders); - return di.getHttpHelper().doMethod(HttpMethod.GET, evaluatedURL, evaluatedQuery, null, mappedHeaders, args.getContentType() , null, getLimit(), di, args.isDynamicParameters()); + return di.getHttpHelper().doMethod(HttpMethod.GET, evaluatedURL, evaluatedQuery, null, mappedHeaders, args.getContentType() , null, getLimit(), di, args.isDynamicParameters(), getTimeout()); } @Override diff --git a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpPostStep.java b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpPostStep.java index 7e4a49ae..d21cfaef 100644 --- a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpPostStep.java +++ b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpPostStep.java @@ -32,7 +32,8 @@ protected ResponseEntity getRequestResponse(DslInstance di) { args.getContentType(), "plaintext".equals(args.getContentType()) ? args.getPlaintext() : null, getLimit(), di, - args.isDynamicParameters()); + args.isDynamicParameters(), + getTimeout()); } @Override diff --git a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpStep.java b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpStep.java index 4dbb62b1..4290ed9d 100644 --- a/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpStep.java +++ b/src/main/java/ee/buerokratt/ruuter/domain/steps/http/HttpStep.java @@ -42,6 +42,7 @@ public abstract class HttpStep extends DslStep { protected Logging logging; protected Integer limit; + protected Integer timeout; @JsonAlias("error") protected String onErrorStep; diff --git a/src/main/java/ee/buerokratt/ruuter/helper/ExternalForwardingHelper.java b/src/main/java/ee/buerokratt/ruuter/helper/ExternalForwardingHelper.java index bcce507d..3b1ff001 100644 --- a/src/main/java/ee/buerokratt/ruuter/helper/ExternalForwardingHelper.java +++ b/src/main/java/ee/buerokratt/ruuter/helper/ExternalForwardingHelper.java @@ -58,10 +58,10 @@ public ResponseEntity forwardRequest(String dsl, Map req .toUriString(); if (methodType.equals(HttpMethod.POST.name())) { - return httpHelper.doMethod(HttpMethod.POST,forwardingUrl, query, body, headers, contentType, null, null, di, false); + return httpHelper.doMethod(HttpMethod.POST,forwardingUrl, query, body, headers, contentType, null, null, di, false, null); } if (methodType.equals(HttpMethod.GET.name())) { - return httpHelper.doMethod(HttpMethod.GET, forwardingUrl, query,null, headers, null, null, null, di, false); + return httpHelper.doMethod(HttpMethod.GET, forwardingUrl, query,null, headers, null, null, null, di, false, null); } throw new InvalidHttpMethodTypeException(methodType); } diff --git a/src/main/java/ee/buerokratt/ruuter/helper/HttpHelper.java b/src/main/java/ee/buerokratt/ruuter/helper/HttpHelper.java index 526b889e..6218d4ac 100644 --- a/src/main/java/ee/buerokratt/ruuter/helper/HttpHelper.java +++ b/src/main/java/ee/buerokratt/ruuter/helper/HttpHelper.java @@ -43,27 +43,28 @@ public class HttpHelper { final private ScriptingHelper scriptingHelper; - public ResponseEntity doPost(String url, Map body, Map query, Map headers, DslInstance di, boolean dynamicBody) { - return doPost(url, body, query, headers, this.getClass().getName(), di, dynamicBody); + public ResponseEntity doPost(String url, Map body, Map query, Map headers, DslInstance di, boolean dynamicBody, Integer timeout) { + return doPost(url, body, query, headers, this.getClass().getName(), di, dynamicBody, timeout); } - public ResponseEntity doPost(String url, Map body, Map query, Map headers, String contentType, DslInstance di, boolean dynamicBody) { - return doMethod(POST, url, query, body,headers, contentType, null, null, di, dynamicBody); + + public ResponseEntity doPost(String url, Map body, Map query, Map headers, String contentType, DslInstance di, boolean dynamicBody, Integer timeout) { + return doMethod(POST, url, query, body,headers, contentType, null, null, di, dynamicBody, timeout); } public ResponseEntity doPostPlaintext(String url, Map body, Map query, Map headers, String plaintext, DslInstance di) { - return doMethod(POST, url, body, query, headers, "plaintext", plaintext, null, di, false); + return doMethod(POST, url, body, query, headers, "plaintext", plaintext, null, di, false, null); } - public ResponseEntity doGet(String url, Map query, Map headers, DslInstance di) { - return doMethod(HttpMethod.GET, url, query, null, headers, null, null, null, di, false); + public ResponseEntity doGet(String url, Map query, Map headers, DslInstance di, Integer timeout) { + return doMethod(HttpMethod.GET, url, query, null, headers, null, null, null, di, false, timeout); } public ResponseEntity doPut(String url, Map body, Map query, Map headers, String contentType, DslInstance di, boolean dynamicBody) { - return doMethod(HttpMethod.PUT, url, query, body,headers, contentType, null, null, di, dynamicBody); + return doMethod(HttpMethod.PUT, url, query, body,headers, contentType, null, null, di, dynamicBody, null); } public ResponseEntity doDelete(String url, Map body, Map query, Map headers, String contentType, DslInstance di) { - return doMethod(HttpMethod.DELETE, url, query, body, headers, contentType, null, null, di, false); + return doMethod(HttpMethod.DELETE, url, query, body, headers, contentType, null, null, di, false, null); } public ResponseEntity doMethod(HttpMethod method, @@ -75,7 +76,8 @@ public ResponseEntity doMethod(HttpMethod method, String plaintextValue, Integer limit, DslInstance instance, - boolean dynamicBody) { + boolean dynamicBody, + Integer timeout) { try { MultiValueMap qp = new LinkedMultiValueMap<>( query.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e-> Arrays.asList(e.getValue().toString())))); @@ -135,7 +137,7 @@ public ResponseEntity doMethod(HttpMethod method, .exchangeStrategies( ExchangeStrategies.builder().codecs( configurer -> configurer.defaultCodecs().maxInMemorySize(finalLimit * 1024 )).build()) - .clientConnector(new ReactorClientHttpConnector(getHttpClient())).build() + .clientConnector(new ReactorClientHttpConnector(getHttpClient(timeout ))).build() .method(method) .uri(url, uriBuilder -> uriBuilder.queryParams(qp).build()) .headers(httpHeaders -> addHeadersIfNotNull(headers, httpHeaders)) @@ -156,12 +158,18 @@ private void addHeadersIfNotNull(Map headers, HttpHeaders httpHe } } - private HttpClient getHttpClient() { + private HttpClient getHttpClient(Integer timeout) { + final Integer finalTimeout = timeout != null ? timeout : + properties.getHttpRequestTimeout() != null ? properties.getHttpRequestTimeout() : + 15000; + + log.info("HTTP request effective timeout: "+ finalTimeout); + return HttpClient.create() - .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 15000) - .responseTimeout(Duration.ofMillis(15000)) + .option(ChannelOption.CONNECT_TIMEOUT_MILLIS, finalTimeout) + .responseTimeout(Duration.ofMillis(finalTimeout)) .doOnConnected(conn -> - conn.addHandlerLast(new ReadTimeoutHandler(15000, TimeUnit.MILLISECONDS)) - .addHandlerLast(new WriteTimeoutHandler(15000, TimeUnit.MILLISECONDS))); + conn.addHandlerLast(new ReadTimeoutHandler(finalTimeout, TimeUnit.MILLISECONDS)) + .addHandlerLast(new WriteTimeoutHandler(finalTimeout, TimeUnit.MILLISECONDS))); } } From 46ecac8ab1cd5355eac45c36ed37afcf3e11e3d3 Mon Sep 17 00:00:00 2001 From: RayDNoper Date: Thu, 24 Oct 2024 22:31:34 +0300 Subject: [PATCH 3/3] Update documentation --- samples/steps/http-get.md | 5 +++++ samples/steps/http-post.md | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/samples/steps/http-get.md b/samples/steps/http-get.md index 553bb401..b7cdf653 100644 --- a/samples/steps/http-get.md +++ b/samples/steps/http-get.md @@ -27,7 +27,12 @@ get_step: * `headers` * *..desired header values* - Scripts can be used for headers values * `result` - name of the variable to store the response of the query in, for use in other steps + - __NB!__ if `result` field is missing, the request will be done immediately and + non-blockingly, so the DSL execution does not wait for response. Any data that would + be sent with response is not handled by Ruuter. * `limit` - limit the size of allowed response in kilobytes (default value is configured in application.yaml) +* `timeout`- (in milliseconds) overwrites http request timeout set in application properties (or if not +defined there, 15000ms) #### How responses are stored with the result field diff --git a/samples/steps/http-post.md b/samples/steps/http-post.md index 67c67aae..d22515f0 100644 --- a/samples/steps/http-post.md +++ b/samples/steps/http-post.md @@ -31,6 +31,9 @@ post_step: * `headers` * *..desired header values* - Scripts can be used for headers values * `result` - name of the variable to store the response of the query in, for use in other steps + - __NB!__ if `result` field is missing, the request will be done immediately and +non-blockingly, so the DSL execution does not wait for response. Any data that would +be sent with response is not handled by Ruuter. * `contentType` - specifies the contenttype to use, currently allowed values: * `"plaintext"` - uses field `plaintext` and mediaType 'text/plain' * `"formdata"` @@ -44,6 +47,8 @@ the key, for example `file:projectdata:Project.csv`, and mediatype "multipart/fo * If left empty, `body` is posted as JSON and 'application/json' is used as mediatype. * `plaintext` - used instead of `body` if a singular plaintext value is needed to be sent * `limit` - limit the size of allowed response in kilobytes (default value is configured in application.yaml) +* `timeout`- (in milliseconds) overwrites http request timeout set in application properties (or if not + defined there, 15000ms) * ***Note: POST step responses are stored the same way as [GET step responses](./http-get.md#How-responses-are-stored-with-the-result-field)***