diff --git a/samples/CONFIGURATION.md b/samples/CONFIGURATION.md index 74240f32..66920f01 100644 --- a/samples/CONFIGURATION.md +++ b/samples/CONFIGURATION.md @@ -197,3 +197,30 @@ logging can be turned on. If this value is set to `true`, a specified error message written to error log and also sent as a response to request with relevant error code. +### External logging + +For further analysis, information about technical error can be stored to OpenSearch. +This logging can be turned on by adding configuration block to the application.yml: +``` + openSearchConfiguration: + url: + index: +``` + +Whenever an exception is thrown while executing any DSL step, an RuuterEvent object +is written to Opensearch with fields: +``` + "timestamp": timestamp in milliseconds, + "level": error level, "RUNTIME" for runtime DSL errors, "STARTUP" for startup parsing errors, + "dslName": name of DSL where error occurred, + "dslMethod": HTTP request method that triggered that DSL, + "stepName": name of DSL step here error occurred, + "statusCode": DSL HTTP return code (if applicable), + "errorCode": DSL HTTP error code (if applicable), + "requestParams": map of request parameters, + "requestHeaders": map of request header parameters, + "requestBody": map of request body values, + "message": error message (if applicable), + "stackTrace": Java stack trace for thrown exception. +``` + diff --git a/src/main/java/ee/buerokratt/ruuter/configuration/ApplicationProperties.java b/src/main/java/ee/buerokratt/ruuter/configuration/ApplicationProperties.java index d2744d68..8249e5d5 100644 --- a/src/main/java/ee/buerokratt/ruuter/configuration/ApplicationProperties.java +++ b/src/main/java/ee/buerokratt/ruuter/configuration/ApplicationProperties.java @@ -26,6 +26,7 @@ public class ApplicationProperties { private CORS cors; private DSL dsl; private InternalRequests internalRequests; + private OpenSearchConfiguration openSearchConfiguration; @Setter @Getter @@ -93,4 +94,11 @@ public static class InternalRequests { private List allowedIPs; private List allowedURLs; } + + @Getter + @Setter + public static class OpenSearchConfiguration { + private String url; + private String index; + } } diff --git a/src/main/java/ee/buerokratt/ruuter/domain/DslInstance.java b/src/main/java/ee/buerokratt/ruuter/domain/DslInstance.java index aecdd3e8..c1cea5eb 100644 --- a/src/main/java/ee/buerokratt/ruuter/domain/DslInstance.java +++ b/src/main/java/ee/buerokratt/ruuter/domain/DslInstance.java @@ -6,6 +6,8 @@ import ee.buerokratt.ruuter.helper.MappingHelper; import ee.buerokratt.ruuter.helper.ScriptingHelper; import ee.buerokratt.ruuter.service.DslService; +import ee.buerokratt.ruuter.service.OpenSearchSender; +import ee.buerokratt.ruuter.service.exception.StepExecutionException; import ee.buerokratt.ruuter.util.LoggingUtils; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -18,13 +20,14 @@ import java.util.Map; import java.util.Objects; import java.util.function.Function; +import java.util.logging.Logger; import java.util.stream.Collectors; - @Slf4j @Data @RequiredArgsConstructor public class DslInstance { private final String name; + private final String method; private final Map steps; private final Map requestBody; private final Map requestQuery; @@ -48,28 +51,58 @@ public class DslInstance { private String errorMessage; private HttpStatus errorStatus; + private final OpenSearchSender openSearchSender; + public void execute() { addGlobalIncomingHeadersToRequestHeaders(); List stepNames = steps.keySet().stream().toList(); recursions = stepNames.stream().collect(Collectors.toMap(Function.identity(), a -> 0)); try { executeStep(stepNames.get(0), stepNames); + } catch (Exception e) { LoggingUtils.logError(log, "Error executing DSL: %s".formatted(name), requestOrigin, "", e); clearReturnValues(); } } + private void logEvent(DslStep stepToExecute, String level, StackTraceElement[] stackTrace) { + openSearchSender.log( + new OpenSearchSender.RuuterEvent( + level, + getName(), + getMethod(), + stepToExecute.getName(), + getReturnStatus(), + (getErrorStatus() != null) ? Integer.valueOf(getErrorStatus().value()) : getReturnStatus(), + getRequestQuery(), + getRequestHeaders(), + getRequestBody(), + getErrorMessage(), + stackTrace + )); + } private void executeStep(String stepName, List stepNames) { DslStep stepToExecute = steps.get(stepName); if (!Objects.equals(recursions.get(stepName), getStepMaxRecursions(stepToExecute))) { - stepToExecute.execute(this); + try { + stepToExecute.execute(this); + } catch (StepExecutionException e) { + logEvent(stepToExecute, "RUNTIME", e.getStackTrace()); + + if (getProperties().getStopInCaseOfException() != null && getProperties().getStopInCaseOfException()) { + Thread.currentThread().interrupt(); + throw new StepExecutionException(name, e); + } + } + recursions.computeIfPresent(stepName, (k, v) -> v + 1); Integer maxRecursions = getStepMaxRecursions(stepToExecute); if (recursions.get(stepName) > 1 && maxRecursions != null && maxRecursions > currentLoopMaxRecursions) { setCurrentLoopMaxRecursions(maxRecursions); } } + if (stepToExecute.isReloadDsl()) { // Only allow reloading if it's enabled in configuration. if (properties.getDsl().isAllowDslReloading()) dslService.reloadDsls(); diff --git a/src/main/java/ee/buerokratt/ruuter/domain/steps/DslStep.java b/src/main/java/ee/buerokratt/ruuter/domain/steps/DslStep.java index 1aa1d263..a5d40ca8 100644 --- a/src/main/java/ee/buerokratt/ruuter/domain/steps/DslStep.java +++ b/src/main/java/ee/buerokratt/ruuter/domain/steps/DslStep.java @@ -28,7 +28,7 @@ public abstract class DslStep { @JsonAlias({"reloadDsls"}) private boolean reloadDsl = false; - public final void execute(DslInstance di) { + public final void execute(DslInstance di) throws StepExecutionException { Span newSpan = di.getTracer().nextSpan().name(name); long startTime = System.currentTimeMillis(); @@ -44,12 +44,12 @@ public final void execute(DslInstance di) { handleFailedResult(di); di.setErrorMessage("ScriptingException: " + see.getMessage()); di.setErrorStatus(HttpStatus.INTERNAL_SERVER_ERROR); + throw new StepExecutionException(name, see); } catch (Exception e) { handleFailedResult(di); - if (di.getProperties().getStopInCaseOfException() != null && di.getProperties().getStopInCaseOfException()) { - Thread.currentThread().interrupt(); - throw new StepExecutionException(name, e); - } + di.setErrorMessage(e.getMessage()); + di.setErrorStatus(HttpStatus.INTERNAL_SERVER_ERROR); + throw new StepExecutionException(name, e); } finally { newSpan.end(); } diff --git a/src/main/java/ee/buerokratt/ruuter/helper/DslMappingHelper.java b/src/main/java/ee/buerokratt/ruuter/helper/DslMappingHelper.java index def5f30b..2a286104 100644 --- a/src/main/java/ee/buerokratt/ruuter/helper/DslMappingHelper.java +++ b/src/main/java/ee/buerokratt/ruuter/helper/DslMappingHelper.java @@ -14,6 +14,7 @@ import ee.buerokratt.ruuter.domain.steps.http.HttpStep; import ee.buerokratt.ruuter.helper.exception.InvalidDslException; import ee.buerokratt.ruuter.helper.exception.InvalidDslStepException; +import ee.buerokratt.ruuter.service.OpenSearchSender; import ee.buerokratt.ruuter.util.FileUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Qualifier; @@ -39,10 +40,29 @@ public class DslMappingHelper { private Properties dslParameters; + private OpenSearchSender openSearchSender; + public DslMappingHelper(@Qualifier("ymlMapper") ObjectMapper mapper) { this.mapper = mapper; } + private void logEvent(String dslName, String dslMethod, String level, StackTraceElement[] stackTrace) { + openSearchSender.log( + new OpenSearchSender.RuuterEvent( + level, + dslName, + dslMethod, + null, + null, + null, + null, + null, + null, + null, + stackTrace + )); + } + public Map getDslSteps(Path path) { try { if (FileUtils.isFiletype(path, properties.getDsl().getProcessedFiletypes()) @@ -57,6 +77,11 @@ public Map getDslSteps(Path path) { throw new IllegalArgumentException(DSL_NOT_YML_FILE_ERROR_MESSAGE); } } catch (Exception e) { + String pathname = path.toString(); + logEvent(pathname.substring(1, pathname.indexOf('/', 1)), + pathname.substring(pathname.indexOf('/', 1)), + "STARTUP", + e.getStackTrace()); throw new InvalidDslException(path.toString(), e.getMessage(), e); } } diff --git a/src/main/java/ee/buerokratt/ruuter/service/DslService.java b/src/main/java/ee/buerokratt/ruuter/service/DslService.java index 2faf403d..c2dc4c9f 100644 --- a/src/main/java/ee/buerokratt/ruuter/service/DslService.java +++ b/src/main/java/ee/buerokratt/ruuter/service/DslService.java @@ -35,6 +35,7 @@ public class DslService { private final MappingHelper mappingHelper; private final HttpHelper httpHelper; private final Tracer tracer; + private final OpenSearchSender openSearchSender; private Map>> dsls; @@ -42,7 +43,9 @@ public class DslService { public static final String UNSUPPORTED_FILETYPE_ERROR_MESSAGE = "Unsupported filetype"; - public DslService(ApplicationProperties properties, DslMappingHelper dslMappingHelper, ScriptingHelper scriptingHelper, Tracer tracer, MappingHelper mappingHelper, HttpHelper httpHelper, ExternalForwardingHelper externalForwardingHelper) { + public DslService(ApplicationProperties properties, DslMappingHelper dslMappingHelper, ScriptingHelper scriptingHelper, + Tracer tracer, MappingHelper mappingHelper, HttpHelper httpHelper, + ExternalForwardingHelper externalForwardingHelper, OpenSearchSender openSearchSender) { this.dslMappingHelper = dslMappingHelper; this.properties = properties; this.dslMappingHelper.properties = properties; @@ -53,6 +56,7 @@ public DslService(ApplicationProperties properties, DslMappingHelper dslMappingH this.mappingHelper = mappingHelper; this.httpHelper = httpHelper; this.externalForwardingHelper = externalForwardingHelper; + this.openSearchSender = openSearchSender; } public void reloadDsls() { @@ -62,7 +66,8 @@ public void reloadDsls() { public Map>> getDsls(String configPath) { Map>> _dsls = - Arrays.stream(Objects.requireNonNull(new File(configPath).listFiles(File::isDirectory))).collect(toMap(File::getName, directory -> { + Arrays.stream(Objects.requireNonNull(new File(configPath).listFiles(File::isDirectory))) + .collect(toMap(File::getName, directory -> { try (Stream paths = Files.walk(getFolderPath(directory.toString())) .filter(path -> !FileUtils.isGuard(path)) .filter(path -> { @@ -98,7 +103,7 @@ public DslInstance execute(String dsl, String requestType, Map r return execute(dsl, requestType, requestBody, requestQuery, requestHeaders, requestOrigin, this.getClass().getName()); } public DslInstance execute(String dsl, String requestType, Map requestBody, Map requestQuery, Map requestHeaders, String requestOrigin, String contentType) { - DslInstance di = new DslInstance(dsl, dsls.get(requestType.toUpperCase()).get(dsl), requestBody, requestQuery, requestHeaders, requestOrigin, this, properties, scriptingHelper, mappingHelper, httpHelper, tracer); + DslInstance di = new DslInstance(dsl, requestType.toUpperCase(), dsls.get(requestType.toUpperCase()).get(dsl), requestBody, requestQuery, requestHeaders, requestOrigin, this, properties, scriptingHelper, mappingHelper, httpHelper, tracer, openSearchSender); if (di.getSteps() != null) { LoggingUtils.logInfo(log, "Request received for DSL: %s".formatted(dsl), requestOrigin, INCOMING_REQUEST); @@ -108,7 +113,7 @@ public DslInstance execute(String dsl, String requestType, Map r return di; }; - DslInstance guard = new DslInstance(dsl, getGuard(requestType.toUpperCase(), dsl), requestBody, requestBody, requestHeaders, requestOrigin, this, properties, scriptingHelper, mappingHelper, httpHelper, tracer); + DslInstance guard = new DslInstance(dsl, requestType.toUpperCase(), getGuard(requestType.toUpperCase(), dsl), requestBody, requestBody, requestHeaders, requestOrigin, this, properties, scriptingHelper, mappingHelper, httpHelper, tracer, openSearchSender); if (guard != null && guard.getSteps() != null) { LoggingUtils.logInfo(log, "Executing guard for DSL: %s".formatted(dsl), requestOrigin, INCOMING_REQUEST); guard.execute(); diff --git a/src/main/java/ee/buerokratt/ruuter/service/OpenSearchSender.java b/src/main/java/ee/buerokratt/ruuter/service/OpenSearchSender.java new file mode 100644 index 00000000..0d6bd830 --- /dev/null +++ b/src/main/java/ee/buerokratt/ruuter/service/OpenSearchSender.java @@ -0,0 +1,82 @@ +package ee.buerokratt.ruuter.service; + +import ee.buerokratt.ruuter.configuration.ApplicationProperties; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.web.reactive.function.client.WebClient; +import reactor.core.publisher.Mono; + +import java.time.Instant; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +@Slf4j +@Service +public class OpenSearchSender { + + @RequiredArgsConstructor + @Data + public static class RuuterEvent { + Long timestamp = Instant.now().toEpochMilli(); + final String level; + + final String dslName; + final String dslMethod; + + final String stepName; + + final Integer statusCode; + final Integer errorCode; + + final Map requestParams; + final Map requestHeaders; + final Map requestBody; + + final String message; + + final StackTraceElement[] stackTrace; + } + + private ApplicationProperties properties; + + private WebClient webClient; + private String indexName; + + private boolean loggingEnabled ; + + public OpenSearchSender(ApplicationProperties properties) { + this.properties = properties; + loggingEnabled = properties.getOpenSearchConfiguration() != null; + if (!loggingEnabled) { + log.warn("OpenSearch logging disabled"); + } + } + + private void createClient() { + webClient = WebClient.create(properties.getOpenSearchConfiguration().getUrl()); + indexName = properties.getOpenSearchConfiguration().getIndex(); + } + + public void log(RuuterEvent ruuterEvent) { + if (properties.getOpenSearchConfiguration() == null) { + return; + } + + if (webClient == null) + createClient(); + + webClient.post() + .uri("/{logIndex}/_doc", indexName) + .bodyValue(ruuterEvent) + .retrieve() + .bodyToMono(Void.class) + .onErrorResume(exception -> { + log.error("Unable to send log to OpenSearch", exception); + return Mono.empty(); + }) + .subscribe(); + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index d000e5e4..0c0b2132 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -31,6 +31,9 @@ application: internalRequests: allowedIPs: ["127.0.0.1", "192.168.0.1", "172.21.0.1"] allowedURLs: ["http://localhost/internalTest"] +# openSearchConfiguration: +# url: http://host.docker.internal:9200 +# index: ruuterlog spring: application: