From 3f86ea4042349e17d98d1d0de2733768c17bd388 Mon Sep 17 00:00:00 2001 From: georgweiss Date: Thu, 1 Jun 2023 14:16:57 +0200 Subject: [PATCH 1/8] Composite upload POC --- .../java/org/phoebus/olog/Application.java | 7 +- .../olog/FileUploadSizeExceededHandler.java | 43 ++++++++ .../org/phoebus/olog/HttpConnectorConfig.java | 3 + .../java/org/phoebus/olog/LogResource.java | 99 ++++++++++++++++++- 4 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java diff --git a/src/main/java/org/phoebus/olog/Application.java b/src/main/java/org/phoebus/olog/Application.java index c8cf399..db963ce 100644 --- a/src/main/java/org/phoebus/olog/Application.java +++ b/src/main/java/org/phoebus/olog/Application.java @@ -1,12 +1,18 @@ package org.phoebus.olog; import co.elastic.clients.elasticsearch.ElasticsearchClient; +import org.apache.catalina.connector.Connector; +import org.apache.coyote.http11.AbstractHttp11Protocol; import org.phoebus.olog.notification.LogEntryNotifier; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.web.embedded.tomcat.TomcatConnectorCustomizer; +import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; +import org.springframework.boot.web.servlet.server.ServletWebServerFactory; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; @@ -135,5 +141,4 @@ public AcceptHeaderResolver acceptHeaderResolver(){ public LogEntryValidator logEntryValidator(){ return new LogEntryValidator(); } - } \ No newline at end of file diff --git a/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java b/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java new file mode 100644 index 0000000..5c9e486 --- /dev/null +++ b/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2020 European Spallation Source ERIC. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +package org.phoebus.olog; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.context.request.WebRequest; +import org.springframework.web.multipart.MaxUploadSizeExceededException; +import org.springframework.web.multipart.MultipartException; + +@ControllerAdvice +public class FileUploadSizeExceededHandler { + + @ExceptionHandler(MaxUploadSizeExceededException.class) + @ResponseStatus(value = HttpStatus.BAD_REQUEST, reason = "Too large") + public void handleMaxSizeExceededException(RuntimeException ex, WebRequest request) { + System.out.println(); + } + + @ExceptionHandler(MultipartException.class) + @ResponseStatus(value = HttpStatus.BAD_REQUEST, reason = "Too large") + public void handleMultipartException(RuntimeException ex, WebRequest request) { + System.out.println(); + } +} diff --git a/src/main/java/org/phoebus/olog/HttpConnectorConfig.java b/src/main/java/org/phoebus/olog/HttpConnectorConfig.java index 03e60e5..08b9254 100644 --- a/src/main/java/org/phoebus/olog/HttpConnectorConfig.java +++ b/src/main/java/org/phoebus/olog/HttpConnectorConfig.java @@ -1,8 +1,10 @@ package org.phoebus.olog; import org.apache.catalina.connector.Connector; +import org.apache.coyote.http11.AbstractHttp11Protocol; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.web.embedded.tomcat.TomcatConnectorCustomizer; import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; import org.springframework.boot.web.servlet.server.ServletWebServerFactory; import org.springframework.context.annotation.Bean; @@ -30,6 +32,7 @@ private Connector getHttpConnector() { Connector connector = new Connector("org.apache.coyote.http11.Http11NioProtocol"); connector.setScheme("http"); connector.setPort(port); + ((AbstractHttp11Protocol )connector.getProtocolHandler()).setMaxSwallowSize(50 * 1024 * 1024); return connector; } } \ No newline at end of file diff --git a/src/main/java/org/phoebus/olog/LogResource.java b/src/main/java/org/phoebus/olog/LogResource.java index 055ad9c..3f29426 100644 --- a/src/main/java/org/phoebus/olog/LogResource.java +++ b/src/main/java/org/phoebus/olog/LogResource.java @@ -26,8 +26,11 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.util.FileCopyUtils; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.WebDataBinder; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.InitBinder; import org.springframework.web.bind.annotation.PathVariable; @@ -39,10 +42,19 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartException; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.server.ResponseStatusException; +import org.springframework.web.servlet.mvc.support.RedirectAttributes; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.MessageDigest; import java.security.Principal; import java.time.Instant; import java.time.temporal.TemporalAmount; @@ -59,6 +71,8 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.zip.CRC32; +import java.util.zip.CheckedInputStream; import static org.phoebus.olog.OlogResourceDescriptors.LOG_RESOURCE_URI; import static org.phoebus.util.time.TimestampFormats.MILLI_FORMAT; @@ -269,9 +283,38 @@ public Log createLog(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, required = return newLogEntry; } + @PostMapping("/composite") + public Log createCompositeLog(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, required = false, defaultValue = "n/a") String clientInfo, + @RequestParam(value = "markup", required = false) String markup, + @RequestParam(value = "inReplyTo", required = false, defaultValue = "-1") String inReplyTo, + @RequestPart("logEntry") Log log, + @RequestPart("attachmentData") List attachmentData, + @RequestPart("files") MultipartFile[] files, + @AuthenticationPrincipal Principal principal) { + + Log createdLog = createLog(clientInfo, markup, log, inReplyTo, principal); + + for(MultipartFile multipartFile : files){ + try { + String checksum = getMD5Checksum(multipartFile.getInputStream()); + Optional attachmentLight = attachmentData.stream().filter(a -> a.checksum.equals(checksum)).findFirst(); + if(attachmentLight.isPresent()){ + uploadAttachment(Long.toString(createdLog.getId()), multipartFile, attachmentLight.get().fileName, + attachmentLight.get().id, + attachmentLight.get().fileMetaDataDescription); + attachmentData.remove(attachmentLight.get()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + return createdLog; + } + @PostMapping("/attachments/{logId}") public Log uploadAttachment(@PathVariable String logId, - @RequestPart("file") MultipartFile file, + @RequestPart("files") MultipartFile file, @RequestPart("filename") String filename, @RequestPart(value = "id", required = false) String id, @RequestPart(value = "fileMetadataDescription", required = false) String fileMetadataDescription) { @@ -526,4 +569,58 @@ private void handleReply(String originalLogEntryId, Log log) { throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Cannot reply to log entry " + originalLogEntryId + " as it does not exist"); } } + + private static class AttachmentLight{ + private String id; + private String fileName; + private String fileMetaDataDescription; + + private String checksum; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public String getFileMetaDataDescription() { + return fileMetaDataDescription; + } + + public void setFileMetaDataDescription(String fileMetaDataDescription) { + this.fileMetaDataDescription = fileMetaDataDescription; + } + + public String getChecksum() { + return checksum; + } + + public void setChecksum(String checksum) { + this.checksum = checksum; + } + } + + private long getChecksumCRC32(InputStream inputStream, int bufferSize) + throws IOException { + CheckedInputStream checkedInputStream = new CheckedInputStream(inputStream, new CRC32()); + byte[] buffer = new byte[bufferSize]; + while (checkedInputStream.read(buffer, 0, buffer.length) >= 0) {} + return checkedInputStream.getChecksum().getValue(); + } + + private static String getMD5Checksum(InputStream inputStream) throws Exception{ + byte[] data = FileCopyUtils.copyToByteArray(inputStream); + byte[] hash = MessageDigest.getInstance("MD5").digest(data); + return new BigInteger(1, hash).toString(16); + } } From 90d1777bfff33cab4c96088914bb51d33bfd707d Mon Sep 17 00:00:00 2001 From: georgweiss Date: Thu, 8 Jun 2023 09:50:01 +0200 Subject: [PATCH 2/8] Minor updates to better handle request exceeding size limits --- .../olog/FileUploadSizeExceededHandler.java | 27 ++++++---- .../org/phoebus/olog/HttpConnectorConfig.java | 7 ++- .../java/org/phoebus/olog/InfoResource.java | 19 +++++-- .../java/org/phoebus/olog/LogResource.java | 52 ++++++++----------- src/main/resources/application.properties | 8 +-- 5 files changed, 65 insertions(+), 48 deletions(-) diff --git a/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java b/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java index 5c9e486..3f27ae3 100644 --- a/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java +++ b/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java @@ -18,7 +18,10 @@ package org.phoebus.olog; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.ResponseStatus; @@ -26,18 +29,24 @@ import org.springframework.web.multipart.MaxUploadSizeExceededException; import org.springframework.web.multipart.MultipartException; +/** + * Handles request exceeding configured sizes (spring.servlet.multipart.max-file-size and + * spring.servlet.multipart.max-request-size). In such cases client will get an HTTP 413 (payload too large) response + * with a (hopefully) useful message. + */ @ControllerAdvice +@SuppressWarnings("unused") public class FileUploadSizeExceededHandler { - @ExceptionHandler(MaxUploadSizeExceededException.class) - @ResponseStatus(value = HttpStatus.BAD_REQUEST, reason = "Too large") - public void handleMaxSizeExceededException(RuntimeException ex, WebRequest request) { - System.out.println(); - } + @Value("${spring.servlet.multipart.max-file-size}") + private String maxFileSize; - @ExceptionHandler(MultipartException.class) - @ResponseStatus(value = HttpStatus.BAD_REQUEST, reason = "Too large") - public void handleMultipartException(RuntimeException ex, WebRequest request) { - System.out.println(); + @Value("${spring.servlet.multipart.max-request-size}") + private String maxRequestSize; + + @ExceptionHandler(MaxUploadSizeExceededException.class) + public ResponseEntity handleMaxSizeExceededException(RuntimeException ex, WebRequest request) { + return new ResponseEntity<>("Log entry exceeds size limits: max size per file: " + maxFileSize + ", max total size: " + maxRequestSize, + HttpStatus.PAYLOAD_TOO_LARGE); } } diff --git a/src/main/java/org/phoebus/olog/HttpConnectorConfig.java b/src/main/java/org/phoebus/olog/HttpConnectorConfig.java index 08b9254..b985d7f 100644 --- a/src/main/java/org/phoebus/olog/HttpConnectorConfig.java +++ b/src/main/java/org/phoebus/olog/HttpConnectorConfig.java @@ -4,7 +4,6 @@ import org.apache.coyote.http11.AbstractHttp11Protocol; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.web.embedded.tomcat.TomcatConnectorCustomizer; import org.springframework.boot.web.embedded.tomcat.TomcatServletWebServerFactory; import org.springframework.boot.web.servlet.server.ServletWebServerFactory; import org.springframework.context.annotation.Bean; @@ -13,6 +12,7 @@ @Configuration @PropertySource("classpath:/application.properties") +@SuppressWarnings("unused") public class HttpConnectorConfig { @Value("${server.http.enable:true}") @@ -32,7 +32,10 @@ private Connector getHttpConnector() { Connector connector = new Connector("org.apache.coyote.http11.Http11NioProtocol"); connector.setScheme("http"); connector.setPort(port); - ((AbstractHttp11Protocol )connector.getProtocolHandler()).setMaxSwallowSize(50 * 1024 * 1024); + // This is needed to be able to send a response if client uploads a log entry + // exceeding configured max sizes. Without this setting Tomcat will simply close the + // connection before a response can be sent. + ((AbstractHttp11Protocol )connector.getProtocolHandler()).setMaxSwallowSize(-1); return connector; } } \ No newline at end of file diff --git a/src/main/java/org/phoebus/olog/InfoResource.java b/src/main/java/org/phoebus/olog/InfoResource.java index 36f4bab..1bdcfe5 100644 --- a/src/main/java/org/phoebus/olog/InfoResource.java +++ b/src/main/java/org/phoebus/olog/InfoResource.java @@ -9,6 +9,7 @@ import com.mongodb.client.MongoClient; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.util.unit.DataSize; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -23,6 +24,7 @@ @RestController @RequestMapping(OLOG_SERVICE_INFO) +@SuppressWarnings("unused") public class InfoResource { @@ -39,6 +41,12 @@ public class InfoResource @Value("${elasticsearch.http.port:9200}") private int port; + @Value("${spring.servlet.multipart.max-file-size}") + private String maxFileSize; + + @Value("${spring.servlet.multipart.max-request-size}") + private String maxRequestSize; + private final static ObjectMapper objectMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT); /** @@ -48,12 +56,12 @@ public class InfoResource @GetMapping public String info() { - Map ologServiceInfo = new LinkedHashMap(); + Map ologServiceInfo = new LinkedHashMap<>(); ologServiceInfo.put("name", "Olog Service"); ologServiceInfo.put("version", version); ElasticsearchClient client = esService.getClient(); - Map elasticInfo = new LinkedHashMap(); + Map elasticInfo = new LinkedHashMap<>(); try { InfoResponse response = client.info(); elasticInfo.put("status", "Connected"); @@ -70,6 +78,12 @@ public String info() { ologServiceInfo.put("elastic", elasticInfo); ologServiceInfo.put("mongoDB", mongoClient.getClusterDescription().getShortDescription()); + Map serverConfigInfo = new LinkedHashMap<>(); + // Provide sizes in MB, arithmetics needed to avoid rounding to 0. + serverConfigInfo.put("Max file size", 1.0 * DataSize.parse(maxFileSize).toKilobytes() / 1024); + serverConfigInfo.put("Max request size", 1.0 * DataSize.parse(maxRequestSize).toKilobytes() / 1024); + + ologServiceInfo.put("server config", serverConfigInfo); try { return objectMapper.writeValueAsString(ologServiceInfo); @@ -78,5 +92,4 @@ public String info() { return "Failed to gather Olog service info"; } } - } diff --git a/src/main/java/org/phoebus/olog/LogResource.java b/src/main/java/org/phoebus/olog/LogResource.java index 3f29426..3647daa 100644 --- a/src/main/java/org/phoebus/olog/LogResource.java +++ b/src/main/java/org/phoebus/olog/LogResource.java @@ -65,6 +65,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -242,11 +243,12 @@ public SearchResult search(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, requi * @param principal The authenticated {@link Principal} of the request. * @return The persisted {@link Log} object. */ - @PutMapping() + @PutMapping(consumes = "application/json") public Log createLog(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, required = false, defaultValue = "n/a") String clientInfo, @RequestParam(value = "markup", required = false) String markup, - @RequestBody Log log, @RequestParam(value = "inReplyTo", required = false, defaultValue = "-1") String inReplyTo, + @RequestPart("logEntry") Log log, + @RequestPart(value = "files", required = false) MultipartFile[] files, @AuthenticationPrincipal Principal principal) { if(log.getLogbooks().isEmpty()){ throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "A log entry must specify at least one logbook"); @@ -276,6 +278,22 @@ public Log createLog(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, required = log = cleanMarkup(markup, log); addPropertiesFromProviders(log); Log newLogEntry = logRepository.save(log); + + if(files != null){ + for(MultipartFile multipartFile : files){ + try { + Optional attachment = log.getAttachments().stream().filter(a -> a.getId().equals(multipartFile.getName())).findFirst(); + if(attachment.isPresent()){ + uploadAttachment(Long.toString(newLogEntry.getId()), multipartFile, attachment.get().getFilename(), + attachment.get().getId(), + attachment.get().getFileMetadataDescription()); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + sendToNotifiers(newLogEntry); logger.log(Level.INFO, "Entry id " + newLogEntry.getId() + " created from " + clientInfo); @@ -283,38 +301,10 @@ public Log createLog(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, required = return newLogEntry; } - @PostMapping("/composite") - public Log createCompositeLog(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, required = false, defaultValue = "n/a") String clientInfo, - @RequestParam(value = "markup", required = false) String markup, - @RequestParam(value = "inReplyTo", required = false, defaultValue = "-1") String inReplyTo, - @RequestPart("logEntry") Log log, - @RequestPart("attachmentData") List attachmentData, - @RequestPart("files") MultipartFile[] files, - @AuthenticationPrincipal Principal principal) { - - Log createdLog = createLog(clientInfo, markup, log, inReplyTo, principal); - - for(MultipartFile multipartFile : files){ - try { - String checksum = getMD5Checksum(multipartFile.getInputStream()); - Optional attachmentLight = attachmentData.stream().filter(a -> a.checksum.equals(checksum)).findFirst(); - if(attachmentLight.isPresent()){ - uploadAttachment(Long.toString(createdLog.getId()), multipartFile, attachmentLight.get().fileName, - attachmentLight.get().id, - attachmentLight.get().fileMetaDataDescription); - attachmentData.remove(attachmentLight.get()); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - return createdLog; - } @PostMapping("/attachments/{logId}") public Log uploadAttachment(@PathVariable String logId, - @RequestPart("files") MultipartFile file, + @RequestPart("file") MultipartFile file, @RequestPart("filename") String filename, @RequestPart(value = "id", required = false) String id, @RequestPart(value = "fileMetadataDescription", required = false) String fileMetadataDescription) { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index a4554f8..a20d753 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -138,9 +138,10 @@ mongo.port:27017 # with the origin(s) on which the web front-end is deployed. #cors.allowed.origins=http://localhost:3000 -################## File upload and request size limits ################## -spring.servlet.multipart.max-file-size=50MB -spring.servlet.multipart.max-request-size=100MB +################## File upload and request size limits #################### +# Unit should be MB (or KB), it is case-sensitive! Invalid unit will inhibit server startup. +spring.servlet.multipart.max-file-size=15MB +spring.servlet.multipart.max-request-size=50MB ################## List of "levels" ################## levels=Urgent,Suggestion,Info,Request,Problem @@ -178,3 +179,4 @@ spring.mvc.static-path-pattern=/Olog/** default.logbook.url= default.tags.url= default.properties.url= + From 00c7b4892a6686a9fb9fb39b0ad1101a05cf9f59 Mon Sep 17 00:00:00 2001 From: georgweiss Date: Tue, 13 Jun 2023 10:37:02 +0200 Subject: [PATCH 3/8] New endpoint to handle combined upload of log entry and attachments --- .../olog/FileUploadSizeExceededHandler.java | 16 +- .../java/org/phoebus/olog/LogResource.java | 211 ++++++++---------- .../org/phoebus/olog/entity/Attachment.java | 12 + .../org/phoebus/olog/LogResourceTest.java | 139 ++++++++++-- 4 files changed, 239 insertions(+), 139 deletions(-) diff --git a/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java b/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java index 3f27ae3..cf5c1af 100644 --- a/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java +++ b/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java @@ -20,8 +20,10 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.ControllerAdvice; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.ResponseStatus; @@ -38,6 +40,13 @@ @SuppressWarnings("unused") public class FileUploadSizeExceededHandler { + /** + * Specifies the allowed origins for CORS requests. Defaults to http://localhost:3000, + * which is useful during development of the web front-end in NodeJS. + */ + @Value("${cors.allowed.origins:http://localhost:3000}") + private String corsAllowedOrigins; + @Value("${spring.servlet.multipart.max-file-size}") private String maxFileSize; @@ -46,7 +55,12 @@ public class FileUploadSizeExceededHandler { @ExceptionHandler(MaxUploadSizeExceededException.class) public ResponseEntity handleMaxSizeExceededException(RuntimeException ex, WebRequest request) { - return new ResponseEntity<>("Log entry exceeds size limits: max size per file: " + maxFileSize + ", max total size: " + maxRequestSize, + // These HTTP headers are needed by browsers in order to handle the 413 response properly. + HttpHeaders headers = new HttpHeaders(); + headers.add("Access-Control-Allow-Origin", corsAllowedOrigins); + headers.add("Access-Control-Allow-Credentials", "true"); + return new ResponseEntity<>("Log entry exceeds size limits: max size per file=" + maxFileSize + ", max total size=" + maxRequestSize, + headers, HttpStatus.PAYLOAD_TOO_LARGE); } } diff --git a/src/main/java/org/phoebus/olog/LogResource.java b/src/main/java/org/phoebus/olog/LogResource.java index 3647daa..e5dba96 100644 --- a/src/main/java/org/phoebus/olog/LogResource.java +++ b/src/main/java/org/phoebus/olog/LogResource.java @@ -9,6 +9,7 @@ import org.phoebus.olog.entity.Attachment; import org.phoebus.olog.entity.Log; import org.phoebus.olog.entity.LogEntryGroupHelper; +import org.phoebus.olog.entity.Logbook; import org.phoebus.olog.entity.Property; import org.phoebus.olog.entity.SearchResult; import org.phoebus.olog.entity.Tag; @@ -26,13 +27,8 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.core.annotation.AuthenticationPrincipal; -import org.springframework.util.FileCopyUtils; import org.springframework.util.MultiValueMap; -import org.springframework.web.bind.WebDataBinder; -import org.springframework.web.bind.annotation.ControllerAdvice; -import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.InitBinder; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; @@ -42,38 +38,26 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.multipart.MultipartException; import org.springframework.web.multipart.MultipartFile; import org.springframework.web.server.ResponseStatusException; -import org.springframework.web.servlet.mvc.support.RedirectAttributes; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; -import java.math.BigInteger; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.MessageDigest; import java.security.Principal; import java.time.Instant; import java.time.temporal.TemporalAmount; -import java.time.temporal.TemporalUnit; import java.time.temporal.UnsupportedTemporalTypeException; import java.util.ArrayList; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.zip.CRC32; -import java.util.zip.CheckedInputStream; import static org.phoebus.olog.OlogResourceDescriptors.LOG_RESOURCE_URI; import static org.phoebus.util.time.TimestampFormats.MILLI_FORMAT; @@ -144,9 +128,7 @@ public Log getLog(@PathVariable String logId) { public ResponseEntity findResources(@PathVariable String logId, @PathVariable String attachmentName) { Optional log = logRepository.findById(logId); if (log.isPresent()) { - Set attachments = log.get().getAttachments().stream().filter(attachment -> { - return attachment.getFilename().equals(attachmentName); - }).collect(Collectors.toSet()); + Set attachments = log.get().getAttachments().stream().filter(attachment -> attachment.getFilename().equals(attachmentName)).collect(Collectors.toSet()); if (attachments.size() == 1) { Attachment attachment = attachments.iterator().next(); this.logger.log(Level.INFO, "Requesting attachment " + attachment.getId() + ": " + attachment.getFilename()); @@ -235,32 +217,33 @@ public SearchResult search(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, requi *

* This may return a HTTP 400 if for instance inReplyTo does not identify an existing log entry, * or if the logbooks listed in the {@link Log} object contains invalid (i.e. non-existing) logbooks. + *

* * @param clientInfo A string sent by client identifying it with respect to version and platform. * @param log A {@link Log} object to be persisted. * @param markup Optional string identifying the wanted markup scheme. - * @param inReplyTo Optional log entry id specifying to which log entry the new log entry is a response. + * @param inReplyTo Optional log entry id specifying to which log entry the new log entry is a reply. * @param principal The authenticated {@link Principal} of the request. * @return The persisted {@link Log} object. */ - @PutMapping(consumes = "application/json") + @PutMapping() + @Deprecated public Log createLog(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, required = false, defaultValue = "n/a") String clientInfo, @RequestParam(value = "markup", required = false) String markup, @RequestParam(value = "inReplyTo", required = false, defaultValue = "-1") String inReplyTo, - @RequestPart("logEntry") Log log, - @RequestPart(value = "files", required = false) MultipartFile[] files, + @RequestBody Log log, @AuthenticationPrincipal Principal principal) { - if(log.getLogbooks().isEmpty()){ + if (log.getLogbooks().isEmpty()) { throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "A log entry must specify at least one logbook"); } - if(log.getTitle().isEmpty()){ + if (log.getTitle().isEmpty()) { throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "A log entry must specify a title"); } if (!inReplyTo.equals("-1")) { handleReply(inReplyTo, log); } log.setOwner(principal.getName()); - Set logbookNames = log.getLogbooks().stream().map(l -> l.getName()).collect(Collectors.toSet()); + Set logbookNames = log.getLogbooks().stream().map(Logbook::getName).collect(Collectors.toSet()); Set persistedLogbookNames = new HashSet<>(); logbookRepository.findAll().forEach(l -> persistedLogbookNames.add(l.getName())); if (!CollectionUtils.containsAll(persistedLogbookNames, logbookNames)) { @@ -268,7 +251,7 @@ public Log createLog(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, required = } Set tags = log.getTags(); if (tags != null && !tags.isEmpty()) { - Set tagNames = tags.stream().map(t -> t.getName()).collect(Collectors.toSet()); + Set tagNames = tags.stream().map(Tag::getName).collect(Collectors.toSet()); Set persistedTags = new HashSet<>(); tagRepository.findAll().forEach(t -> persistedTags.add(t.getName())); if (!CollectionUtils.containsAll(persistedTags, tagNames)) { @@ -278,19 +261,54 @@ public Log createLog(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, required = log = cleanMarkup(markup, log); addPropertiesFromProviders(log); Log newLogEntry = logRepository.save(log); + sendToNotifiers(newLogEntry); - if(files != null){ - for(MultipartFile multipartFile : files){ - try { - Optional attachment = log.getAttachments().stream().filter(a -> a.getId().equals(multipartFile.getName())).findFirst(); - if(attachment.isPresent()){ - uploadAttachment(Long.toString(newLogEntry.getId()), multipartFile, attachment.get().getFilename(), - attachment.get().getId(), - attachment.get().getFileMetadataDescription()); - } - } catch (Exception e) { - throw new RuntimeException(e); - } + logger.log(Level.INFO, "Entry id " + newLogEntry.getId() + " created from " + clientInfo); + + return newLogEntry; + } + + /** + * Creates a new log entry. If the inReplyTo parameters identifies an existing log entry, + * this method will treat the new log entry as a reply. + *

+ * This may return a HTTP 400 if for instance inReplyTo does not identify an existing log entry, + * or if the logbooks listed in the {@link Log} object contains invalid (i.e. non-existing) logbooks. + *

+ *

Client calling this endpoint must set Content-Type=multipart/form-data.

+ * + * @param clientInfo A string sent by client identifying it with respect to version and platform. + * @param logEntry A {@link Log} object to be persisted. + * @param markup Optional string identifying the wanted markup scheme. + * @param inReplyTo Optional log entry id specifying to which log entry the new log entry is a reply. + * @param files Optional array of {@link MultipartFile}s representing attachments. These must appear in the same + * order as the {@link Attachment} items in the list of {@link Attachment}s of the log entry. + * @param principal The authenticated {@link Principal} of the request. + * @return The persisted {@link Log} object. + */ + @PutMapping("/multipart") + public Log createLog(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, required = false, defaultValue = "n/a") String clientInfo, + @RequestParam(value = "markup", required = false) String markup, + @RequestParam(value = "inReplyTo", required = false, defaultValue = "-1") String inReplyTo, + @RequestPart("logEntry") Log logEntry, + @RequestPart(value = "files", required = false) MultipartFile[] files, + @AuthenticationPrincipal Principal principal) { + + if (files != null && logEntry.getAttachments() != null && files.length != logEntry.getAttachments().size()) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Attachment data invalid: file count does not match attachment count"); + } + + Log newLogEntry = createLog(clientInfo, markup, inReplyTo, logEntry, principal); + + if (files != null) { + Iterator attachmentIterator = logEntry.getAttachments().iterator(); + for (int i = 0; i < files.length; i++) { + Attachment attachment = attachmentIterator.next(); + uploadAttachment(Long.toString(newLogEntry.getId()), + files[i], + attachment.getFilename(), + attachment.getId(), + attachment.getFileMetadataDescription()); } } @@ -368,32 +386,7 @@ public Log updateLog(@PathVariable String logId, persistedLog.setTitle(log.getTitle()); persistedLog = cleanMarkup(markup, persistedLog); - Log newLogEntry = logRepository.update(persistedLog); - return newLogEntry; - } else { - throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Failed to retrieve log with id: " + logId); - } - } - - - /** - * Endpoint supporting upload of multiple files, i.e. saving the client from sending one POST request per file. - * Calls {@link #uploadAttachment(String, MultipartFile, String, String, String)} internally, using the original file's - * name and content type. - * - * @param logId A (numerical) id of a {@link Log} - * @param files The files subject to upload. - * @return The persisted {@link Log} object. - */ - @SuppressWarnings("unused") - @PostMapping(value = "/attachments-multi/{logId}", consumes = "multipart/form-data") - public Log uploadMultipleAttachments(@PathVariable String logId, - @RequestPart("file") MultipartFile[] files) { - if (logRepository.findById(logId).isPresent()) { - for (MultipartFile file : files) { - uploadAttachment(logId, file, file.getOriginalFilename(), file.getName(), file.getContentType()); - } - return logRepository.findById(logId).get(); + return logRepository.update(persistedLog); } else { throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Failed to retrieve log with id: " + logId); } @@ -409,7 +402,7 @@ public void groupLogEntries(@RequestBody List logEntryIds) { // the same group. If not, throw exception. synchronized (logGroupSyncObject) { for (Long id : logEntryIds) { - Optional log = null; + Optional log; try { log = logRepository.findById(Long.toString(id)); } catch (ResponseStatusException exception) { @@ -451,13 +444,13 @@ public void groupLogEntries(@RequestBody List logEntryIds) { * error handling or logging has to be done in the {@link LogEntryNotifier}, but exceptions are * handled here in order to not abort if any of the providers fails. * - * @param log + * @param log The log entry */ private void sendToNotifiers(Log log) { if (logEntryNotifiers.isEmpty()) { return; } - taskExecutor.execute(() -> logEntryNotifiers.stream().forEach(n -> { + taskExecutor.execute(() -> logEntryNotifiers.forEach(n -> { try { n.notify(log); } catch (Exception e) { @@ -479,6 +472,30 @@ private Log cleanMarkup(String markup, Log log) { return log; } + /** + * Endpoint supporting upload of multiple files, i.e. saving the client from sending one POST request per file. + * Calls {@link #uploadAttachment(String, MultipartFile, String, String, String)} internally, using the original file's + * name and content type. + * + * @param logId A (numerical) id of a {@link Log} + * @param files The files subject to upload. + * @return The persisted {@link Log} object. + */ + @SuppressWarnings("unused") + @PostMapping(value = "/attachments-multi/{logId}", consumes = "multipart/form-data") + @Deprecated + public Log uploadMultipleAttachments(@PathVariable String logId, + @RequestPart("file") MultipartFile[] files) { + if (logRepository.findById(logId).isPresent()) { + for (MultipartFile file : files) { + uploadAttachment(logId, file, file.getOriginalFilename(), file.getName(), file.getContentType()); + } + return logRepository.findById(logId).get(); + } else { + throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Failed to retrieve log with id: " + logId); + } + } + /** * This will retrieve {@link Property}s from {@link LogPropertyProvider}s, if any are registered * over SPI. @@ -559,58 +576,4 @@ private void handleReply(String originalLogEntryId, Log log) { throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Cannot reply to log entry " + originalLogEntryId + " as it does not exist"); } } - - private static class AttachmentLight{ - private String id; - private String fileName; - private String fileMetaDataDescription; - - private String checksum; - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getFileName() { - return fileName; - } - - public void setFileName(String fileName) { - this.fileName = fileName; - } - - public String getFileMetaDataDescription() { - return fileMetaDataDescription; - } - - public void setFileMetaDataDescription(String fileMetaDataDescription) { - this.fileMetaDataDescription = fileMetaDataDescription; - } - - public String getChecksum() { - return checksum; - } - - public void setChecksum(String checksum) { - this.checksum = checksum; - } - } - - private long getChecksumCRC32(InputStream inputStream, int bufferSize) - throws IOException { - CheckedInputStream checkedInputStream = new CheckedInputStream(inputStream, new CRC32()); - byte[] buffer = new byte[bufferSize]; - while (checkedInputStream.read(buffer, 0, buffer.length) >= 0) {} - return checkedInputStream.getChecksum().getValue(); - } - - private static String getMD5Checksum(InputStream inputStream) throws Exception{ - byte[] data = FileCopyUtils.copyToByteArray(inputStream); - byte[] hash = MessageDigest.getInstance("MD5").digest(data); - return new BigInteger(1, hash).toString(16); - } } diff --git a/src/main/java/org/phoebus/olog/entity/Attachment.java b/src/main/java/org/phoebus/olog/entity/Attachment.java index 1c6736a..62241bd 100644 --- a/src/main/java/org/phoebus/olog/entity/Attachment.java +++ b/src/main/java/org/phoebus/olog/entity/Attachment.java @@ -24,6 +24,10 @@ public class Attachment @JsonIgnore private InputStreamSource attachment; + private String checksum; + + + /** * Creates a new instance of Attachment. */ @@ -139,4 +143,12 @@ public void setFileMetadataDescription(String fileMetadataDescription) { this.fileMetadataDescription = fileMetadataDescription; } + + public String getChecksum() { + return checksum; + } + + public void setChecksum(String checksum) { + this.checksum = checksum; + } } diff --git a/src/test/java/org/phoebus/olog/LogResourceTest.java b/src/test/java/org/phoebus/olog/LogResourceTest.java index 2895a91..63d0992 100644 --- a/src/test/java/org/phoebus/olog/LogResourceTest.java +++ b/src/test/java/org/phoebus/olog/LogResourceTest.java @@ -25,6 +25,7 @@ import org.mockito.ArgumentMatcher; import org.mockito.Mockito; import org.mockito.internal.util.collections.Sets; +import org.phoebus.olog.entity.Attachment; import org.phoebus.olog.entity.Attribute; import org.phoebus.olog.entity.Log; import org.phoebus.olog.entity.Log.LogBuilder; @@ -36,8 +37,11 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.autoconfigure.web.servlet.WebMvcTest; import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.mock.web.MockMultipartFile; +import org.springframework.mock.web.MockMultipartHttpServletRequest; +import org.springframework.mock.web.MockPart; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.ContextHierarchy; import org.springframework.test.context.TestPropertySource; @@ -47,10 +51,15 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; +import org.springframework.web.multipart.MultipartHttpServletRequest; +import org.springframework.web.multipart.MultipartRequest; +import org.springframework.web.multipart.support.StandardMultipartHttpServletRequest; import org.springframework.web.server.ResponseStatusException; +import javax.servlet.http.Part; import java.time.Instant; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; @@ -355,23 +364,125 @@ public void testCreateAttachment() throws Exception { reset(logRepository); } - /** - * Test only endpoint URI - * - * @throws Exception - */ @Test - public void testCreateMultipleAttachments() throws Exception { - when(logRepository.findById("1")).thenReturn(Optional.of(log1)); + public void testCreateLogMultipart() throws Exception{ + Attachment attachment = new Attachment(); + attachment.setId("attachmentId"); + attachment.setFilename("filename1.txt"); + Log log = LogBuilder.createLog() + .id(1L) + .owner("user") + .title("title") + .withLogbooks(Set.of(logbook1, logbook2)) + .withTags(Set.of(tag1, tag2)) + .source("description1") + .description("description1") + .createDate(now) + .modifyDate(now) + .level("Urgent") + .build(); + Set attachments = new HashSet<>(); + attachments.add(attachment); + log.setAttachments(attachments); + MockMultipartFile file1 = + new MockMultipartFile("files", "filename1.txt", "text/plain", "some xml".getBytes()); + MockMultipartFile log1 = new MockMultipartFile("logEntry", "","application/json", objectMapper.writeValueAsString(log).getBytes()); + + when(logbookRepository.findAll()).thenReturn(Arrays.asList(logbook1, logbook2)); + when(tagRepository.findAll()).thenReturn(Arrays.asList(tag1, tag2)); + when(logRepository.save(argThat(new LogMatcher(log)))).thenReturn(log); + when(logRepository.findById("1")).thenReturn(Optional.of(log)); + MockHttpServletRequestBuilder request = + MockMvcRequestBuilders.multipart(HttpMethod.PUT, + "/" + OlogResourceDescriptors.LOG_RESOURCE_URI + "/multipart") + .file(file1) + .file(log1) + .header(HttpHeaders.AUTHORIZATION, AUTHORIZATION) + .header(HttpHeaders.CONTENT_TYPE, "multipart/form-data") + .contentType(JSON); + MvcResult result = mockMvc.perform(request).andExpect(status().isOk()).andReturn(); + + Log savedLog = objectMapper.readValue(result.getResponse().getContentAsString(), Log.class); + assertEquals(Long.valueOf(1L), savedLog.getId()); + reset(logRepository); + } + + @Test + public void testCreateLogMultipartNoAttachments() throws Exception{ + Log log = LogBuilder.createLog() + .id(1L) + .owner("user") + .title("title") + .withLogbooks(Set.of(logbook1, logbook2)) + .withTags(Set.of(tag1, tag2)) + .source("description1") + .description("description1") + .createDate(now) + .modifyDate(now) + .level("Urgent") + .build(); + MockMultipartFile log1 = new MockMultipartFile("logEntry", "","application/json", objectMapper.writeValueAsString(log).getBytes()); + + when(logbookRepository.findAll()).thenReturn(Arrays.asList(logbook1, logbook2)); + when(tagRepository.findAll()).thenReturn(Arrays.asList(tag1, tag2)); + when(logRepository.save(argThat(new LogMatcher(log)))).thenReturn(log); + when(logRepository.findById("1")).thenReturn(Optional.of(log)); + MockHttpServletRequestBuilder request = + MockMvcRequestBuilders.multipart(HttpMethod.PUT, + "/" + OlogResourceDescriptors.LOG_RESOURCE_URI + "/multipart") + .file(log1) + .header(HttpHeaders.AUTHORIZATION, AUTHORIZATION) + .header(HttpHeaders.CONTENT_TYPE, "multipart/form-data") + .contentType(JSON); + MvcResult result = mockMvc.perform(request).andExpect(status().isOk()).andReturn(); + + Log savedLog = objectMapper.readValue(result.getResponse().getContentAsString(), Log.class); + assertEquals(Long.valueOf(1L), savedLog.getId()); + reset(logRepository); + } + + @Test + public void testCreateLogMultipartFileAndAttachmentMismatch() throws Exception{ + Attachment attachment = new Attachment(); + attachment.setId("attachmentId"); + attachment.setFilename("filename1.txt"); + Attachment attachment2 = new Attachment(); + attachment2.setId("attachmentId2"); + attachment2.setFilename("filename2.txt"); + Log log = LogBuilder.createLog() + .id(1L) + .owner("user") + .title("title") + .withLogbooks(Set.of(logbook1, logbook2)) + .withTags(Set.of(tag1, tag2)) + .source("description1") + .description("description1") + .createDate(now) + .modifyDate(now) + .level("Urgent") + .build(); + Set attachments = new HashSet<>(); + attachments.add(attachment); + attachments.add(attachment2); + log.setAttachments(attachments); MockMultipartFile file1 = - new MockMultipartFile("file", "filename1.txt", "text/plain", "some xml".getBytes()); - MockMultipartFile file2 = - new MockMultipartFile("file", "filename2.txt", "text/plain", "some xml".getBytes()); - mockMvc.perform(MockMvcRequestBuilders.multipart("/" + OlogResourceDescriptors.LOG_RESOURCE_URI + "/attachments-multi/1") + new MockMultipartFile("files", "filename1.txt", "text/plain", "some xml".getBytes()); + MockMultipartFile log1 = new MockMultipartFile("logEntry", "","application/json", objectMapper.writeValueAsString(log).getBytes()); + + when(logbookRepository.findAll()).thenReturn(Arrays.asList(logbook1, logbook2)); + when(tagRepository.findAll()).thenReturn(Arrays.asList(tag1, tag2)); + when(logRepository.save(argThat(new LogMatcher(log)))).thenReturn(log); + when(logRepository.findById("1")).thenReturn(Optional.of(log)); + MockHttpServletRequestBuilder request = + MockMvcRequestBuilders.multipart(HttpMethod.PUT, + "/" + OlogResourceDescriptors.LOG_RESOURCE_URI + "/multipart") .file(file1) - .file(file2) - .header(HttpHeaders.AUTHORIZATION, AUTHORIZATION)) - .andExpect(status().is(200)); + .file(log1) + .header(HttpHeaders.AUTHORIZATION, AUTHORIZATION) + .header(HttpHeaders.CONTENT_TYPE, "multipart/form-data") + .contentType(JSON); + mockMvc.perform(request).andExpect(status().isBadRequest()); + reset(logRepository); } From 322e75a863c9a080079ad726cdb9eef05c5c7737 Mon Sep 17 00:00:00 2001 From: georgweiss Date: Tue, 13 Jun 2023 13:59:00 +0200 Subject: [PATCH 4/8] Update documentation: adding new endpoint --- src/site/sphinx/index.rst | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/site/sphinx/index.rst b/src/site/sphinx/index.rst index 0c5ee39..76bdf9b 100644 --- a/src/site/sphinx/index.rst +++ b/src/site/sphinx/index.rst @@ -131,7 +131,7 @@ time based searches will ensure that these log entries are also found even if th Quick Start ############ -Download and install elasticsearch (verision 6.3) from elastic.com +Download and install elasticsearch (verision 8.3) from elastic.com Download and install mongodb from mongodb Configure the service @@ -158,6 +158,7 @@ Creating a Log Entry Create a simple log entry +NOTE: deprecated, will be removed in future commits. Replaced by https://localhost:8181/Olog/logs/multipart **PUT** https://localhost:8181/Olog/logs .. code-block:: json @@ -174,6 +175,39 @@ Create a simple log entry ] } +Create a log entry, optionally with file attachments + +**PUT** https://localhost:8181/Olog/logs/multipart + +.. code-block:: json + + { + "owner":"log", + "description":"Beam Dump due to Major power dip Current Alarms Booster transmitter switched back to lower state.", + "level":"Info", + "title":"Some title", + "logbooks":[ + { + "name":"Operations" + } + ], + "attachments":[ + {"id": "82dd67fa-09df-11ee-be56-0242ac120002", "name":"MyScreenShot.png"}, + {"id": "c02948ad-4bbd-432f-aa4d-a687a54f8d40", "name":"MySpreadsheet.xlsx"} + ] + } + +**NOTE** Attachment ids must be unique, e.g. UUID. When creating a log entry - optionally with attachments - client **must**: + +#. Use a multipart request and set the Content-Type to "multipart/form-data", even if no attachments are present. +#. If attachments are present: add one request part per attachment file, in the order they appear in the log entry. Each +file must be added using "files" as the name for the part. +#. Add the log entry as a request part with content type "application/json". The name of the part must be "logEntry". + +Client must also be prepared to handle a HTTP 413 (payload too large) response in case the attached files exceed +file and request size limits configured in the service. + + Reply to a log entry. This uses the same end point as when creating a log entry, but client must send the unique id of the log entry to which the new one is a reply. From 7534d11a1a2ecf1fe59ce47f4e6b93ed11a70e13 Mon Sep 17 00:00:00 2001 From: georgweiss Date: Thu, 15 Jun 2023 13:53:06 +0200 Subject: [PATCH 5/8] Bug fix attachments file name handling --- src/main/java/org/phoebus/olog/LogResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/phoebus/olog/LogResource.java b/src/main/java/org/phoebus/olog/LogResource.java index e5dba96..7b166b4 100644 --- a/src/main/java/org/phoebus/olog/LogResource.java +++ b/src/main/java/org/phoebus/olog/LogResource.java @@ -306,7 +306,7 @@ public Log createLog(@RequestHeader(value = OLOG_CLIENT_INFO_HEADER, required = Attachment attachment = attachmentIterator.next(); uploadAttachment(Long.toString(newLogEntry.getId()), files[i], - attachment.getFilename(), + files[i].getOriginalFilename(), attachment.getId(), attachment.getFileMetadataDescription()); } From f27bb132efd8141e65037270644e53e718ddb880 Mon Sep 17 00:00:00 2001 From: georgweiss Date: Wed, 21 Jun 2023 11:59:09 +0200 Subject: [PATCH 6/8] Java-friendly names for max file size and max request size --- src/main/java/org/phoebus/olog/InfoResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/phoebus/olog/InfoResource.java b/src/main/java/org/phoebus/olog/InfoResource.java index 1bdcfe5..04f2a9a 100644 --- a/src/main/java/org/phoebus/olog/InfoResource.java +++ b/src/main/java/org/phoebus/olog/InfoResource.java @@ -80,8 +80,8 @@ public String info() { Map serverConfigInfo = new LinkedHashMap<>(); // Provide sizes in MB, arithmetics needed to avoid rounding to 0. - serverConfigInfo.put("Max file size", 1.0 * DataSize.parse(maxFileSize).toKilobytes() / 1024); - serverConfigInfo.put("Max request size", 1.0 * DataSize.parse(maxRequestSize).toKilobytes() / 1024); + serverConfigInfo.put("maxFileSize", 1.0 * DataSize.parse(maxFileSize).toKilobytes() / 1024); + serverConfigInfo.put("maxRequestSize", 1.0 * DataSize.parse(maxRequestSize).toKilobytes() / 1024); ologServiceInfo.put("server config", serverConfigInfo); From c780454e7daca0b2595e9d09c2d9f7e6b26ed8b8 Mon Sep 17 00:00:00 2001 From: georgweiss Date: Wed, 21 Jun 2023 13:16:40 +0200 Subject: [PATCH 7/8] More Java-friendly info endpoint data ids --- src/main/java/org/phoebus/olog/InfoResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/phoebus/olog/InfoResource.java b/src/main/java/org/phoebus/olog/InfoResource.java index 04f2a9a..fe09216 100644 --- a/src/main/java/org/phoebus/olog/InfoResource.java +++ b/src/main/java/org/phoebus/olog/InfoResource.java @@ -83,7 +83,7 @@ public String info() { serverConfigInfo.put("maxFileSize", 1.0 * DataSize.parse(maxFileSize).toKilobytes() / 1024); serverConfigInfo.put("maxRequestSize", 1.0 * DataSize.parse(maxRequestSize).toKilobytes() / 1024); - ologServiceInfo.put("server config", serverConfigInfo); + ologServiceInfo.put("serverConfig", serverConfigInfo); try { return objectMapper.writeValueAsString(ologServiceInfo); From 93c6862574b4ded61afaa87049fe13874998c2da Mon Sep 17 00:00:00 2001 From: georgweiss Date: Tue, 27 Jun 2023 10:46:03 +0200 Subject: [PATCH 8/8] Upload size limits need default values --- .../org/phoebus/olog/FileUploadSizeExceededHandler.java | 7 +------ src/main/java/org/phoebus/olog/InfoResource.java | 5 +++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java b/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java index cf5c1af..c3094e7 100644 --- a/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java +++ b/src/main/java/org/phoebus/olog/FileUploadSizeExceededHandler.java @@ -47,11 +47,6 @@ public class FileUploadSizeExceededHandler { @Value("${cors.allowed.origins:http://localhost:3000}") private String corsAllowedOrigins; - @Value("${spring.servlet.multipart.max-file-size}") - private String maxFileSize; - - @Value("${spring.servlet.multipart.max-request-size}") - private String maxRequestSize; @ExceptionHandler(MaxUploadSizeExceededException.class) public ResponseEntity handleMaxSizeExceededException(RuntimeException ex, WebRequest request) { @@ -59,7 +54,7 @@ public ResponseEntity handleMaxSizeExceededException(RuntimeException ex HttpHeaders headers = new HttpHeaders(); headers.add("Access-Control-Allow-Origin", corsAllowedOrigins); headers.add("Access-Control-Allow-Credentials", "true"); - return new ResponseEntity<>("Log entry exceeds size limits: max size per file=" + maxFileSize + ", max total size=" + maxRequestSize, + return new ResponseEntity<>("Log entry exceeds size limits", headers, HttpStatus.PAYLOAD_TOO_LARGE); } diff --git a/src/main/java/org/phoebus/olog/InfoResource.java b/src/main/java/org/phoebus/olog/InfoResource.java index fe09216..d9128ca 100644 --- a/src/main/java/org/phoebus/olog/InfoResource.java +++ b/src/main/java/org/phoebus/olog/InfoResource.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.mongodb.client.MongoClient; +import org.apache.catalina.Server; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.util.unit.DataSize; @@ -41,10 +42,10 @@ public class InfoResource @Value("${elasticsearch.http.port:9200}") private int port; - @Value("${spring.servlet.multipart.max-file-size}") + @Value("${spring.servlet.multipart.max-file-size:15MB}") private String maxFileSize; - @Value("${spring.servlet.multipart.max-request-size}") + @Value("${spring.servlet.multipart.max-request-size:50MB}") private String maxRequestSize; private final static ObjectMapper objectMapper = new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);