diff --git a/server/src/main/java/com/adobe/testing/s3mock/FileStoreController.java b/server/src/main/java/com/adobe/testing/s3mock/FileStoreController.java index efc6e8830..c3b816450 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/FileStoreController.java +++ b/server/src/main/java/com/adobe/testing/s3mock/FileStoreController.java @@ -30,8 +30,10 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.commons.lang3.StringUtils.isEmpty; import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; import static org.apache.commons.lang3.StringUtils.substringAfter; import static org.apache.commons.lang3.StringUtils.substringBefore; +import static org.eclipse.jetty.util.UrlEncoded.encodeString; import static org.springframework.http.HttpHeaders.IF_MATCH; import static org.springframework.http.HttpHeaders.IF_NONE_MATCH; import static org.springframework.http.HttpStatus.BAD_REQUEST; @@ -330,12 +332,17 @@ public ListBucketResult listObjectsInsideBucket(@PathVariable final String bucke } } + String returnPrefix = prefix; + Set returnCommonPrefixes = commonPrefixes; + if (useUrlEncoding) { contents = applyUrlEncoding(contents); + returnPrefix = isNotEmpty(prefix) ? encodeString(prefix) : prefix; + returnCommonPrefixes = applyUrlEncoding(commonPrefixes); } - return new ListBucketResult(bucketName, prefix, marker, maxKeys, isTruncated, encodingtype, - nextMarker, contents, commonPrefixes); + return new ListBucketResult(bucketName, returnPrefix, marker, maxKeys, isTruncated, + encodingtype, nextMarker, contents, returnCommonPrefixes); } catch (final IOException e) { LOG.error(String.format("Object(s) could not retrieved from bucket %s", bucketName)); response.sendError(500, e.getMessage()); @@ -345,11 +352,15 @@ public ListBucketResult listObjectsInsideBucket(@PathVariable final String bucke } private List applyUrlEncoding(final List contents) { - return contents.stream().map(c -> new BucketContents(UrlEncoded.encodeString(c.getKey()), + return contents.stream().map(c -> new BucketContents(encodeString(c.getKey()), c.getLastModified(), c.getEtag(), c.getSize(), c.getStorageClass(), c.getOwner())).collect( Collectors.toList()); } + private Set applyUrlEncoding(final Set contents) { + return contents.stream().map(UrlEncoded::encodeString).collect(Collectors.toSet()); + } + /** * Collapse all bucket elements with keys starting with some prefix up to the given delimiter into * one prefix entry. Collapsed elements are removed from the contents list. @@ -452,14 +463,21 @@ public ListBucketResultV2 listObjectsInsideBucketV2(@PathVariable final String b collapseCommonPrefixes(prefix, delimiter, filteredContents, commonPrefixes); } + String returnPrefix = prefix; + String returnStartAfter = startAfter; + Set returnCommonPrefixes = commonPrefixes; + if (useUrlEncoding) { filteredContents = applyUrlEncoding(filteredContents); + returnPrefix = isNotEmpty(prefix) ? encodeString(prefix) : prefix; + returnStartAfter = isNotEmpty(startAfter) ? encodeString(startAfter) : startAfter; + returnCommonPrefixes = applyUrlEncoding(commonPrefixes); } - return new ListBucketResultV2(bucketName, prefix, maxKeysParam, - isTruncated, filteredContents, commonPrefixes, + return new ListBucketResultV2(bucketName, returnPrefix, maxKeysParam, + isTruncated, filteredContents, returnCommonPrefixes, continuationToken, String.valueOf(filteredContents.size()), - nextContinuationToken, startAfter, encodingtype); + nextContinuationToken, returnStartAfter, encodingtype); } catch (final IOException e) { LOG.error(String.format("Object(s) could not retrieved from bucket %s", bucketName)); response.sendError(500, e.getMessage()); diff --git a/server/src/main/java/com/adobe/testing/s3mock/dto/ListBucketResult.java b/server/src/main/java/com/adobe/testing/s3mock/dto/ListBucketResult.java index 90a244f5c..7fda0b158 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/dto/ListBucketResult.java +++ b/server/src/main/java/com/adobe/testing/s3mock/dto/ListBucketResult.java @@ -17,6 +17,7 @@ package com.adobe.testing.s3mock.dto; import com.adobe.testing.s3mock.domain.BucketContents; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonRootName; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; @@ -30,6 +31,7 @@ * Represents a result of listing objects that reside in a Bucket. */ @JsonRootName("ListBucketResult") +@JsonInclude(JsonInclude.Include.NON_EMPTY) public class ListBucketResult implements Serializable { @JsonProperty("Name") @@ -98,7 +100,8 @@ public ListBucketResult(final String name, this.nextMarker = nextMarker; this.contents = new ArrayList<>(); this.contents.addAll(contents); - this.commonPrefixes = new CommonPrefixes(commonPrefixes); + this.commonPrefixes = commonPrefixes == null || commonPrefixes.isEmpty() ? null : + new CommonPrefixes(commonPrefixes); } @XmlElement(name = "Name") diff --git a/server/src/main/java/com/adobe/testing/s3mock/dto/ListBucketResultV2.java b/server/src/main/java/com/adobe/testing/s3mock/dto/ListBucketResultV2.java index 43d2a3034..b0bda3869 100644 --- a/server/src/main/java/com/adobe/testing/s3mock/dto/ListBucketResultV2.java +++ b/server/src/main/java/com/adobe/testing/s3mock/dto/ListBucketResultV2.java @@ -17,6 +17,7 @@ package com.adobe.testing.s3mock.dto; import com.adobe.testing.s3mock.domain.BucketContents; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonRootName; import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper; @@ -30,6 +31,7 @@ * Represents a result of listing objects that reside in a Bucket. */ @JsonRootName("ListBucketResult") +@JsonInclude(JsonInclude.Include.NON_EMPTY) public class ListBucketResultV2 implements Serializable { @JsonProperty("Name") @@ -95,11 +97,12 @@ public ListBucketResultV2(final String name, final String prefix, final String m final String encodingType) { this.name = name; this.prefix = prefix; - this.maxKeys = Integer.valueOf(maxKeys); + this.maxKeys = Integer.parseInt(maxKeys); this.isTruncated = isTruncated; this.contents = new ArrayList<>(); this.contents.addAll(contents); - this.commonPrefixes = new CommonPrefixes(commonPrefixes); + this.commonPrefixes = commonPrefixes == null || commonPrefixes.isEmpty() ? null : + new CommonPrefixes(commonPrefixes); this.continuationToken = continuationToken; this.keyCount = keyCount; this.nextContinuationToken = nextContinuationToken; diff --git a/server/src/test/java/com/adobe/testing/s3mock/its/ListObjectIT.java b/server/src/test/java/com/adobe/testing/s3mock/its/ListObjectIT.java index f2a460c00..8693f65bd 100644 --- a/server/src/test/java/com/adobe/testing/s3mock/its/ListObjectIT.java +++ b/server/src/test/java/com/adobe/testing/s3mock/its/ListObjectIT.java @@ -29,10 +29,8 @@ import com.amazonaws.services.s3.model.ListObjectsV2Result; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3ObjectSummary; -import java.net.URLDecoder; import java.util.ArrayList; import java.util.Arrays; -import java.util.stream.Collectors; import org.eclipse.jetty.util.UrlEncoded; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; @@ -89,7 +87,7 @@ Param encodedKeys(final String... expectedKeys) { String[] decodedKeys() { return Arrays.stream(expectedKeys) - .map(URLDecoder::decode) + .map(UrlEncoded::decodeString) .toArray(String[]::new); } @@ -173,11 +171,19 @@ public void listV1(final Param parameters) { String.join("\n ", l.getCommonPrefixes()) // ); + String[] expectedPrefixes = parameters.expectedPrefixes; + // AmazonS3#listObjects does not decode the prefixes, need to encode expected values + if (parameters.expectedEncoding != null) { + expectedPrefixes = Arrays.stream(parameters.expectedPrefixes) + .map(UrlEncoded::encodeString) + .toArray(String[]::new); + } + assertThat("Returned keys are correct", l.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(toList()), parameters.expectedKeys.length > 0 ? contains(parameters.expectedKeys) : empty()); assertThat("Returned prefixes are correct", new ArrayList<>(l.getCommonPrefixes()), - parameters.expectedPrefixes.length > 0 ? contains(parameters.expectedPrefixes) : empty()); + parameters.expectedPrefixes.length > 0 ? contains(expectedPrefixes) : empty()); assertThat("Returned encodingType is correct", l.getEncodingType(), is(equalTo(parameters.expectedEncoding))); } @@ -196,21 +202,22 @@ public void listV2(final Param parameters) { .withEncodingType(parameters.expectedEncoding)); LOGGER.info( - "list V1, prefix='{}', delimiter='{}', startAfter='{}': " + "list V2, prefix='{}', delimiter='{}', startAfter='{}': " + "\n Objects: \n {}\n Prefixes: \n {}\n", // parameters.prefix, // parameters.delimiter, // parameters.startAfter, // - l.getObjectSummaries().stream().map(s -> URLDecoder.decode(s.getKey())) + l.getObjectSummaries().stream().map(s -> UrlEncoded.decodeString(s.getKey())) .collect(joining("\n ")), // String.join("\n ", l.getCommonPrefixes()) // ); // listV2 automatically decodes the keys so the expected keys have to be decoded String[] expectedDecodedKeys = parameters.decodedKeys(); assertThat("Returned keys are correct", - l.getObjectSummaries().stream().map(s -> s.getKey()).collect(toList()), + l.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(toList()), parameters.expectedKeys.length > 0 ? contains(expectedDecodedKeys) : empty()); - assertThat("Returned prefixes are correct", l.getCommonPrefixes().stream().collect(toList()), + // AmazonS3#listObjectsV2 returns decoded prefixes + assertThat("Returned prefixes are correct", new ArrayList<>(l.getCommonPrefixes()), parameters.expectedPrefixes.length > 0 ? contains(parameters.expectedPrefixes) : empty()); assertThat("Returned encodingType is correct", l.getEncodingType(), is(equalTo(parameters.expectedEncoding)));