Fix encoded responses for aws-cli

aws-cli expects no empty elements in encoded responses and will fail
the request if it encounters an empty element.
At least aws-java-sdk V1 and V2 worked fine with the current
implementation, though.

Fixes #257

server/src/main/java/com/adobe/testing/s3mock/FileStoreController.java

@@ -30,8 +30,10 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.commons.lang3.StringUtils.isEmpty;
 import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
 import static org.apache.commons.lang3.StringUtils.substringAfter;
 import static org.apache.commons.lang3.StringUtils.substringBefore;
+import static org.eclipse.jetty.util.UrlEncoded.encodeString;
 import static org.springframework.http.HttpHeaders.IF_MATCH;
 import static org.springframework.http.HttpHeaders.IF_NONE_MATCH;
 import static org.springframework.http.HttpStatus.BAD_REQUEST;
@@ -330,12 +332,17 @@ public ListBucketResult listObjectsInsideBucket(@PathVariable final String bucke
         }
       }
 
+      String returnPrefix = prefix;
+      Set<String> returnCommonPrefixes = commonPrefixes;
+
       if (useUrlEncoding) {
         contents = applyUrlEncoding(contents);
+        returnPrefix = isNotEmpty(prefix) ? encodeString(prefix) : prefix;
+        returnCommonPrefixes = applyUrlEncoding(commonPrefixes);
       }
 
-      return new ListBucketResult(bucketName, prefix, marker, maxKeys, isTruncated, encodingtype,
-          nextMarker, contents, commonPrefixes);
+      return new ListBucketResult(bucketName, returnPrefix, marker, maxKeys, isTruncated,
+          encodingtype, nextMarker, contents, returnCommonPrefixes);
     } catch (final IOException e) {
       LOG.error(String.format("Object(s) could not retrieved from bucket %s", bucketName));
       response.sendError(500, e.getMessage());
@@ -345,11 +352,15 @@ public ListBucketResult listObjectsInsideBucket(@PathVariable final String bucke
   }
 
   private List<BucketContents> applyUrlEncoding(final List<BucketContents> contents) {
-    return contents.stream().map(c -> new BucketContents(UrlEncoded.encodeString(c.getKey()),
+    return contents.stream().map(c -> new BucketContents(encodeString(c.getKey()),
         c.getLastModified(), c.getEtag(), c.getSize(), c.getStorageClass(), c.getOwner())).collect(
         Collectors.toList());
   }
 
+  private Set<String> applyUrlEncoding(final Set<String> contents) {
+    return contents.stream().map(UrlEncoded::encodeString).collect(Collectors.toSet());
+  }
+
   /**
    * Collapse all bucket elements with keys starting with some prefix up to the given delimiter into
    * one prefix entry. Collapsed elements are removed from the contents list.
@@ -452,14 +463,21 @@ public ListBucketResultV2 listObjectsInsideBucketV2(@PathVariable final String b
         collapseCommonPrefixes(prefix, delimiter, filteredContents, commonPrefixes);
       }
 
+      String returnPrefix = prefix;
+      String returnStartAfter = startAfter;
+      Set<String> returnCommonPrefixes = commonPrefixes;
+
       if (useUrlEncoding) {
         filteredContents = applyUrlEncoding(filteredContents);
+        returnPrefix = isNotEmpty(prefix) ? encodeString(prefix) : prefix;
+        returnStartAfter = isNotEmpty(startAfter) ? encodeString(startAfter) : startAfter;
+        returnCommonPrefixes = applyUrlEncoding(commonPrefixes);
       }
 
-      return new ListBucketResultV2(bucketName, prefix, maxKeysParam,
-          isTruncated, filteredContents, commonPrefixes,
+      return new ListBucketResultV2(bucketName, returnPrefix, maxKeysParam,
+          isTruncated, filteredContents, returnCommonPrefixes,
           continuationToken, String.valueOf(filteredContents.size()),
-          nextContinuationToken, startAfter, encodingtype);
+          nextContinuationToken, returnStartAfter, encodingtype);
     } catch (final IOException e) {
       LOG.error(String.format("Object(s) could not retrieved from bucket %s", bucketName));
       response.sendError(500, e.getMessage());

server/src/main/java/com/adobe/testing/s3mock/dto/ListBucketResult.java

@@ -17,6 +17,7 @@
 package com.adobe.testing.s3mock.dto;
 
 import com.adobe.testing.s3mock.domain.BucketContents;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonRootName;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
@@ -30,6 +31,7 @@
  * Represents a result of listing objects that reside in a Bucket.
  */
 @JsonRootName("ListBucketResult")
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
 public class ListBucketResult implements Serializable {
 
   @JsonProperty("Name")
@@ -98,7 +100,8 @@ public ListBucketResult(final String name,
     this.nextMarker = nextMarker;
     this.contents = new ArrayList<>();
     this.contents.addAll(contents);
-    this.commonPrefixes = new CommonPrefixes(commonPrefixes);
+    this.commonPrefixes = commonPrefixes == null || commonPrefixes.isEmpty() ? null :
+        new CommonPrefixes(commonPrefixes);
   }
 
   @XmlElement(name = "Name")

server/src/main/java/com/adobe/testing/s3mock/dto/ListBucketResultV2.java

@@ -17,6 +17,7 @@
 package com.adobe.testing.s3mock.dto;
 
 import com.adobe.testing.s3mock.domain.BucketContents;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonRootName;
 import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
@@ -30,6 +31,7 @@
  * Represents a result of listing objects that reside in a Bucket.
  */
 @JsonRootName("ListBucketResult")
+@JsonInclude(JsonInclude.Include.NON_EMPTY)
 public class ListBucketResultV2 implements Serializable {
 
   @JsonProperty("Name")
@@ -95,11 +97,12 @@ public ListBucketResultV2(final String name, final String prefix, final String m
       final String encodingType) {
     this.name = name;
     this.prefix = prefix;
-    this.maxKeys = Integer.valueOf(maxKeys);
+    this.maxKeys = Integer.parseInt(maxKeys);
     this.isTruncated = isTruncated;
     this.contents = new ArrayList<>();
     this.contents.addAll(contents);
-    this.commonPrefixes = new CommonPrefixes(commonPrefixes);
+    this.commonPrefixes = commonPrefixes == null || commonPrefixes.isEmpty() ? null :
+        new CommonPrefixes(commonPrefixes);
     this.continuationToken = continuationToken;
     this.keyCount = keyCount;
     this.nextContinuationToken = nextContinuationToken;

server/src/test/java/com/adobe/testing/s3mock/its/ListObjectIT.java

@@ -29,10 +29,8 @@
 import com.amazonaws.services.s3.model.ListObjectsV2Result;
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
-import java.net.URLDecoder;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.stream.Collectors;
 import org.eclipse.jetty.util.UrlEncoded;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.params.ParameterizedTest;
@@ -89,7 +87,7 @@ Param encodedKeys(final String... expectedKeys) {
 
     String[] decodedKeys() {
       return Arrays.stream(expectedKeys)
-          .map(URLDecoder::decode)
+          .map(UrlEncoded::decodeString)
           .toArray(String[]::new);
     }
 
@@ -173,11 +171,19 @@ public void listV1(final Param parameters) {
         String.join("\n    ", l.getCommonPrefixes()) //
     );
 
+    String[] expectedPrefixes = parameters.expectedPrefixes;
+    // AmazonS3#listObjects does not decode the prefixes, need to encode expected values
+    if (parameters.expectedEncoding != null) {
+      expectedPrefixes = Arrays.stream(parameters.expectedPrefixes)
+          .map(UrlEncoded::encodeString)
+          .toArray(String[]::new);
+    }
+
     assertThat("Returned keys are correct",
         l.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(toList()),
         parameters.expectedKeys.length > 0 ? contains(parameters.expectedKeys) : empty());
     assertThat("Returned prefixes are correct", new ArrayList<>(l.getCommonPrefixes()),
-        parameters.expectedPrefixes.length > 0 ? contains(parameters.expectedPrefixes) : empty());
+        parameters.expectedPrefixes.length > 0 ? contains(expectedPrefixes) : empty());
     assertThat("Returned encodingType is correct", l.getEncodingType(),
         is(equalTo(parameters.expectedEncoding)));
   }
@@ -196,21 +202,22 @@ public void listV2(final Param parameters) {
         .withEncodingType(parameters.expectedEncoding));
 
     LOGGER.info(
-        "list V1, prefix='{}', delimiter='{}', startAfter='{}': "
+        "list V2, prefix='{}', delimiter='{}', startAfter='{}': "
             + "\n  Objects: \n    {}\n  Prefixes: \n    {}\n", //
         parameters.prefix, //
         parameters.delimiter, //
         parameters.startAfter, //
-        l.getObjectSummaries().stream().map(s -> URLDecoder.decode(s.getKey()))
+        l.getObjectSummaries().stream().map(s -> UrlEncoded.decodeString(s.getKey()))
             .collect(joining("\n    ")), //
             String.join("\n    ", l.getCommonPrefixes()) //
     );
     // listV2 automatically decodes the keys so the expected keys have to be decoded
     String[] expectedDecodedKeys = parameters.decodedKeys();
     assertThat("Returned keys are correct",
-        l.getObjectSummaries().stream().map(s -> s.getKey()).collect(toList()),
+        l.getObjectSummaries().stream().map(S3ObjectSummary::getKey).collect(toList()),
         parameters.expectedKeys.length > 0 ? contains(expectedDecodedKeys) : empty());
-    assertThat("Returned prefixes are correct", l.getCommonPrefixes().stream().collect(toList()),
+    // AmazonS3#listObjectsV2 returns decoded prefixes
+    assertThat("Returned prefixes are correct", new ArrayList<>(l.getCommonPrefixes()),
         parameters.expectedPrefixes.length > 0 ? contains(parameters.expectedPrefixes) : empty());
     assertThat("Returned encodingType is correct", l.getEncodingType(),
         is(equalTo(parameters.expectedEncoding)));