virtual host enhancement.

.github/workflows/build.yml

@@ -55,6 +55,7 @@ jobs:
         run: ./gradlew mergeReports
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./build/reports/jacoco/mergeReports/mergeReports.xml
+          token: ${{ secrets.CODECOV_GLOBAL_UPLOAD_TOKEN }}

.github/workflows/pr.yml

@@ -53,6 +53,7 @@ jobs:
         run: ./gradlew mergeReports
 
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
-          files: ./build/reports/jacoco/mergeReports/mergeReports.xml
+          files: ./build/reports/jacoco/mergeReports/mergeReports.xml
+          token: ${{ secrets.CODECOV_GLOBAL_UPLOAD_TOKEN }}

local-s3-interationtest/src/test/java/com/robothy/s3/test/VirtualHostIntegrationTest.java

@@ -0,0 +1,71 @@
+package com.robothy.s3.test;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.robothy.s3.jupiter.LocalS3;
+import com.robothy.s3.jupiter.LocalS3Endpoint;
+import org.junit.jupiter.api.Test;
+
+import java.net.InetAddress;
+
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class VirtualHostIntegrationTest {
+
+  @Test
+  @LocalS3
+  void testVirtualHostWithLocalEndpoint(LocalS3Endpoint endpoint) {
+    AmazonS3ClientBuilder builder = AmazonS3Client.builder()
+        .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("local-s3-access-key", "local-s3-secret-key")))
+        .withClientConfiguration(new ClientConfiguration()
+            .withDnsResolver(host -> InetAddress.getAllByName("localhost"))
+            .withConnectionTimeout(1000)
+            .withSocketTimeout(1000));
+    builder.setEndpointConfiguration(endpoint.toAmazonS3EndpointConfiguration());
+
+    AmazonS3 client = builder.build();
+    client.createBucket("my-bucket");
+    assertTrue(client.doesBucketExistV2("my-bucket"));
+  }
+
+  @Test
+  @LocalS3
+  void testVirtualHostWithAwsLegacyGlobalEndpoint(LocalS3Endpoint endpoint) {
+    AmazonS3ClientBuilder builder = AmazonS3Client.builder()
+        .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("local-s3-access-key", "local-s3-secret-key")))
+        .withClientConfiguration(new ClientConfiguration()
+            .withDnsResolver(host -> InetAddress.getAllByName("localhost"))
+            .withConnectionTimeout(1000)
+            .withSocketTimeout(1000));
+    builder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://s3.amazonaws.com:" + endpoint.port(), "local"));
+    AmazonS3 s3 = builder.build();
+    s3.createBucket("my-bucket");
+    assertTrue(s3.doesBucketExistV2("my-bucket"));
+
+    s3.putObject("my-bucket", "my-key", "Hello, World!");
+    assertTrue(s3.doesObjectExist("my-bucket", "my-key"));
+  }
+
+  @Test
+  @LocalS3
+  void testVirtualHostWithAwsEndpoint(LocalS3Endpoint endpoint) {
+    AmazonS3ClientBuilder builder = AmazonS3Client.builder()
+        .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("local-s3-access-key", "local-s3-secret-key")))
+        .withClientConfiguration(new ClientConfiguration()
+            .withDnsResolver(host -> InetAddress.getAllByName("localhost"))
+            .withConnectionTimeout(1000)
+            .withSocketTimeout(1000))
+        .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://s3.ap-southeast-1.amazonaws.com:" + endpoint.port(), "region1"));
+    AmazonS3 s3 = builder.build();
+    s3.createBucket("my-bucket");
+    assertTrue(s3.doesBucketExistV2("my-bucket"));
+    s3.putObject("my-bucket", "my-key", "Hello, World!");
+    assertTrue(s3.doesObjectExist("my-bucket", "my-key"));
+  }
+
+}

local-s3-rest/src/main/java/com/robothy/s3/rest/handler/LocalS3Router.java

@@ -62,7 +62,7 @@ List<Route> matchPath(Map<String, List<Route>> pathRules, HttpRequest request) {
 
     Map<CharSequence, List<String>> params = request.getParams();
 
-    Optional<BucketRegion> bucketRegion = VirtualHostParser.getBucketRegionFromHost(request.getHeaders().get(HttpHeaderNames.HOST));
+    Optional<BucketRegion> bucketRegion = VirtualHostParser.getBucketRegionFromHost(request.getHeaders().get(HttpHeaderNames.HOST.toString()));
     boolean bucketNameInPath = !bucketRegion.isPresent() || !bucketRegion.get().getBucketName().isPresent();
     String bucketName;
     String objectKey = null;

local-s3-rest/src/main/java/com/robothy/s3/rest/model/request/BucketRegion.java

@@ -3,12 +3,14 @@
 import lombok.AllArgsConstructor;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
+import lombok.ToString;
 
 import java.util.Optional;
 
 @EqualsAndHashCode
 @AllArgsConstructor
 @Getter
+@ToString
 public class BucketRegion {
 
   private final String region;

local-s3-rest/src/main/java/com/robothy/s3/rest/utils/VirtualHostParser.java

@@ -9,21 +9,36 @@
 // bucket-name.s3.{region-id}.{domain}
 public class VirtualHostParser {
 
-  static Set<String> SUPPORTED_DOMAINS = Set.of(".localhost", ".amazonaws.com", ".local");
+  static final String AWS_DOMAIN = ".amazonaws.com";
+
+  static final Set<String> LOCAL_DOMAINS = Set.of(".localhost", ".127.0.0.1", ".0.0.0.0");
 
   public static Optional<BucketRegion> getBucketRegionFromHost(String host) {
     if (StringUtils.isBlank(host)) {
       return Optional.empty();
     }
 
-    host = host.trim();
-    Optional<String> supportedDomainOpt = SUPPORTED_DOMAINS.stream().filter(host::endsWith)
+    host = removePortIfExist(host.trim());
+
+    if (host.endsWith(AWS_DOMAIN)) {
+      return parseHostUnderAwsDomain(host);
+    }
+
+
+    Optional<String> localDomainOpt = LOCAL_DOMAINS.stream().filter(host::endsWith)
         .map(domainWithDotPrefix -> domainWithDotPrefix.substring(1)).findFirst();
-    if (!supportedDomainOpt.isPresent()) {
+    if (!localDomainOpt.isPresent()) {
       return Optional.empty();
     }
 
-    String domain = supportedDomainOpt.get();
+    String finalHost = host;
+    return LOCAL_DOMAINS.stream().filter(host::endsWith)
+        .findFirst()
+        .flatMap(localDomainPrependDot -> parseHostFromLocalDomain(finalHost, localDomainPrependDot));
+  }
+
+  static Optional<BucketRegion> parseHostUnderAwsDomain(String host) {
+    String domain = AWS_DOMAIN.substring(1);
     if (isLegacyGlobalEndpoint(host, domain)) { // {bucketName}.s3.{domain}
       return parseLegacyEndpoint(host, domain);
     }
@@ -79,4 +94,20 @@ static boolean isLegacyGlobalEndpoint(String hostWithDomain, String domain) {
     return hostWithDomain.endsWith(s3WithDomain);
   }
 
+  static String removePortIfExist(String host) {
+    int colonIdx = host.lastIndexOf(':');
+    if (colonIdx > 0) {
+      return host.substring(0, colonIdx);
+    }
+    return host;
+  }
+
+  static Optional<BucketRegion> parseHostFromLocalDomain(String host, String localDomainPrependDot) {
+    if (localDomainPrependDot.endsWith(host)) {
+      return Optional.empty();
+    }
+
+    return Optional.of(new BucketRegion("local", host.substring(0, host.length() - localDomainPrependDot.length())));
+  }
+
 }

local-s3-rest/src/test/java/com/robothy/s3/rest/handler/LocalS3RouterTest.java

@@ -73,7 +73,7 @@ void matchPath() {
     assertTrue(bucketOperation2.parameter("key").isEmpty());
 
     HttpRequest bucketOperation3 = HttpRequest.builder().path("/").build();
-    bucketOperation3.getHeaders().put(HttpHeaderNames.HOST, "images.example.com.s3.us-east-1.amazonaws.com");
+    bucketOperation3.getHeaders().put(HttpHeaderNames.HOST.toString(), "images.example.com.s3.us-east-1.amazonaws.com");
     assertSame(bucketPathRule, localS3Router.matchPath(rules, bucketOperation3));
     assertEquals("images.example.com", bucketOperation3.parameter("bucket").get());
     assertFalse(bucketOperation3.parameter("key").isPresent());
@@ -99,9 +99,9 @@ void matchPath() {
     assertEquals("dir/sub-dir/", objectOperation4.parameter("key").get());
 
     HttpRequest objectOperation5 = HttpRequest.builder().path("/a/dir/sub-dir/")
-        .headers(Map.of(HttpHeaderNames.HOST, "bucket1.s3.region1.localhost")).build();
+        .headers(Map.of(HttpHeaderNames.HOST.toString(), "bucket1.s3.localhost")).build();
     assertSame(objectPathRule, localS3Router.matchPath(rules, objectOperation5));
-    assertEquals("bucket1", objectOperation5.parameter("bucket").get());
+    assertEquals("bucket1.s3", objectOperation5.parameter("bucket").get());
     assertEquals("a/dir/sub-dir/", objectOperation5.parameter("key").get());
   }
 

local-s3-rest/src/test/java/com/robothy/s3/rest/utils/VirtualHostParserTest.java

@@ -8,33 +8,50 @@
 class VirtualHostParserTest {
 
   @Test
-  void getBucketRegionFromHost() {
+  void getBucketRegionFromHostUnderAwsDomain() {
     assertFalse(VirtualHostParser.getBucketRegionFromHost(null).isPresent());
     assertFalse(VirtualHostParser.getBucketRegionFromHost("").isPresent());
 
     assertEquals(new BucketRegion("region1", null),
-        VirtualHostParser.getBucketRegionFromHost("s3.region1.local").get());
+        VirtualHostParser.getBucketRegionFromHost("s3.region1.amazonaws.com").get());
     assertFalse(VirtualHostParser.getBucketRegionFromHost("s3.region1.unsupported.domain").isPresent());
 
-    assertFalse(VirtualHostParser.getBucketRegionFromHost(".s3.region1.local").isPresent());
+    assertFalse(VirtualHostParser.getBucketRegionFromHost(".s3.region1.amazonaws.com").isPresent());
     assertEquals(new BucketRegion("eu-west-1", null), VirtualHostParser.getBucketRegionFromHost("s3.eu-west-1.amazonaws.com").get());
 
     assertEquals(new BucketRegion("region1", "bucket1"),
-        VirtualHostParser.getBucketRegionFromHost("bucket1.s3.region1.local").get());
+        VirtualHostParser.getBucketRegionFromHost("bucket1.s3.region1.amazonaws.com").get());
     assertEquals(new BucketRegion("region1", "bucket1"),
         VirtualHostParser.getBucketRegionFromHost("bucket1.s3.region1.amazonaws.com").get());
 
     assertEquals(new BucketRegion("region2", "www.example.com"),
-        VirtualHostParser.getBucketRegionFromHost("www.example.com.s3.region2.local").get());
+        VirtualHostParser.getBucketRegionFromHost("www.example.com.s3.region2.amazonaws.com").get());
     assertEquals(new BucketRegion("ap-east-1", "www.example.com"),
         VirtualHostParser.getBucketRegionFromHost("www.example.com.s3.ap-east-1.amazonaws.com").get());
 
     // if using the legacy endpoint, then set the default region "local".
     assertEquals(new BucketRegion("local", "bucket1"),
-        VirtualHostParser.getBucketRegionFromHost("bucket1.s3.localhost").get());
+        VirtualHostParser.getBucketRegionFromHost("bucket1.s3.amazonaws.com").get());
+    assertEquals(new BucketRegion("local", "www.example.com"),
+        VirtualHostParser.getBucketRegionFromHost("www.example.com.s3.amazonaws.com").get());
+    assertFalse(VirtualHostParser.getBucketRegionFromHost(".s3.amazonaws.com").isPresent());
+  }
+
+
+  @Test
+  public void getBucketRegionFromLocalDomain() {
+    assertEquals(new BucketRegion("local", "bucket1"),
+        VirtualHostParser.getBucketRegionFromHost("bucket1.localhost").get());
     assertEquals(new BucketRegion("local", "www.example.com"),
-        VirtualHostParser.getBucketRegionFromHost("www.example.com.s3.localhost").get());
-    assertFalse(VirtualHostParser.getBucketRegionFromHost(".s3.localhost").isPresent());
+        VirtualHostParser.getBucketRegionFromHost("www.example.com.localhost").get());
+    assertEquals(new BucketRegion("local", "bucket.s3"),
+        VirtualHostParser.getBucketRegionFromHost("bucket.s3.localhost").get());
+    assertEquals(new BucketRegion("local", "bucket.s3."),
+        VirtualHostParser.getBucketRegionFromHost("bucket.s3..localhost").get());
+
+    assertFalse(VirtualHostParser.getBucketRegionFromHost("localhost").isPresent());
+    assertFalse(VirtualHostParser.getBucketRegionFromHost(".localhost").isPresent());
+    assertFalse(VirtualHostParser.getBucketRegionFromHost("127.0.0.1").isPresent());
   }
 
 }