From 364bf554667324fd99f537a89ac80d73baaf5861 Mon Sep 17 00:00:00 2001 From: robothy Date: Thu, 11 Apr 2024 17:02:58 +0800 Subject: [PATCH 1/6] virtual host enhancement. --- .../s3/test/VirtualHostIntegrationTest.java | 71 +++++++++++++++++++ .../s3/rest/handler/LocalS3Router.java | 2 +- .../s3/rest/model/request/BucketRegion.java | 2 + .../s3/rest/utils/VirtualHostParser.java | 41 +++++++++-- .../s3/rest/handler/LocalS3RouterTest.java | 6 +- .../s3/rest/utils/VirtualHostParserTest.java | 32 ++++++--- 6 files changed, 137 insertions(+), 17 deletions(-) create mode 100644 local-s3-interationtest/src/test/java/com/robothy/s3/test/VirtualHostIntegrationTest.java diff --git a/local-s3-interationtest/src/test/java/com/robothy/s3/test/VirtualHostIntegrationTest.java b/local-s3-interationtest/src/test/java/com/robothy/s3/test/VirtualHostIntegrationTest.java new file mode 100644 index 0000000..26d6adc --- /dev/null +++ b/local-s3-interationtest/src/test/java/com/robothy/s3/test/VirtualHostIntegrationTest.java @@ -0,0 +1,71 @@ +package com.robothy.s3.test; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.robothy.s3.jupiter.LocalS3; +import com.robothy.s3.jupiter.LocalS3Endpoint; +import org.junit.jupiter.api.Test; + +import java.net.InetAddress; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class VirtualHostIntegrationTest { + + @Test + @LocalS3 + void testVirtualHostWithLocalEndpoint(LocalS3Endpoint endpoint) { + AmazonS3ClientBuilder builder = AmazonS3Client.builder() + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("local-s3-access-key", "local-s3-secret-key"))) + .withClientConfiguration(new ClientConfiguration() + .withDnsResolver(host -> InetAddress.getAllByName("localhost")) + .withConnectionTimeout(1000) + .withSocketTimeout(1000)); + builder.setEndpointConfiguration(endpoint.toAmazonS3EndpointConfiguration()); + + AmazonS3 client = builder.build(); + client.createBucket("my-bucket"); + assertTrue(client.doesBucketExistV2("my-bucket")); + } + + @Test + @LocalS3 + void testVirtualHostWithAwsLegacyGlobalEndpoint(LocalS3Endpoint endpoint) { + AmazonS3ClientBuilder builder = AmazonS3Client.builder() + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("local-s3-access-key", "local-s3-secret-key"))) + .withClientConfiguration(new ClientConfiguration() + .withDnsResolver(host -> InetAddress.getAllByName("localhost")) + .withConnectionTimeout(1000) + .withSocketTimeout(1000)); + builder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://s3.amazonaws.com:" + endpoint.port(), "local")); + AmazonS3 s3 = builder.build(); + s3.createBucket("my-bucket"); + assertTrue(s3.doesBucketExistV2("my-bucket")); + + s3.putObject("my-bucket", "my-key", "Hello, World!"); + assertTrue(s3.doesObjectExist("my-bucket", "my-key")); + } + + @Test + @LocalS3 + void testVirtualHostWithAwsEndpoint(LocalS3Endpoint endpoint) { + AmazonS3ClientBuilder builder = AmazonS3Client.builder() + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials("local-s3-access-key", "local-s3-secret-key"))) + .withClientConfiguration(new ClientConfiguration() + .withDnsResolver(host -> InetAddress.getAllByName("localhost")) + .withConnectionTimeout(1000) + .withSocketTimeout(1000)) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("http://s3.ap-southeast-1.amazonaws.com:" + endpoint.port(), "region1")); + AmazonS3 s3 = builder.build(); + s3.createBucket("my-bucket"); + assertTrue(s3.doesBucketExistV2("my-bucket")); + s3.putObject("my-bucket", "my-key", "Hello, World!"); + assertTrue(s3.doesObjectExist("my-bucket", "my-key")); + } + +} diff --git a/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/LocalS3Router.java b/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/LocalS3Router.java index e7df894..4056762 100644 --- a/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/LocalS3Router.java +++ b/local-s3-rest/src/main/java/com/robothy/s3/rest/handler/LocalS3Router.java @@ -62,7 +62,7 @@ List matchPath(Map> pathRules, HttpRequest request) { Map> params = request.getParams(); - Optional bucketRegion = VirtualHostParser.getBucketRegionFromHost(request.getHeaders().get(HttpHeaderNames.HOST)); + Optional bucketRegion = VirtualHostParser.getBucketRegionFromHost(request.getHeaders().get(HttpHeaderNames.HOST.toString())); boolean bucketNameInPath = !bucketRegion.isPresent() || !bucketRegion.get().getBucketName().isPresent(); String bucketName; String objectKey = null; diff --git a/local-s3-rest/src/main/java/com/robothy/s3/rest/model/request/BucketRegion.java b/local-s3-rest/src/main/java/com/robothy/s3/rest/model/request/BucketRegion.java index 21788d3..9d5e05b 100644 --- a/local-s3-rest/src/main/java/com/robothy/s3/rest/model/request/BucketRegion.java +++ b/local-s3-rest/src/main/java/com/robothy/s3/rest/model/request/BucketRegion.java @@ -3,12 +3,14 @@ import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.ToString; import java.util.Optional; @EqualsAndHashCode @AllArgsConstructor @Getter +@ToString public class BucketRegion { private final String region; diff --git a/local-s3-rest/src/main/java/com/robothy/s3/rest/utils/VirtualHostParser.java b/local-s3-rest/src/main/java/com/robothy/s3/rest/utils/VirtualHostParser.java index 5b09678..3203958 100644 --- a/local-s3-rest/src/main/java/com/robothy/s3/rest/utils/VirtualHostParser.java +++ b/local-s3-rest/src/main/java/com/robothy/s3/rest/utils/VirtualHostParser.java @@ -9,21 +9,36 @@ // bucket-name.s3.{region-id}.{domain} public class VirtualHostParser { - static Set SUPPORTED_DOMAINS = Set.of(".localhost", ".amazonaws.com", ".local"); + static final String AWS_DOMAIN = ".amazonaws.com"; + + static final Set LOCAL_DOMAINS = Set.of(".localhost", ".127.0.0.1", ".0.0.0.0"); public static Optional getBucketRegionFromHost(String host) { if (StringUtils.isBlank(host)) { return Optional.empty(); } - host = host.trim(); - Optional supportedDomainOpt = SUPPORTED_DOMAINS.stream().filter(host::endsWith) + host = removePortIfExist(host.trim()); + + if (host.endsWith(AWS_DOMAIN)) { + return parseHostUnderAwsDomain(host); + } + + + Optional localDomainOpt = LOCAL_DOMAINS.stream().filter(host::endsWith) .map(domainWithDotPrefix -> domainWithDotPrefix.substring(1)).findFirst(); - if (!supportedDomainOpt.isPresent()) { + if (!localDomainOpt.isPresent()) { return Optional.empty(); } - String domain = supportedDomainOpt.get(); + String finalHost = host; + return LOCAL_DOMAINS.stream().filter(host::endsWith) + .findFirst() + .flatMap(localDomainPrependDot -> parseHostFromLocalDomain(finalHost, localDomainPrependDot)); + } + + static Optional parseHostUnderAwsDomain(String host) { + String domain = AWS_DOMAIN.substring(1); if (isLegacyGlobalEndpoint(host, domain)) { // {bucketName}.s3.{domain} return parseLegacyEndpoint(host, domain); } @@ -79,4 +94,20 @@ static boolean isLegacyGlobalEndpoint(String hostWithDomain, String domain) { return hostWithDomain.endsWith(s3WithDomain); } + static String removePortIfExist(String host) { + int colonIdx = host.lastIndexOf(':'); + if (colonIdx > 0) { + return host.substring(0, colonIdx); + } + return host; + } + + static Optional parseHostFromLocalDomain(String host, String localDomainPrependDot) { + if (localDomainPrependDot.endsWith(host)) { + return Optional.empty(); + } + + return Optional.of(new BucketRegion("local", host.substring(0, host.length() - localDomainPrependDot.length()))); + } + } diff --git a/local-s3-rest/src/test/java/com/robothy/s3/rest/handler/LocalS3RouterTest.java b/local-s3-rest/src/test/java/com/robothy/s3/rest/handler/LocalS3RouterTest.java index f2e3f44..220d019 100644 --- a/local-s3-rest/src/test/java/com/robothy/s3/rest/handler/LocalS3RouterTest.java +++ b/local-s3-rest/src/test/java/com/robothy/s3/rest/handler/LocalS3RouterTest.java @@ -73,7 +73,7 @@ void matchPath() { assertTrue(bucketOperation2.parameter("key").isEmpty()); HttpRequest bucketOperation3 = HttpRequest.builder().path("/").build(); - bucketOperation3.getHeaders().put(HttpHeaderNames.HOST, "images.example.com.s3.us-east-1.amazonaws.com"); + bucketOperation3.getHeaders().put(HttpHeaderNames.HOST.toString(), "images.example.com.s3.us-east-1.amazonaws.com"); assertSame(bucketPathRule, localS3Router.matchPath(rules, bucketOperation3)); assertEquals("images.example.com", bucketOperation3.parameter("bucket").get()); assertFalse(bucketOperation3.parameter("key").isPresent()); @@ -99,9 +99,9 @@ void matchPath() { assertEquals("dir/sub-dir/", objectOperation4.parameter("key").get()); HttpRequest objectOperation5 = HttpRequest.builder().path("/a/dir/sub-dir/") - .headers(Map.of(HttpHeaderNames.HOST, "bucket1.s3.region1.localhost")).build(); + .headers(Map.of(HttpHeaderNames.HOST.toString(), "bucket1.s3.localhost")).build(); assertSame(objectPathRule, localS3Router.matchPath(rules, objectOperation5)); - assertEquals("bucket1", objectOperation5.parameter("bucket").get()); + assertEquals("bucket1.s3", objectOperation5.parameter("bucket").get()); assertEquals("a/dir/sub-dir/", objectOperation5.parameter("key").get()); } diff --git a/local-s3-rest/src/test/java/com/robothy/s3/rest/utils/VirtualHostParserTest.java b/local-s3-rest/src/test/java/com/robothy/s3/rest/utils/VirtualHostParserTest.java index 8d5841a..d643d7a 100644 --- a/local-s3-rest/src/test/java/com/robothy/s3/rest/utils/VirtualHostParserTest.java +++ b/local-s3-rest/src/test/java/com/robothy/s3/rest/utils/VirtualHostParserTest.java @@ -8,33 +8,49 @@ class VirtualHostParserTest { @Test - void getBucketRegionFromHost() { + void getBucketRegionFromHostUnderAwsDomain() { assertFalse(VirtualHostParser.getBucketRegionFromHost(null).isPresent()); assertFalse(VirtualHostParser.getBucketRegionFromHost("").isPresent()); assertEquals(new BucketRegion("region1", null), - VirtualHostParser.getBucketRegionFromHost("s3.region1.local").get()); + VirtualHostParser.getBucketRegionFromHost("s3.region1.amazonaws.com").get()); assertFalse(VirtualHostParser.getBucketRegionFromHost("s3.region1.unsupported.domain").isPresent()); - assertFalse(VirtualHostParser.getBucketRegionFromHost(".s3.region1.local").isPresent()); + assertFalse(VirtualHostParser.getBucketRegionFromHost(".s3.region1.amazonaws.com").isPresent()); assertEquals(new BucketRegion("eu-west-1", null), VirtualHostParser.getBucketRegionFromHost("s3.eu-west-1.amazonaws.com").get()); assertEquals(new BucketRegion("region1", "bucket1"), - VirtualHostParser.getBucketRegionFromHost("bucket1.s3.region1.local").get()); + VirtualHostParser.getBucketRegionFromHost("bucket1.s3.region1.amazonaws.com").get()); assertEquals(new BucketRegion("region1", "bucket1"), VirtualHostParser.getBucketRegionFromHost("bucket1.s3.region1.amazonaws.com").get()); assertEquals(new BucketRegion("region2", "www.example.com"), - VirtualHostParser.getBucketRegionFromHost("www.example.com.s3.region2.local").get()); + VirtualHostParser.getBucketRegionFromHost("www.example.com.s3.region2.amazonaws.com").get()); assertEquals(new BucketRegion("ap-east-1", "www.example.com"), VirtualHostParser.getBucketRegionFromHost("www.example.com.s3.ap-east-1.amazonaws.com").get()); // if using the legacy endpoint, then set the default region "local". assertEquals(new BucketRegion("local", "bucket1"), - VirtualHostParser.getBucketRegionFromHost("bucket1.s3.localhost").get()); + VirtualHostParser.getBucketRegionFromHost("bucket1.s3.amazonaws.com").get()); + assertEquals(new BucketRegion("local", "www.example.com"), + VirtualHostParser.getBucketRegionFromHost("www.example.com.s3.amazonaws.com").get()); + assertFalse(VirtualHostParser.getBucketRegionFromHost(".s3.amazonaws.com").isPresent()); + } + + + @Test + public void getBucketRegionFromLocalDomain() { + assertEquals(new BucketRegion("local", "bucket1"), + VirtualHostParser.getBucketRegionFromHost("bucket1.localhost").get()); assertEquals(new BucketRegion("local", "www.example.com"), - VirtualHostParser.getBucketRegionFromHost("www.example.com.s3.localhost").get()); - assertFalse(VirtualHostParser.getBucketRegionFromHost(".s3.localhost").isPresent()); + VirtualHostParser.getBucketRegionFromHost("www.example.com.localhost").get()); + assertEquals(new BucketRegion("local", "bucket.s3"), + VirtualHostParser.getBucketRegionFromHost("bucket.s3.localhost").get()); + assertEquals(new BucketRegion("local", "bucket.s3."), + VirtualHostParser.getBucketRegionFromHost("bucket.s3..localhost").get()); + + assertFalse(VirtualHostParser.getBucketRegionFromHost("localhost").isPresent()); + assertFalse(VirtualHostParser.getBucketRegionFromHost("127.0.0.1").isPresent()); } } \ No newline at end of file From 93c1603a93b3033514b443288710fe21ecf37e2e Mon Sep 17 00:00:00 2001 From: robothy Date: Thu, 11 Apr 2024 17:14:20 +0800 Subject: [PATCH 2/6] upgrade codecov-action --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 02a4bdd..e4d826b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,6 +55,6 @@ jobs: run: ./gradlew mergeReports - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4.3.0 with: files: ./build/reports/jacoco/mergeReports/mergeReports.xml From e88b61c81cac5b5bac0cf90326b30927dada8b91 Mon Sep 17 00:00:00 2001 From: robothy Date: Thu, 11 Apr 2024 17:26:43 +0800 Subject: [PATCH 3/6] set Codecov token. --- .github/workflows/build.yml | 1 + .github/workflows/pr.yml | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e4d826b..15c4713 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,3 +58,4 @@ jobs: uses: codecov/codecov-action@v4.3.0 with: files: ./build/reports/jacoco/mergeReports/mergeReports.xml + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3f035b4..b8cae84 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -55,4 +55,5 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: - files: ./build/reports/jacoco/mergeReports/mergeReports.xml \ No newline at end of file + files: ./build/reports/jacoco/mergeReports/mergeReports.xml + token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file From 8912e0c892af5ac9c8fffce671a78a32ca4129c2 Mon Sep 17 00:00:00 2001 From: robothy Date: Thu, 11 Apr 2024 17:37:33 +0800 Subject: [PATCH 4/6] update token --- .github/workflows/build.yml | 2 +- .github/workflows/pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 15c4713..fb448a9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,4 +58,4 @@ jobs: uses: codecov/codecov-action@v4.3.0 with: files: ./build/reports/jacoco/mergeReports/mergeReports.xml - token: ${{ secrets.CODECOV_TOKEN }} + token: ${{ secrets.CODECOV_GLOBAL_UPLOAD_TOKEN }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b8cae84..ba5bbf6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -56,4 +56,4 @@ jobs: uses: codecov/codecov-action@v3 with: files: ./build/reports/jacoco/mergeReports/mergeReports.xml - token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + token: ${{ secrets.CODECOV_GLOBAL_UPLOAD_TOKEN }} \ No newline at end of file From b7879fdd84056a815653931f59df1f1643cf3ca2 Mon Sep 17 00:00:00 2001 From: robothy Date: Thu, 11 Apr 2024 17:46:16 +0800 Subject: [PATCH 5/6] update codecov-actions version --- .github/workflows/build.yml | 2 +- .github/workflows/pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fb448a9..d4c8a97 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,7 +55,7 @@ jobs: run: ./gradlew mergeReports - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4.3.0 + uses: codecov/codecov-action@v4 with: files: ./build/reports/jacoco/mergeReports/mergeReports.xml token: ${{ secrets.CODECOV_GLOBAL_UPLOAD_TOKEN }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ba5bbf6..83e69da 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -53,7 +53,7 @@ jobs: run: ./gradlew mergeReports - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: files: ./build/reports/jacoco/mergeReports/mergeReports.xml token: ${{ secrets.CODECOV_GLOBAL_UPLOAD_TOKEN }} \ No newline at end of file From b00251c6521d8bb396e4c2da545f7cb0b94cd933 Mon Sep 17 00:00:00 2001 From: robothy Date: Thu, 11 Apr 2024 17:58:15 +0800 Subject: [PATCH 6/6] add test case --- .../java/com/robothy/s3/rest/utils/VirtualHostParserTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/local-s3-rest/src/test/java/com/robothy/s3/rest/utils/VirtualHostParserTest.java b/local-s3-rest/src/test/java/com/robothy/s3/rest/utils/VirtualHostParserTest.java index d643d7a..3c73915 100644 --- a/local-s3-rest/src/test/java/com/robothy/s3/rest/utils/VirtualHostParserTest.java +++ b/local-s3-rest/src/test/java/com/robothy/s3/rest/utils/VirtualHostParserTest.java @@ -50,6 +50,7 @@ public void getBucketRegionFromLocalDomain() { VirtualHostParser.getBucketRegionFromHost("bucket.s3..localhost").get()); assertFalse(VirtualHostParser.getBucketRegionFromHost("localhost").isPresent()); + assertFalse(VirtualHostParser.getBucketRegionFromHost(".localhost").isPresent()); assertFalse(VirtualHostParser.getBucketRegionFromHost("127.0.0.1").isPresent()); }