From f99142997f80a2e76e0660ed0434a2a4e5334f5d Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 29 Sep 2023 13:37:24 +0200 Subject: [PATCH 01/64] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e536fd9..9c48b3ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ All notable changes to this project will be documented in this file. +## [[NEXT]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/vNEXT) 2023 + ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 ### Bug Fixes From e4f4235c490f3018a2def44be340bc1dec753ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Mon, 23 Oct 2023 12:04:17 +0200 Subject: [PATCH 02/64] Add a security filter to activate an API Key mechanism on endpoints --- CHANGELOG.md | 1 + .../iexec/sms/utils/ApiKeyRequestFilter.java | 78 +++++++++++++++++++ .../sms/utils/ApiKeyRequestFilterTest.java | 53 +++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java create mode 100644 src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c48b3ed..88ea9f8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file. ## [[NEXT]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/vNEXT) 2023 +- Add a security filter to activate an API Key mechanism on endpoints. (#207) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 diff --git a/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java b/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java new file mode 100644 index 00000000..3393922c --- /dev/null +++ b/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java @@ -0,0 +1,78 @@ +/* + * Copyright 2023 IEXEC BLOCKCHAIN TECH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.iexec.sms.utils; + +import lombok.Getter; +import org.springframework.web.filter.GenericFilterBean; + +import lombok.extern.slf4j.Slf4j; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; + +/** + * ApiKeyRequestFilter is a security filter that can be activated to protect endpoints. + *

+ * It is based on the use of an API Key that the caller must fill in via the X-API-KEY header. + * If this filter is activated and the caller does not enter an API Key, then a 401 will be raised + */ +@Slf4j +public class ApiKeyRequestFilter extends GenericFilterBean { + + + private static final String API_KEY_HEADER_NAME = "X-API-KEY"; //Name of header in which api key is expected + @Getter + private String apiKey = ""; //The filter API Key + @Getter + private boolean isEnabled = false; + + public ApiKeyRequestFilter(String apiKey){ + if (null != apiKey && !apiKey.isBlank()){ + this.apiKey = apiKey; + this.isEnabled = true; + }else{ + log.warn("API Key filter is not enabled"); + } + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + + if (this.isEnabled()) { + HttpServletRequest req = (HttpServletRequest) request; + String key = req.getHeader(API_KEY_HEADER_NAME) == null ? "" : req.getHeader(API_KEY_HEADER_NAME); + if ( key.isBlank() || !key.equalsIgnoreCase(this.getApiKey()) ) { + HttpServletResponse resp = (HttpServletResponse) response; + String error = "You are not authorized to access this endpoint"; + + resp.reset(); + resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED); + response.setContentLength(error.length()); + response.getWriter().write(error); + }else{ + chain.doFilter(request, response); + } + }else{ + chain.doFilter(request, response); + } + } +} diff --git a/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java b/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java new file mode 100644 index 00000000..7813efc6 --- /dev/null +++ b/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java @@ -0,0 +1,53 @@ +package com.iexec.sms.utils; + +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockFilterChain; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +import javax.servlet.http.HttpServletResponse; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ApiKeyRequestFilterTest { + + private final String apiKey = "e54fdf4s56df4g"; + + @Test + void shouldPassTheFilterWhenFilterIsActiveAndApiKeyIsCorrect() throws Exception { + ApiKeyRequestFilter filter = new ApiKeyRequestFilter(apiKey); + MockHttpServletRequest req = new MockHttpServletRequest(); + MockHttpServletResponse res = new MockHttpServletResponse(); + MockFilterChain chain = new MockFilterChain(); + + req.addHeader("X-API-KEY", apiKey); + + filter.doFilter(req,res,chain); + + assertEquals(HttpServletResponse.SC_OK, res.getStatus()); + } + + @Test + void shouldPassTheFilterWhenFilterIsInactive() throws Exception { + ApiKeyRequestFilter filter = new ApiKeyRequestFilter(null); + MockHttpServletRequest req = new MockHttpServletRequest(); + MockHttpServletResponse res = new MockHttpServletResponse(); + MockFilterChain chain = new MockFilterChain(); + + filter.doFilter(req,res,chain); + + assertEquals(HttpServletResponse.SC_OK, res.getStatus()); + } + + @Test + void shouldNotPassTheFilterWhenFilterIsActiveAndApiKeyIsIncorrect() throws Exception { + ApiKeyRequestFilter filter = new ApiKeyRequestFilter(apiKey); + MockHttpServletRequest req = new MockHttpServletRequest(); + MockHttpServletResponse res = new MockHttpServletResponse(); + MockFilterChain chain = new MockFilterChain(); + + filter.doFilter(req,res,chain); + + assertEquals(HttpServletResponse.SC_UNAUTHORIZED, res.getStatus()); + } +} From 342cb0829a1700c17e554ce3a22acecba67be81c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Mon, 23 Oct 2023 12:06:38 +0200 Subject: [PATCH 03/64] Add header copyright on test file --- .../iexec/sms/utils/ApiKeyRequestFilterTest.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java b/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java index 7813efc6..165ec02e 100644 --- a/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java +++ b/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java @@ -1,3 +1,19 @@ +/* + * Copyright 2023 IEXEC BLOCKCHAIN TECH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.iexec.sms.utils; import org.junit.jupiter.api.Test; From f8122f23ef3d73c12038ce98b7b78ba07d708458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Mon, 23 Oct 2023 12:19:11 +0200 Subject: [PATCH 04/64] Add more tests --- .../sms/utils/ApiKeyRequestFilterTest.java | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java b/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java index 165ec02e..d887bd9f 100644 --- a/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java +++ b/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java @@ -44,7 +44,7 @@ void shouldPassTheFilterWhenFilterIsActiveAndApiKeyIsCorrect() throws Exception } @Test - void shouldPassTheFilterWhenFilterIsInactive() throws Exception { + void shouldPassTheFilterWhenTheFilterIsInactiveDueToAnApiKeyConfiguredToNull() throws Exception { ApiKeyRequestFilter filter = new ApiKeyRequestFilter(null); MockHttpServletRequest req = new MockHttpServletRequest(); MockHttpServletResponse res = new MockHttpServletResponse(); @@ -55,6 +55,30 @@ void shouldPassTheFilterWhenFilterIsInactive() throws Exception { assertEquals(HttpServletResponse.SC_OK, res.getStatus()); } + @Test + void shouldPassTheFilterWhenTheFilterIsInactiveDueToAnApiKeyConfiguredToBlank() throws Exception { + ApiKeyRequestFilter filter = new ApiKeyRequestFilter(""); + MockHttpServletRequest req = new MockHttpServletRequest(); + MockHttpServletResponse res = new MockHttpServletResponse(); + MockFilterChain chain = new MockFilterChain(); + + filter.doFilter(req,res,chain); + + assertEquals(HttpServletResponse.SC_OK, res.getStatus()); + } + + @Test + void shouldNotPassTheFilterWhenFilterIsActiveAndApiKeyIsNotFilled() throws Exception { + ApiKeyRequestFilter filter = new ApiKeyRequestFilter(apiKey); + MockHttpServletRequest req = new MockHttpServletRequest(); + MockHttpServletResponse res = new MockHttpServletResponse(); + MockFilterChain chain = new MockFilterChain(); + + filter.doFilter(req,res,chain); + + assertEquals(HttpServletResponse.SC_UNAUTHORIZED, res.getStatus()); + } + @Test void shouldNotPassTheFilterWhenFilterIsActiveAndApiKeyIsIncorrect() throws Exception { ApiKeyRequestFilter filter = new ApiKeyRequestFilter(apiKey); @@ -62,6 +86,8 @@ void shouldNotPassTheFilterWhenFilterIsActiveAndApiKeyIsIncorrect() throws Excep MockHttpServletResponse res = new MockHttpServletResponse(); MockFilterChain chain = new MockFilterChain(); + req.addHeader("X-API-KEY", "INCORRECT API KEY"); + filter.doFilter(req,res,chain); assertEquals(HttpServletResponse.SC_UNAUTHORIZED, res.getStatus()); From 4c045586fadccef2c507d61493b81afcd0385745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Mon, 23 Oct 2023 14:22:34 +0200 Subject: [PATCH 05/64] Tests refactoring and code reformating --- .../iexec/sms/utils/ApiKeyRequestFilter.java | 16 +++---- .../sms/utils/ApiKeyRequestFilterTest.java | 43 +++++++------------ 2 files changed, 24 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java b/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java index 3393922c..50d6cfbb 100644 --- a/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java +++ b/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java @@ -17,9 +17,9 @@ package com.iexec.sms.utils; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.springframework.web.filter.GenericFilterBean; -import lombok.extern.slf4j.Slf4j; import javax.servlet.FilterChain; import javax.servlet.ServletException; import javax.servlet.ServletRequest; @@ -35,7 +35,7 @@ * If this filter is activated and the caller does not enter an API Key, then a 401 will be raised */ @Slf4j -public class ApiKeyRequestFilter extends GenericFilterBean { +public class ApiKeyRequestFilter extends GenericFilterBean { private static final String API_KEY_HEADER_NAME = "X-API-KEY"; //Name of header in which api key is expected @@ -44,11 +44,11 @@ public class ApiKeyRequestFilter extends GenericFilterBean { @Getter private boolean isEnabled = false; - public ApiKeyRequestFilter(String apiKey){ - if (null != apiKey && !apiKey.isBlank()){ + public ApiKeyRequestFilter(String apiKey) { + if (null != apiKey && !apiKey.isBlank()) { this.apiKey = apiKey; this.isEnabled = true; - }else{ + } else { log.warn("API Key filter is not enabled"); } } @@ -60,7 +60,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha if (this.isEnabled()) { HttpServletRequest req = (HttpServletRequest) request; String key = req.getHeader(API_KEY_HEADER_NAME) == null ? "" : req.getHeader(API_KEY_HEADER_NAME); - if ( key.isBlank() || !key.equalsIgnoreCase(this.getApiKey()) ) { + if (key.isBlank() || !key.equalsIgnoreCase(this.getApiKey())) { HttpServletResponse resp = (HttpServletResponse) response; String error = "You are not authorized to access this endpoint"; @@ -68,10 +68,10 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha resp.setStatus(HttpServletResponse.SC_UNAUTHORIZED); response.setContentLength(error.length()); response.getWriter().write(error); - }else{ + } else { chain.doFilter(request, response); } - }else{ + } else { chain.doFilter(request, response); } } diff --git a/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java b/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java index d887bd9f..684ee207 100644 --- a/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java +++ b/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java @@ -16,6 +16,7 @@ package com.iexec.sms.utils; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.mock.web.MockFilterChain; import org.springframework.mock.web.MockHttpServletRequest; @@ -28,17 +29,22 @@ class ApiKeyRequestFilterTest { private final String apiKey = "e54fdf4s56df4g"; + private MockHttpServletRequest req; + private MockHttpServletResponse res; + private MockFilterChain chain; + + @BeforeEach + public void init() { + req = new MockHttpServletRequest(); + res = new MockHttpServletResponse(); + chain = new MockFilterChain(); + } @Test void shouldPassTheFilterWhenFilterIsActiveAndApiKeyIsCorrect() throws Exception { ApiKeyRequestFilter filter = new ApiKeyRequestFilter(apiKey); - MockHttpServletRequest req = new MockHttpServletRequest(); - MockHttpServletResponse res = new MockHttpServletResponse(); - MockFilterChain chain = new MockFilterChain(); - req.addHeader("X-API-KEY", apiKey); - - filter.doFilter(req,res,chain); + filter.doFilter(req, res, chain); assertEquals(HttpServletResponse.SC_OK, res.getStatus()); } @@ -46,11 +52,7 @@ void shouldPassTheFilterWhenFilterIsActiveAndApiKeyIsCorrect() throws Exception @Test void shouldPassTheFilterWhenTheFilterIsInactiveDueToAnApiKeyConfiguredToNull() throws Exception { ApiKeyRequestFilter filter = new ApiKeyRequestFilter(null); - MockHttpServletRequest req = new MockHttpServletRequest(); - MockHttpServletResponse res = new MockHttpServletResponse(); - MockFilterChain chain = new MockFilterChain(); - - filter.doFilter(req,res,chain); + filter.doFilter(req, res, chain); assertEquals(HttpServletResponse.SC_OK, res.getStatus()); } @@ -58,11 +60,7 @@ void shouldPassTheFilterWhenTheFilterIsInactiveDueToAnApiKeyConfiguredToNull() t @Test void shouldPassTheFilterWhenTheFilterIsInactiveDueToAnApiKeyConfiguredToBlank() throws Exception { ApiKeyRequestFilter filter = new ApiKeyRequestFilter(""); - MockHttpServletRequest req = new MockHttpServletRequest(); - MockHttpServletResponse res = new MockHttpServletResponse(); - MockFilterChain chain = new MockFilterChain(); - - filter.doFilter(req,res,chain); + filter.doFilter(req, res, chain); assertEquals(HttpServletResponse.SC_OK, res.getStatus()); } @@ -70,11 +68,7 @@ void shouldPassTheFilterWhenTheFilterIsInactiveDueToAnApiKeyConfiguredToBlank() @Test void shouldNotPassTheFilterWhenFilterIsActiveAndApiKeyIsNotFilled() throws Exception { ApiKeyRequestFilter filter = new ApiKeyRequestFilter(apiKey); - MockHttpServletRequest req = new MockHttpServletRequest(); - MockHttpServletResponse res = new MockHttpServletResponse(); - MockFilterChain chain = new MockFilterChain(); - - filter.doFilter(req,res,chain); + filter.doFilter(req, res, chain); assertEquals(HttpServletResponse.SC_UNAUTHORIZED, res.getStatus()); } @@ -82,13 +76,8 @@ void shouldNotPassTheFilterWhenFilterIsActiveAndApiKeyIsNotFilled() throws Excep @Test void shouldNotPassTheFilterWhenFilterIsActiveAndApiKeyIsIncorrect() throws Exception { ApiKeyRequestFilter filter = new ApiKeyRequestFilter(apiKey); - MockHttpServletRequest req = new MockHttpServletRequest(); - MockHttpServletResponse res = new MockHttpServletResponse(); - MockFilterChain chain = new MockFilterChain(); - req.addHeader("X-API-KEY", "INCORRECT API KEY"); - - filter.doFilter(req,res,chain); + filter.doFilter(req, res, chain); assertEquals(HttpServletResponse.SC_UNAUTHORIZED, res.getStatus()); } From b332908e0efbb655821adb0204290addfc5ae92d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Mon, 23 Oct 2023 14:51:53 +0200 Subject: [PATCH 06/64] Improving the code --- .../iexec/sms/utils/ApiKeyRequestFilter.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java b/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java index 50d6cfbb..b033b280 100644 --- a/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java +++ b/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java @@ -16,7 +16,6 @@ package com.iexec.sms.utils; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.web.filter.GenericFilterBean; @@ -39,16 +38,17 @@ public class ApiKeyRequestFilter extends GenericFilterBean { private static final String API_KEY_HEADER_NAME = "X-API-KEY"; //Name of header in which api key is expected - @Getter - private String apiKey = ""; //The filter API Key - @Getter - private boolean isEnabled = false; + private final String apiKey; //The filter API Key + + private final boolean isEnabled; public ApiKeyRequestFilter(String apiKey) { if (null != apiKey && !apiKey.isBlank()) { this.apiKey = apiKey; this.isEnabled = true; } else { + this.apiKey = null; + this.isEnabled = false; log.warn("API Key filter is not enabled"); } } @@ -57,10 +57,11 @@ public ApiKeyRequestFilter(String apiKey) { public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { - if (this.isEnabled()) { + if (this.isEnabled) { HttpServletRequest req = (HttpServletRequest) request; - String key = req.getHeader(API_KEY_HEADER_NAME) == null ? "" : req.getHeader(API_KEY_HEADER_NAME); - if (key.isBlank() || !key.equalsIgnoreCase(this.getApiKey())) { + + String key = req.getHeader(API_KEY_HEADER_NAME); + if (!this.apiKey.equalsIgnoreCase(key)) { HttpServletResponse resp = (HttpServletResponse) response; String error = "You are not authorized to access this endpoint"; From 98184d154d589925d011423a4abee1e268ed8475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Mon, 23 Oct 2023 15:22:34 +0200 Subject: [PATCH 07/64] Use of a ParameterizedTest instead of 3 separate tests --- .../sms/utils/ApiKeyRequestFilterTest.java | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java b/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java index 684ee207..1507b3dc 100644 --- a/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java +++ b/src/test/java/com/iexec/sms/utils/ApiKeyRequestFilterTest.java @@ -18,6 +18,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; import org.springframework.mock.web.MockFilterChain; import org.springframework.mock.web.MockHttpServletRequest; import org.springframework.mock.web.MockHttpServletResponse; @@ -40,28 +43,15 @@ public void init() { chain = new MockFilterChain(); } - @Test - void shouldPassTheFilterWhenFilterIsActiveAndApiKeyIsCorrect() throws Exception { - ApiKeyRequestFilter filter = new ApiKeyRequestFilter(apiKey); - req.addHeader("X-API-KEY", apiKey); - filter.doFilter(req, res, chain); - - assertEquals(HttpServletResponse.SC_OK, res.getStatus()); - } - - @Test - void shouldPassTheFilterWhenTheFilterIsInactiveDueToAnApiKeyConfiguredToNull() throws Exception { - ApiKeyRequestFilter filter = new ApiKeyRequestFilter(null); + @ParameterizedTest + @NullSource + @ValueSource(strings = {"", apiKey}) + void shouldBeOk(String requestApiKey) throws Exception { + ApiKeyRequestFilter filter = new ApiKeyRequestFilter(requestApiKey); + if (null != requestApiKey && !requestApiKey.isBlank()) { + req.addHeader("X-API-KEY", requestApiKey); + } filter.doFilter(req, res, chain); - - assertEquals(HttpServletResponse.SC_OK, res.getStatus()); - } - - @Test - void shouldPassTheFilterWhenTheFilterIsInactiveDueToAnApiKeyConfiguredToBlank() throws Exception { - ApiKeyRequestFilter filter = new ApiKeyRequestFilter(""); - filter.doFilter(req, res, chain); - assertEquals(HttpServletResponse.SC_OK, res.getStatus()); } From 58c59a9a592c95b6432a62aff6b39b41fe20cc9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Mon, 23 Oct 2023 15:43:07 +0200 Subject: [PATCH 08/64] Improving the doc --- src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java b/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java index b033b280..1c698f0b 100644 --- a/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java +++ b/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java @@ -31,7 +31,10 @@ * ApiKeyRequestFilter is a security filter that can be activated to protect endpoints. *

* It is based on the use of an API Key that the caller must fill in via the X-API-KEY header. - * If this filter is activated and the caller does not enter an API Key, then a 401 will be raised + *

+ * If an API Key is configured, the filter will be activated and requests will have to present a valid API Key, + * if this is not the case, a 401 message is sent. + * If no API Key is configured, then the filter will not be activated and requests will run unchecked. */ @Slf4j public class ApiKeyRequestFilter extends GenericFilterBean { From 130d823d5b76712776b6d6fee7ab0f0e2de2b0cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Tue, 24 Oct 2023 15:38:57 +0200 Subject: [PATCH 09/64] add admin endpoints foundation --- CHANGELOG.md | 30 ++++- .../com/iexec/sms/admin/AdminController.java | 76 ++++++++++++ .../com/iexec/sms/admin/AdminService.java | 32 +++++ .../iexec/sms/config/ApiKeyFilterConfig.java | 41 +++++++ .../iexec/sms/utils/ApiKeyRequestFilter.java | 2 +- src/main/resources/application.yml | 19 +-- .../iexec/sms/admin/AdminControllerTests.java | 110 ++++++++++++++++++ .../iexec/sms/admin/AdminServiceTests.java | 35 ++++++ .../sms/config/ApiKeyFilterConfigTests.java | 41 +++++++ 9 files changed, 375 insertions(+), 11 deletions(-) create mode 100644 src/main/java/com/iexec/sms/admin/AdminController.java create mode 100644 src/main/java/com/iexec/sms/admin/AdminService.java create mode 100644 src/main/java/com/iexec/sms/config/ApiKeyFilterConfig.java create mode 100644 src/test/java/com/iexec/sms/admin/AdminControllerTests.java create mode 100644 src/test/java/com/iexec/sms/admin/AdminServiceTests.java create mode 100644 src/test/java/com/iexec/sms/config/ApiKeyFilterConfigTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 88ea9f8b..735e73b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,14 +3,21 @@ All notable changes to this project will be documented in this file. ## [[NEXT]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/vNEXT) 2023 + +### New Features + - Add a security filter to activate an API Key mechanism on endpoints. (#207) +- Create admin endpoints foundation. (#208) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 ### Bug Fixes + - Fix and harmonize `Dockerfile entrypoint` in all Spring Boot applications. (#194) - Check authorization before working with web2 or web3 secrets. (#200) + ### Quality + - Upgrade to Gradle 8.2.1 with up-to-date plugins. (#193) - Use `JpaRepository` in all repository classes for improved features. (#195) - Remove session display option to prevent information leaks. (#197) @@ -18,7 +25,9 @@ All notable changes to this project will be documented in this file. - Immutable `TeeAppProperties` class with `@Builder` pattern. (#201) - Fix Scone generated sessions permissions. (#202) - Remove `VersionService#isSnapshot`. (#204) + ### Dependency Upgrades + - Upgrade to `eclipse-temurin` 11.0.20. (#191) - Upgrade to Spring Boot 2.7.14. (#192) - Upgrade to Spring Dependency Management Plugin 1.1.3. (#192) @@ -30,35 +39,47 @@ All notable changes to this project will be documented in this file. ## [[8.2.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.2.0) 2023-08-11 ### New Features + - Export metrics on secrets counts. (#181) + ### Quality + - Remove `nexus.intra.iex.ec` repository. (#180) - Parameterize build of TEE applications while PR is not started. This allows faster builds. (#182 #184) - Refactor secrets measures. (#185) - Update `sconify.sh` script and use latest `5.7.2-wal` sconifier. (#186 #187 #188) - Add `/metrics` endpoint. (#183) + ### Dependency Upgrades + - Upgrade to `jenkins-library` 2.6.0. (#182) ## [[8.1.2]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.1.2) 2023-06-27 ### Dependency Upgrades + - Upgrade to `iexec-commons-poco` 3.0.5. (#178) ## [[8.1.1]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.1.1) 2023-06-23 ### Dependency Upgrades + - Upgrade to `iexec-common` 8.2.1. (#176) - Upgrade to `iexec-commons-poco` 3.0.4. (#176) ## [[8.1.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.1.0) 2023-06-07 ### New Features + - Enable Prometheus actuator. (#166) + ### Bug Fixes + - Remove unused dependencies. (#168) - Use DatasetAddress in `IEXEC_DATASET_FILENAME` environment variable. (#172) + ### Dependency Upgrades + - Upgrade to `feign` 11.10. (#167) - Upgrade to `iexec-common` 8.2.0. (#169 #170 #171 #173) - Add new `iexec-commons-poco` 3.0.2 dependency. (#169 #170 #171 #173) @@ -66,22 +87,29 @@ All notable changes to this project will be documented in this file. ## [[8.0.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.0.0) 2023-03-06 ### New Features + * Support SMS in enclave for Scone TEE tasks. * Support Gramine framework for TEE tasks. * Add `GET /up` client method in iexec-sms-library. * Return a same `SmsClient` from the `SmsClientProvider` of iexec-sms-library when calling a same SMS URL. * Add iExec banner at startup. * Show application version on banner. + ### Bug Fixes + * Remove TLS context on server. * Remove `GET /secrets` endpoints. * Remove non-TEE workflow. * Remove enclave entrypoints from Gramine sessions since already present in manifests of applications. * Update Scone transformation parameters to enable health checks in SMS in enclave. + ### Quality + * Refactor secret model. * Improve code quality. + ### Dependency Upgrades + * Upgrade to Spring Boot 2.6.14. * Upgrade to Gradle 7.6. * Upgrade OkHttp to 4.9.0. @@ -124,7 +152,7 @@ All notable changes to this project will be documented in this file. * Add TEE pre-compute stage for iExec Workers (confidential tasks inputs). * Enable confidential task on iExec Workers with production enclave mode. - (pre-compute, compute and post-compute stages). + (pre-compute, compute and post-compute stages). * Expose trusted TEE configuration for iExec Workers. * Add custom options for security policies. * Disable requester post-compute. diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java new file mode 100644 index 00000000..b4a36ebe --- /dev/null +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -0,0 +1,76 @@ +/* + * Copyright 2023 IEXEC BLOCKCHAIN TECH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.iexec.sms.admin; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +@Slf4j +@RestController("/admin") +public class AdminController { + + private final Semaphore lock = new Semaphore(1); + + private final AdminService adminService; + + public AdminController(AdminService adminService) { + this.adminService = adminService; + } + + @PostMapping("/backup") + public ResponseEntity createBackup() { + try { + if (tryToAcquireLock()) { + return ResponseEntity.ok(this.adminService.createDatabaseBackupFile()); + } else { + return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); + } finally { + lock.release(); + } + } + + @PostMapping("/{storageID}/restore-backup") + public ResponseEntity restoreBackup(@PathVariable String storageID, @RequestParam(required = true) String fileName) { + try { + if (tryToAcquireLock()) { + return ResponseEntity.ok(this.adminService.restoreDatabaseFromBackupFile(storageID, fileName)); + } else { + return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); + } finally { + lock.release(); + } + } + + private boolean tryToAcquireLock() throws InterruptedException { + return lock.tryAcquire(100, TimeUnit.MILLISECONDS); + } +} diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java new file mode 100644 index 00000000..4d31aaaa --- /dev/null +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -0,0 +1,32 @@ +/* + * Copyright 2023 IEXEC BLOCKCHAIN TECH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.iexec.sms.admin; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class AdminService { + + public String createDatabaseBackupFile() { + return "createDatabaseBackupFile is not implemented"; + } + + public String restoreDatabaseFromBackupFile(String storageId, String fileName) { + return "restoreDatabaseFromBackupFile is not implemented"; + } +} diff --git a/src/main/java/com/iexec/sms/config/ApiKeyFilterConfig.java b/src/main/java/com/iexec/sms/config/ApiKeyFilterConfig.java new file mode 100644 index 00000000..d6df1e50 --- /dev/null +++ b/src/main/java/com/iexec/sms/config/ApiKeyFilterConfig.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 IEXEC BLOCKCHAIN TECH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.iexec.sms.config; + +import com.iexec.sms.utils.ApiKeyRequestFilter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConditionalOnExpression( + "T(org.apache.commons.lang3.StringUtils).isNotEmpty('${server.admin-api-key:}')" +) +public class ApiKeyFilterConfig { + + @Bean + public FilterRegistrationBean filterRegistrationBean(@Value("${server.admin-api-key}") String apiKey) { + FilterRegistrationBean registrationBean = new FilterRegistrationBean<>(); + ApiKeyRequestFilter apiKeyRequestFilter = new ApiKeyRequestFilter(apiKey); + + registrationBean.setFilter(apiKeyRequestFilter); + registrationBean.addUrlPatterns("/admin/*"); + return registrationBean; + } +} diff --git a/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java b/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java index 1c698f0b..df2b8d29 100644 --- a/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java +++ b/src/main/java/com/iexec/sms/utils/ApiKeyRequestFilter.java @@ -33,7 +33,7 @@ * It is based on the use of an API Key that the caller must fill in via the X-API-KEY header. *

* If an API Key is configured, the filter will be activated and requests will have to present a valid API Key, - * if this is not the case, a 401 message is sent. + * if this is not the case, a 401 message is sent. * If no API Key is configured, then the filter will not be activated and requests will run unchecked. */ @Slf4j diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index b6f063b3..ad913ce2 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,15 +1,16 @@ server: port: ${IEXEC_SMS_PORT:13300} + admin-api-key: ${IEXEC_SMS_ADMIN_API_KEY:} -# Not sure it's a good idea but here is a link for an embedded mongodb -# https://www.baeldung.com/spring-boot-embedded-mongodb -#spring: -# data: -# mongodb: -# database: iexec-sms -# host: ${IEXEC_SMS_MONGO_HOST:localhost} -# port: ${IEXEC_SMS_MONGO_PORT:37017} - #ssl-enabled: true + # Not sure it's a good idea but here is a link for an embedded mongodb + # https://www.baeldung.com/spring-boot-embedded-mongodb + #spring: + # data: + # mongodb: + # database: iexec-sms + # host: ${IEXEC_SMS_MONGO_HOST:localhost} + # port: ${IEXEC_SMS_MONGO_PORT:37017} + #ssl-enabled: true # Embedded H2 inside JVM spring: diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java new file mode 100644 index 00000000..e381425e --- /dev/null +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -0,0 +1,110 @@ +/* + * Copyright 2023 IEXEC BLOCKCHAIN TECH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.iexec.sms.admin; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class AdminControllerTests { + + @Mock + private AdminService adminService; + @InjectMocks + private AdminController adminController; + + @BeforeEach + void init() { + MockitoAnnotations.openMocks(this); + } + + + @Test + void testBackupInAdminController() { + assertEquals(HttpStatus.OK, adminController.createBackup().getStatusCode()); + } + + @Test + void testRestoreInAdminController() { + assertEquals(HttpStatus.OK, adminController.restoreBackup("", "").getStatusCode()); + } + + @Test + void testSemaphoreBackupInAdminController() { + AdminController adminControllerForSemaphore = new AdminController(new AdminService() { + @Override + public String createDatabaseBackupFile() { + try { + System.out.println("Long createDatabaseBackupFile action is running ..."); + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return adminService.createDatabaseBackupFile(); + } + }); + + final List> one = new ArrayList<>(); + ResponseEntity two; + Thread thread = new Thread(() -> { + one.add(adminControllerForSemaphore.createBackup()); + }); + thread.start(); + + two = adminControllerForSemaphore.createBackup(); + + assertEquals(HttpStatus.TOO_MANY_REQUESTS, one.get(0).getStatusCode()); + assertEquals(HttpStatus.OK, two.getStatusCode()); + } + + @Test + void testSemaphoreInRestoreAdminController() { + AdminController adminControllerForSemaphore = new AdminController(new AdminService() { + @Override + public String restoreDatabaseFromBackupFile(String storageId, String fileName) { + try { + System.out.println("Long restoreDatabaseFromBackupFile action is running ..."); + Thread.sleep(1000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return adminService.restoreDatabaseFromBackupFile("", ""); + } + }); + + final List> one = new ArrayList<>(); + ResponseEntity two; + Thread thread = new Thread(() -> { + one.add(adminControllerForSemaphore.restoreBackup("", "")); + }); + thread.start(); + + two = adminControllerForSemaphore.restoreBackup("", ""); + + assertEquals(HttpStatus.TOO_MANY_REQUESTS, one.get(0).getStatusCode()); + assertEquals(HttpStatus.OK, two.getStatusCode()); + } +} diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java new file mode 100644 index 00000000..2929a654 --- /dev/null +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023 IEXEC BLOCKCHAIN TECH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.iexec.sms.admin; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class AdminServiceTests { + + private final AdminService adminService = new AdminService(); + + @Test + void shouldReturnNotImplementedWhenCallingBackup() { + Assertions.assertEquals("createDatabaseBackupFile is not implemented", adminService.createDatabaseBackupFile()); + } + + @Test + void shouldReturnNotImplementedWhenCallingRestore() { + Assertions.assertEquals("restoreDatabaseFromBackupFile is not implemented", adminService.restoreDatabaseFromBackupFile("", "")); + } +} diff --git a/src/test/java/com/iexec/sms/config/ApiKeyFilterConfigTests.java b/src/test/java/com/iexec/sms/config/ApiKeyFilterConfigTests.java new file mode 100644 index 00000000..6ef93836 --- /dev/null +++ b/src/test/java/com/iexec/sms/config/ApiKeyFilterConfigTests.java @@ -0,0 +1,41 @@ +/* + * Copyright 2023 IEXEC BLOCKCHAIN TECH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.iexec.sms.config; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.context.annotation.UserConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +class ApiKeyFilterConfigTests { + + private final ApplicationContextRunner runner = new ApplicationContextRunner() + .withConfiguration(UserConfigurations.of(ApiKeyFilterConfig.class)); + + @Test + void shouldCreateFilterWhenAPIKeyIsFilled() { + runner.withPropertyValues("server.admin-api-key=879") + .run(context -> assertThat(context).hasSingleBean(ApiKeyFilterConfig.class)); + } + + @Test + void shouldNotCreateFilterWhenAPIKeyIsNotFilled() { + runner.withPropertyValues("server.admin-api-key=") + .run(context -> assertThat(context).doesNotHaveBean(ApiKeyFilterConfig.class)); + } +} From 50126e3200177ad596087474270a4a76f7a6bfe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Tue, 24 Oct 2023 16:00:07 +0200 Subject: [PATCH 10/64] Switch to sl4j instead of println and rename variables --- .../com/iexec/sms/admin/AdminService.java | 2 +- src/main/resources/application.yml | 10 ------- .../iexec/sms/admin/AdminControllerTests.java | 30 ++++++++++--------- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 4d31aaaa..c4c3f652 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -25,7 +25,7 @@ public class AdminService { public String createDatabaseBackupFile() { return "createDatabaseBackupFile is not implemented"; } - + public String restoreDatabaseFromBackupFile(String storageId, String fileName) { return "restoreDatabaseFromBackupFile is not implemented"; } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index ad913ce2..7dedea5e 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -2,16 +2,6 @@ server: port: ${IEXEC_SMS_PORT:13300} admin-api-key: ${IEXEC_SMS_ADMIN_API_KEY:} - # Not sure it's a good idea but here is a link for an embedded mongodb - # https://www.baeldung.com/spring-boot-embedded-mongodb - #spring: - # data: - # mongodb: - # database: iexec-sms - # host: ${IEXEC_SMS_MONGO_HOST:localhost} - # port: ${IEXEC_SMS_MONGO_PORT:37017} - #ssl-enabled: true - # Embedded H2 inside JVM spring: profiles: diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index e381425e..f90ac479 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -16,6 +16,7 @@ package com.iexec.sms.admin; +import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InjectMocks; @@ -29,6 +30,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; +@Slf4j class AdminControllerTests { @Mock @@ -58,7 +60,7 @@ void testSemaphoreBackupInAdminController() { @Override public String createDatabaseBackupFile() { try { - System.out.println("Long createDatabaseBackupFile action is running ..."); + log.info("Long createDatabaseBackupFile action is running ..."); Thread.sleep(1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -67,17 +69,17 @@ public String createDatabaseBackupFile() { } }); - final List> one = new ArrayList<>(); - ResponseEntity two; + final List> firstResponse = new ArrayList<>(); + ResponseEntity secondResponse; Thread thread = new Thread(() -> { - one.add(adminControllerForSemaphore.createBackup()); + firstResponse.add(adminControllerForSemaphore.createBackup()); }); thread.start(); - two = adminControllerForSemaphore.createBackup(); + secondResponse = adminControllerForSemaphore.createBackup(); - assertEquals(HttpStatus.TOO_MANY_REQUESTS, one.get(0).getStatusCode()); - assertEquals(HttpStatus.OK, two.getStatusCode()); + assertEquals(HttpStatus.TOO_MANY_REQUESTS, firstResponse.get(0).getStatusCode()); + assertEquals(HttpStatus.OK, secondResponse.getStatusCode()); } @Test @@ -86,7 +88,7 @@ void testSemaphoreInRestoreAdminController() { @Override public String restoreDatabaseFromBackupFile(String storageId, String fileName) { try { - System.out.println("Long restoreDatabaseFromBackupFile action is running ..."); + log.info("Long restoreDatabaseFromBackupFile action is running ..."); Thread.sleep(1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -95,16 +97,16 @@ public String restoreDatabaseFromBackupFile(String storageId, String fileName) { } }); - final List> one = new ArrayList<>(); - ResponseEntity two; + final List> firstResponse = new ArrayList<>(); + ResponseEntity secondResponse; Thread thread = new Thread(() -> { - one.add(adminControllerForSemaphore.restoreBackup("", "")); + firstResponse.add(adminControllerForSemaphore.restoreBackup("", "")); }); thread.start(); - two = adminControllerForSemaphore.restoreBackup("", ""); + secondResponse = adminControllerForSemaphore.restoreBackup("", ""); - assertEquals(HttpStatus.TOO_MANY_REQUESTS, one.get(0).getStatusCode()); - assertEquals(HttpStatus.OK, two.getStatusCode()); + assertEquals(HttpStatus.TOO_MANY_REQUESTS, firstResponse.get(0).getStatusCode()); + assertEquals(HttpStatus.OK, secondResponse.getStatusCode()); } } From 45c04824075553c06517d0d0f7ce1d88108885ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Wed, 25 Oct 2023 10:46:27 +0200 Subject: [PATCH 11/64] Switch to ReentrantLock instead of semaphore --- .../com/iexec/sms/admin/AdminController.java | 15 ++-- .../iexec/sms/admin/AdminControllerTests.java | 70 +++++++++++++------ 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index b4a36ebe..0a0ade99 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -23,14 +23,14 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; @Slf4j @RestController("/admin") public class AdminController { - private final Semaphore lock = new Semaphore(1); + private final ReentrantLock rLock = new ReentrantLock(true); private final AdminService adminService; @@ -50,7 +50,9 @@ public ResponseEntity createBackup() { Thread.currentThread().interrupt(); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); } finally { - lock.release(); + if (rLock.isHeldByCurrentThread()) { + rLock.unlock(); + } } } @@ -66,11 +68,14 @@ public ResponseEntity restoreBackup(@PathVariable String storageID, @Req Thread.currentThread().interrupt(); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); } finally { - lock.release(); + if (rLock.isHeldByCurrentThread()) { + rLock.unlock(); + } } } private boolean tryToAcquireLock() throws InterruptedException { - return lock.tryAcquire(100, TimeUnit.MILLISECONDS); + return rLock.tryLock(100, TimeUnit.MILLISECONDS); } + } diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index f90ac479..de3cb3cf 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -26,6 +26,7 @@ import org.springframework.http.ResponseEntity; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -43,7 +44,6 @@ void init() { MockitoAnnotations.openMocks(this); } - @Test void testBackupInAdminController() { assertEquals(HttpStatus.OK, adminController.createBackup().getStatusCode()); @@ -55,13 +55,13 @@ void testRestoreInAdminController() { } @Test - void testSemaphoreBackupInAdminController() { - AdminController adminControllerForSemaphore = new AdminController(new AdminService() { + void testTooManyRequestOnBackupInAdminController() throws InterruptedException { + AdminController adminControllerWithLongAction = new AdminController(new AdminService() { @Override public String createDatabaseBackupFile() { try { log.info("Long createDatabaseBackupFile action is running ..."); - Thread.sleep(1000); + Thread.sleep(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } @@ -69,27 +69,39 @@ public String createDatabaseBackupFile() { } }); - final List> firstResponse = new ArrayList<>(); - ResponseEntity secondResponse; - Thread thread = new Thread(() -> { - firstResponse.add(adminControllerForSemaphore.createBackup()); + final List> responses = Collections.synchronizedList(new ArrayList<>(3)); + + Thread firstThread = new Thread(() -> { + responses.add(adminControllerWithLongAction.createBackup()); + }); + + Thread secondThread = new Thread(() -> { + responses.add(adminControllerWithLongAction.createBackup()); }); - thread.start(); - secondResponse = adminControllerForSemaphore.createBackup(); + firstThread.start(); + secondThread.start(); + responses.add(adminControllerWithLongAction.createBackup()); + + secondThread.join(); + firstThread.join(); + + + long code200 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.OK).count(); + long code429 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS).count(); - assertEquals(HttpStatus.TOO_MANY_REQUESTS, firstResponse.get(0).getStatusCode()); - assertEquals(HttpStatus.OK, secondResponse.getStatusCode()); + assertEquals(1, code200); + assertEquals(2, code429); } @Test - void testSemaphoreInRestoreAdminController() { - AdminController adminControllerForSemaphore = new AdminController(new AdminService() { + void testTooManyRequestOnRestoreAdminController() throws InterruptedException { + AdminController adminControllerWithLongAction = new AdminController(new AdminService() { @Override public String restoreDatabaseFromBackupFile(String storageId, String fileName) { try { log.info("Long restoreDatabaseFromBackupFile action is running ..."); - Thread.sleep(1000); + Thread.sleep(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } @@ -97,16 +109,28 @@ public String restoreDatabaseFromBackupFile(String storageId, String fileName) { } }); - final List> firstResponse = new ArrayList<>(); - ResponseEntity secondResponse; - Thread thread = new Thread(() -> { - firstResponse.add(adminControllerForSemaphore.restoreBackup("", "")); + final List> responses = Collections.synchronizedList(new ArrayList<>(3)); + + Thread firstThread = new Thread(() -> { + responses.add(adminControllerWithLongAction.restoreBackup("", "")); + }); + + Thread secondThread = new Thread(() -> { + responses.add(adminControllerWithLongAction.restoreBackup("", "")); }); - thread.start(); - secondResponse = adminControllerForSemaphore.restoreBackup("", ""); + firstThread.start(); + secondThread.start(); + responses.add(adminControllerWithLongAction.restoreBackup("", "")); + + secondThread.join(); + firstThread.join(); + + + long code200 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.OK).count(); + long code429 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS).count(); - assertEquals(HttpStatus.TOO_MANY_REQUESTS, firstResponse.get(0).getStatusCode()); - assertEquals(HttpStatus.OK, secondResponse.getStatusCode()); + assertEquals(1, code200); + assertEquals(2, code429); } } From 2fc4b3d090a7bb4c61ed0ceb720d829f46f03b4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Wed, 25 Oct 2023 12:10:29 +0200 Subject: [PATCH 12/64] Add regions in Test and JavaDoc in controller --- .../com/iexec/sms/admin/AdminController.java | 29 +++++++++++++++++++ .../iexec/sms/admin/AdminControllerTests.java | 14 +++++---- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index 0a0ade99..53085b5b 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -38,6 +38,19 @@ public AdminController(AdminService adminService) { this.adminService = adminService; } + /* + * Endpoint to initiate a database backup. + * + * This method allows the client to trigger a database backup operation. + * The backup process will create a snapshot of the current database + * and store it for future recovery purposes. + * + * @return A response entity indicating the status and details of the backup operation. + * + * HTTP 201 (Created) - If the backup request is successful. + * HTTP 429 (Too Many Requests) - If another operation (backup/restore/delete) is already in progress. + * HTTP 500 (Internal Server Error) - If an unexpected error occurs during the restore process. + */ @PostMapping("/backup") public ResponseEntity createBackup() { try { @@ -56,6 +69,22 @@ public ResponseEntity createBackup() { } } + /** + * Endpoint to restore a database backup. + *

+ * This method allows the client to initiate the restoration of a database backup + * from a specified dump file, identified by the 'fileName', locate in a location specified by the 'storageID'. + * + * @param storageID The unique identifier for the storage location of the dump in hexadecimal. + * @param fileName The name of the dump file to be restored. + * @return A response entity indicating the status and details of the restore operation. + *

+ * HTTP 200 (OK) - If the backup has been successfully restored. + * HTTP 400 (Bad Request) - If 'fileName' is missing or 'storageID' does not match an existing directory. + * HTTP 404 (Not Found) - If the backup file specified by 'fileName' does not exist. + * HTTP 429 (Too Many Requests) - If another operation (backup/restore/delete) is already in progress. + * HTTP 500 (Internal Server Error) - If an unexpected error occurs during the restore process. + */ @PostMapping("/{storageID}/restore-backup") public ResponseEntity restoreBackup(@PathVariable String storageID, @RequestParam(required = true) String fileName) { try { diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index de3cb3cf..cae2ad93 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -44,16 +44,12 @@ void init() { MockitoAnnotations.openMocks(this); } + // region backup @Test void testBackupInAdminController() { assertEquals(HttpStatus.OK, adminController.createBackup().getStatusCode()); } - @Test - void testRestoreInAdminController() { - assertEquals(HttpStatus.OK, adminController.restoreBackup("", "").getStatusCode()); - } - @Test void testTooManyRequestOnBackupInAdminController() throws InterruptedException { AdminController adminControllerWithLongAction = new AdminController(new AdminService() { @@ -93,6 +89,13 @@ public String createDatabaseBackupFile() { assertEquals(1, code200); assertEquals(2, code429); } + // endregion + + // region restore-backup + @Test + void testRestoreInAdminController() { + assertEquals(HttpStatus.OK, adminController.restoreBackup("", "").getStatusCode()); + } @Test void testTooManyRequestOnRestoreAdminController() throws InterruptedException { @@ -133,4 +136,5 @@ public String restoreDatabaseFromBackupFile(String storageId, String fileName) { assertEquals(1, code200); assertEquals(2, code429); } + // endregion } From 2390da98ee8953388e9fff52aa87c3f4e27c512a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Wed, 25 Oct 2023 13:21:15 +0200 Subject: [PATCH 13/64] harmonisation of doc --- src/main/java/com/iexec/sms/admin/AdminController.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index 53085b5b..d11d1514 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -38,15 +38,15 @@ public AdminController(AdminService adminService) { this.adminService = adminService; } - /* + /** * Endpoint to initiate a database backup. - * + *

* This method allows the client to trigger a database backup operation. * The backup process will create a snapshot of the current database * and store it for future recovery purposes. * * @return A response entity indicating the status and details of the backup operation. - * + *

* HTTP 201 (Created) - If the backup request is successful. * HTTP 429 (Too Many Requests) - If another operation (backup/restore/delete) is already in progress. * HTTP 500 (Internal Server Error) - If an unexpected error occurs during the restore process. From 69dfb19cc21e653f894cf416a56fe148e40dec67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Wed, 25 Oct 2023 16:32:52 +0200 Subject: [PATCH 14/64] Improved javadoc --- src/main/java/com/iexec/sms/admin/AdminController.java | 9 ++++----- src/test/java/com/iexec/sms/admin/AdminServiceTests.java | 4 ++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index d11d1514..a0fb2b50 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -47,7 +47,7 @@ public AdminController(AdminService adminService) { * * @return A response entity indicating the status and details of the backup operation. *

- * HTTP 201 (Created) - If the backup request is successful. + * HTTP 201 (Created) - If the backup has been successfully created. * HTTP 429 (Too Many Requests) - If another operation (backup/restore/delete) is already in progress. * HTTP 500 (Internal Server Error) - If an unexpected error occurs during the restore process. */ @@ -73,15 +73,14 @@ public ResponseEntity createBackup() { * Endpoint to restore a database backup. *

* This method allows the client to initiate the restoration of a database backup - * from a specified dump file, identified by the 'fileName', locate in a location specified by the 'storageID'. + * from a specified dump file, identified by the {@code fileName}, locate in a location specified by the {@code storageID}. * * @param storageID The unique identifier for the storage location of the dump in hexadecimal. * @param fileName The name of the dump file to be restored. * @return A response entity indicating the status and details of the restore operation. - *

* HTTP 200 (OK) - If the backup has been successfully restored. - * HTTP 400 (Bad Request) - If 'fileName' is missing or 'storageID' does not match an existing directory. - * HTTP 404 (Not Found) - If the backup file specified by 'fileName' does not exist. + * HTTP 400 (Bad Request) - If {@code fileName} is missing or {@code storageID} does not match an existing directory. + * HTTP 404 (Not Found) - If the backup file specified by {@code fileName} does not exist. * HTTP 429 (Too Many Requests) - If another operation (backup/restore/delete) is already in progress. * HTTP 500 (Internal Server Error) - If an unexpected error occurs during the restore process. */ diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 2929a654..4a061ccc 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -23,13 +23,17 @@ class AdminServiceTests { private final AdminService adminService = new AdminService(); + // region backup @Test void shouldReturnNotImplementedWhenCallingBackup() { Assertions.assertEquals("createDatabaseBackupFile is not implemented", adminService.createDatabaseBackupFile()); } + // endregion + // region restore-backup @Test void shouldReturnNotImplementedWhenCallingRestore() { Assertions.assertEquals("restoreDatabaseFromBackupFile is not implemented", adminService.restoreDatabaseFromBackupFile("", "")); } + // endregion } From 459bc5a6001f16d21fc3e4411fd60af1f1594896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Wed, 25 Oct 2023 16:51:47 +0200 Subject: [PATCH 15/64] Fix typo --- src/main/java/com/iexec/sms/admin/AdminController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index a0fb2b50..037b3828 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -49,7 +49,7 @@ public AdminController(AdminService adminService) { *

* HTTP 201 (Created) - If the backup has been successfully created. * HTTP 429 (Too Many Requests) - If another operation (backup/restore/delete) is already in progress. - * HTTP 500 (Internal Server Error) - If an unexpected error occurs during the restore process. + * HTTP 500 (Internal Server Error) - If an unexpected error occurs during the backup process. */ @PostMapping("/backup") public ResponseEntity createBackup() { @@ -73,7 +73,7 @@ public ResponseEntity createBackup() { * Endpoint to restore a database backup. *

* This method allows the client to initiate the restoration of a database backup - * from a specified dump file, identified by the {@code fileName}, locate in a location specified by the {@code storageID}. + * from a specified dump file, identified by the {@code fileName}, located in a location specified by the {@code storageID}. * * @param storageID The unique identifier for the storage location of the dump in hexadecimal. * @param fileName The name of the dump file to be restored. From 1e28cf310229dd4532b0ee533977bb40bbda513c Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Thu, 26 Oct 2023 09:01:44 +0200 Subject: [PATCH 16/64] Update h2 dependency configuration to `implementation` --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 50ec0a6d..5c76c9a7 100644 --- a/build.gradle +++ b/build.gradle @@ -71,7 +71,7 @@ dependencies { implementation 'org.springframework.retry:spring-retry' // H2 implementation 'org.springframework.boot:spring-boot-starter-data-jpa' - runtimeOnly 'com.h2database:h2:2.2.222' + implementation 'com.h2database:h2:2.2.222' // Spring Doc implementation 'org.springdoc:springdoc-openapi-ui:1.6.3' From 56b8e33649ac4a077e1d41411840aa608ea169e1 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Thu, 26 Oct 2023 09:02:39 +0200 Subject: [PATCH 17/64] Add `replicate-backup` skeleton in `AdminService` --- .../java/com/iexec/sms/admin/AdminService.java | 4 ++++ .../com/iexec/sms/admin/AdminServiceTests.java | 14 +++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index c4c3f652..47302471 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -26,6 +26,10 @@ public String createDatabaseBackupFile() { return "createDatabaseBackupFile is not implemented"; } + public String replicateDatabaseBackupFile(String storageID, String fileName) { + return "replicateDatabaseBackupFile is not implemented"; + } + public String restoreDatabaseFromBackupFile(String storageId, String fileName) { return "restoreDatabaseFromBackupFile is not implemented"; } diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 4a061ccc..0d6e904a 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -16,9 +16,10 @@ package com.iexec.sms.admin; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; + class AdminServiceTests { private final AdminService adminService = new AdminService(); @@ -26,14 +27,21 @@ class AdminServiceTests { // region backup @Test void shouldReturnNotImplementedWhenCallingBackup() { - Assertions.assertEquals("createDatabaseBackupFile is not implemented", adminService.createDatabaseBackupFile()); + assertEquals("createDatabaseBackupFile is not implemented", adminService.createDatabaseBackupFile()); + } + // endregion + + // region replicate-backup + @Test + void shouldReturnNotImplementedWhenCallingReplicate() { + assertEquals("replicateDatabaseBackupFile is not implemented", adminService.replicateDatabaseBackupFile("", "")); } // endregion // region restore-backup @Test void shouldReturnNotImplementedWhenCallingRestore() { - Assertions.assertEquals("restoreDatabaseFromBackupFile is not implemented", adminService.restoreDatabaseFromBackupFile("", "")); + assertEquals("restoreDatabaseFromBackupFile is not implemented", adminService.restoreDatabaseFromBackupFile("", "")); } // endregion } From 5351d235c389ec5061c73f8a3a390050f29d6344 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Thu, 26 Oct 2023 10:34:10 +0200 Subject: [PATCH 18/64] Add `replicate-backup` skeleton in `AdminController` --- .../com/iexec/sms/admin/AdminController.java | 91 +++++++--- .../iexec/sms/admin/AdminControllerTests.java | 159 +++++++++++++++--- 2 files changed, 211 insertions(+), 39 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index 037b3828..9ac75dc1 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -16,6 +16,7 @@ package com.iexec.sms.admin; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.PathVariable; @@ -23,6 +24,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.nio.file.FileSystemNotFoundException; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @@ -46,26 +48,65 @@ public AdminController(AdminService adminService) { * and store it for future recovery purposes. * * @return A response entity indicating the status and details of the backup operation. - *

- * HTTP 201 (Created) - If the backup has been successfully created. - * HTTP 429 (Too Many Requests) - If another operation (backup/restore/delete) is already in progress. - * HTTP 500 (Internal Server Error) - If an unexpected error occurs during the backup process. + *

*/ @PostMapping("/backup") public ResponseEntity createBackup() { try { if (tryToAcquireLock()) { - return ResponseEntity.ok(this.adminService.createDatabaseBackupFile()); + return ResponseEntity.ok(adminService.createDatabaseBackupFile()); } else { return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); } finally { - if (rLock.isHeldByCurrentThread()) { - rLock.unlock(); + tryToReleaseLock(); + } + } + + /** + * Endpoint to replicate a database backup. + *

+ * This method allows the replication of the backup toward another storage. + * + * @param storageID The unique identifier for the storage location of the dump in hexadecimal. + * @param fileName The name of the file copied on the persistent storage. + * @return A response entity indicating the status and details of the replication operation. + *

    + *
  • HTTP 200 (OK) - If the backup has been successfully replicated. + *
  • HTTP 400 (Bad Request) - If {@code fileName} is missing or {@code storageID} does not match an existing directory. + *
  • HTTP 404 (Not Found) - If the backup file specified by {@code fileName} does not exist. + *
  • HTTP 429 (Too Many Requests) - If another operation (backup/restore/delete) is already in progress. + *
  • HTTP 500 (Internal Server Error) - If an unexpected error occurs during the replication process. + *
+ */ + @PostMapping("/{storageID}/replicate-backup") + public ResponseEntity replicateBackup(@PathVariable String storageID, @RequestParam String fileName) { + try { + if (StringUtils.isBlank(storageID) || StringUtils.isBlank(fileName)) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); + } + if (!tryToAcquireLock()) { + return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); } + return ResponseEntity.ok(adminService.replicateDatabaseBackupFile(storageID, fileName)); + } catch (FileSystemNotFoundException e) { + return ResponseEntity.notFound().build(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); + } finally { + tryToReleaseLock(); } } @@ -78,27 +119,33 @@ public ResponseEntity createBackup() { * @param storageID The unique identifier for the storage location of the dump in hexadecimal. * @param fileName The name of the dump file to be restored. * @return A response entity indicating the status and details of the restore operation. - * HTTP 200 (OK) - If the backup has been successfully restored. - * HTTP 400 (Bad Request) - If {@code fileName} is missing or {@code storageID} does not match an existing directory. - * HTTP 404 (Not Found) - If the backup file specified by {@code fileName} does not exist. - * HTTP 429 (Too Many Requests) - If another operation (backup/restore/delete) is already in progress. - * HTTP 500 (Internal Server Error) - If an unexpected error occurs during the restore process. + *
    + *
  • HTTP 200 (OK) - If the backup has been successfully restored. + *
  • HTTP 400 (Bad Request) - If {@code fileName} is missing or {@code storageID} does not match an existing directory. + *
  • HTTP 404 (Not Found) - If the backup file specified by {@code fileName} does not exist. + *
  • HTTP 429 (Too Many Requests) - If another operation (backup/restore/delete) is already in progress. + *
  • HTTP 500 (Internal Server Error) - If an unexpected error occurs during the restore process. + *
*/ @PostMapping("/{storageID}/restore-backup") - public ResponseEntity restoreBackup(@PathVariable String storageID, @RequestParam(required = true) String fileName) { + public ResponseEntity restoreBackup(@PathVariable String storageID, @RequestParam String fileName) { try { - if (tryToAcquireLock()) { - return ResponseEntity.ok(this.adminService.restoreDatabaseFromBackupFile(storageID, fileName)); - } else { + if (StringUtils.isBlank(storageID) || StringUtils.isBlank(fileName)) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); + } + if (!tryToAcquireLock()) { return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); } + return ResponseEntity.ok(adminService.restoreDatabaseFromBackupFile(storageID, fileName)); + } catch (FileSystemNotFoundException e) { + return ResponseEntity.notFound().build(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); } finally { - if (rLock.isHeldByCurrentThread()) { - rLock.unlock(); - } + tryToReleaseLock(); } } @@ -106,4 +153,10 @@ private boolean tryToAcquireLock() throws InterruptedException { return rLock.tryLock(100, TimeUnit.MILLISECONDS); } + private void tryToReleaseLock() { + if (rLock.isHeldByCurrentThread()) { + rLock.unlock(); + } + } + } diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index cae2ad93..c836db1d 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -19,21 +19,35 @@ import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; +import org.springframework.test.util.ReflectionTestUtils; +import java.nio.file.FileSystemNotFoundException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; @Slf4j class AdminControllerTests { + private static final String STORAGE_ID = "storageID"; + private static final String FILE_NAME = "backup.sql"; + + @Mock + private ReentrantLock rLock; @Mock private AdminService adminService; @InjectMocks @@ -46,12 +60,25 @@ void init() { // region backup @Test - void testBackupInAdminController() { + void testBackup() { assertEquals(HttpStatus.OK, adminController.createBackup().getStatusCode()); } @Test - void testTooManyRequestOnBackupInAdminController() throws InterruptedException { + void testInternalServerErrorOnBackup() { + when(adminService.createDatabaseBackupFile()).thenThrow(RuntimeException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.createBackup().getStatusCode()); + } + + @Test + void testInterruptedThreadErrorOnBackup() throws InterruptedException { + ReflectionTestUtils.setField(adminController, "rLock", rLock); + when(rLock.tryLock(100, TimeUnit.MILLISECONDS)).thenThrow(InterruptedException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.createBackup().getStatusCode()); + } + + @Test + void testTooManyRequestOnBackup() throws InterruptedException { AdminController adminControllerWithLongAction = new AdminController(new AdminService() { @Override public String createDatabaseBackupFile() { @@ -67,17 +94,79 @@ public String createDatabaseBackupFile() { final List> responses = Collections.synchronizedList(new ArrayList<>(3)); - Thread firstThread = new Thread(() -> { - responses.add(adminControllerWithLongAction.createBackup()); - }); + Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.createBackup())); + Thread secondThread = new Thread(() -> responses.add(adminControllerWithLongAction.createBackup())); + + firstThread.start(); + secondThread.start(); + responses.add(adminControllerWithLongAction.createBackup()); + + secondThread.join(); + firstThread.join(); + + + long code200 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.OK).count(); + long code429 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS).count(); + + assertEquals(1, code200); + assertEquals(2, code429); + } + // endregion + + // region replicate-backup + @Test + void testReplicate() { + assertEquals(HttpStatus.OK, adminController.replicateBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + } + + @ParameterizedTest + @MethodSource("provideBadRequestParameters") + void testBadRequestOnReplicate(String storageID, String fileName) { + assertEquals(HttpStatus.BAD_REQUEST, adminController.replicateBackup(storageID, fileName).getStatusCode()); + } + + @Test + void testInternalServerErrorOnReplicate() { + when(adminService.replicateDatabaseBackupFile(STORAGE_ID, FILE_NAME)).thenThrow(RuntimeException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.replicateBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + } + + @Test + void testInterruptedThreadOnReplicate() throws InterruptedException { + ReflectionTestUtils.setField(adminController, "rLock", rLock); + when(rLock.tryLock(100, TimeUnit.MILLISECONDS)).thenThrow(InterruptedException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.replicateBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + } + + @Test + void testNotFoundOnReplicate() { + when(adminService.replicateDatabaseBackupFile(STORAGE_ID, FILE_NAME)).thenThrow(FileSystemNotFoundException.class); + assertEquals(HttpStatus.NOT_FOUND, adminController.replicateBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + } - Thread secondThread = new Thread(() -> { - responses.add(adminControllerWithLongAction.createBackup()); + @Test + void testTooManyRequestOnReplicate() throws InterruptedException { + AdminController adminControllerWithLongAction = new AdminController(new AdminService() { + @Override + public String replicateDatabaseBackupFile(String storageId, String fileName) { + try { + log.info("Long replicateDatabaseBackupFile action is running ..."); + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return adminService.replicateDatabaseBackupFile(storageId, fileName); + } }); + final List> responses = Collections.synchronizedList(new ArrayList<>(3)); + + Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.replicateBackup(STORAGE_ID, FILE_NAME))); + Thread secondThread = new Thread(() -> responses.add(adminControllerWithLongAction.replicateBackup(STORAGE_ID, FILE_NAME))); + firstThread.start(); secondThread.start(); - responses.add(adminControllerWithLongAction.createBackup()); + responses.add(adminControllerWithLongAction.replicateBackup(STORAGE_ID, FILE_NAME)); secondThread.join(); firstThread.join(); @@ -93,12 +182,37 @@ public String createDatabaseBackupFile() { // region restore-backup @Test - void testRestoreInAdminController() { - assertEquals(HttpStatus.OK, adminController.restoreBackup("", "").getStatusCode()); + void testRestore() { + assertEquals(HttpStatus.OK, adminController.restoreBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + } + + @ParameterizedTest + @MethodSource("provideBadRequestParameters") + void testBadRequestOnRestore(String storageID, String fileName) { + assertEquals(HttpStatus.BAD_REQUEST, adminController.restoreBackup(storageID, fileName).getStatusCode()); + } + + @Test + void testInternalServerErrorOnRestore() { + when(adminService.restoreDatabaseFromBackupFile(STORAGE_ID, FILE_NAME)).thenThrow(RuntimeException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.restoreBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + } + + @Test + void testInterruptedThreadOnRestore() throws InterruptedException { + ReflectionTestUtils.setField(adminController, "rLock", rLock); + when(rLock.tryLock(100, TimeUnit.MILLISECONDS)).thenThrow(InterruptedException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.restoreBackup(STORAGE_ID, FILE_NAME).getStatusCode()); } @Test - void testTooManyRequestOnRestoreAdminController() throws InterruptedException { + void testNotFoundOnRestore() { + when(adminService.restoreDatabaseFromBackupFile(STORAGE_ID, FILE_NAME)).thenThrow(FileSystemNotFoundException.class); + assertEquals(HttpStatus.NOT_FOUND, adminController.restoreBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + } + + @Test + void testTooManyRequestOnRestore() throws InterruptedException { AdminController adminControllerWithLongAction = new AdminController(new AdminService() { @Override public String restoreDatabaseFromBackupFile(String storageId, String fileName) { @@ -108,23 +222,18 @@ public String restoreDatabaseFromBackupFile(String storageId, String fileName) { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - return adminService.restoreDatabaseFromBackupFile("", ""); + return adminService.restoreDatabaseFromBackupFile(storageId, fileName); } }); final List> responses = Collections.synchronizedList(new ArrayList<>(3)); - Thread firstThread = new Thread(() -> { - responses.add(adminControllerWithLongAction.restoreBackup("", "")); - }); - - Thread secondThread = new Thread(() -> { - responses.add(adminControllerWithLongAction.restoreBackup("", "")); - }); + Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.restoreBackup(STORAGE_ID, FILE_NAME))); + Thread secondThread = new Thread(() -> responses.add(adminControllerWithLongAction.restoreBackup(STORAGE_ID, FILE_NAME))); firstThread.start(); secondThread.start(); - responses.add(adminControllerWithLongAction.restoreBackup("", "")); + responses.add(adminControllerWithLongAction.restoreBackup(STORAGE_ID, FILE_NAME)); secondThread.join(); firstThread.join(); @@ -137,4 +246,14 @@ public String restoreDatabaseFromBackupFile(String storageId, String fileName) { assertEquals(2, code429); } // endregion + + private static Stream provideBadRequestParameters() { + return Stream.of( + Arguments.of(null, null), + Arguments.of("", ""), + Arguments.of(" ", " "), + Arguments.of(STORAGE_ID, " "), + Arguments.of(" ", FILE_NAME) + ); + } } From e71665cc48324d52253c7e97d734a76c11a78124 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Thu, 26 Oct 2023 11:13:28 +0200 Subject: [PATCH 19/64] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 735e73b0..b9bb5be1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ All notable changes to this project will be documented in this file. ### New Features - Add a security filter to activate an API Key mechanism on endpoints. (#207) -- Create admin endpoints foundation. (#208) +- Create admin endpoints foundation. (#208 #209) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 From c5018a011e9b1dbcdf074f60ea38dea5f5f488a1 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Thu, 26 Oct 2023 14:45:27 +0200 Subject: [PATCH 20/64] Add H2 database connection informations and storage ID decoding method --- .../com/iexec/sms/admin/AdminController.java | 21 ++++++++++++ .../com/iexec/sms/admin/AdminService.java | 15 +++++++++ .../iexec/sms/admin/AdminControllerTests.java | 32 +++++++++++++++++-- .../iexec/sms/admin/AdminServiceTests.java | 2 +- 4 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index 9ac75dc1..488102c2 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -25,6 +25,8 @@ import org.springframework.web.bind.annotation.RestController; import java.nio.file.FileSystemNotFoundException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @@ -149,6 +151,25 @@ public ResponseEntity restoreBackup(@PathVariable String storageID, @Req } } + /** + * Converts {@code storageID} to an ascii string and checks if it is an existing folder. + * + * @param storageID The hexadecimal representation of an ascii String + * @return The storage path if it exists, throws a {@code FileSystemNotFoundException} otherwise. + */ + String getStoragePathFromID(String storageID) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < storageID.length(); i += 2) { + String str = storageID.substring(i, i + 2); + sb.append((char) Integer.parseInt(str, 16)); + } + final String output = sb.toString(); + if (!Files.isDirectory(Path.of(output))) { + throw new FileSystemNotFoundException("Storage ID " + storageID + " does not match an existing folder"); + } + return output; + } + private boolean tryToAcquireLock() throws InterruptedException { return rLock.tryLock(100, TimeUnit.MILLISECONDS); } diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 47302471..cf3d07be 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -16,12 +16,27 @@ package com.iexec.sms.admin; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @Slf4j @Service public class AdminService { + private final String datasourceUrl; + private final String datasourceUsername; + private final String datasourcePassword; + + public AdminService( + @Value("${spring.datasource.url}") String datasourceUrl, + @Value("${spring.datasource.username}") String datasourceUsername, + @Value("${spring.datasource.password}") String datasourcePassword + ) { + this.datasourceUrl = datasourceUrl; + this.datasourceUsername = datasourceUsername; + this.datasourcePassword = datasourcePassword; + } + public String createDatabaseBackupFile() { return "createDatabaseBackupFile is not implemented"; } diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index c836db1d..ace94365 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -30,6 +31,7 @@ import org.springframework.test.util.ReflectionTestUtils; import java.nio.file.FileSystemNotFoundException; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -38,6 +40,7 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; import static org.mockito.Mockito.when; @Slf4j @@ -79,7 +82,7 @@ void testInterruptedThreadErrorOnBackup() throws InterruptedException { @Test void testTooManyRequestOnBackup() throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService() { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { @Override public String createDatabaseBackupFile() { try { @@ -146,7 +149,7 @@ void testNotFoundOnReplicate() { @Test void testTooManyRequestOnReplicate() throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService() { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { @Override public String replicateDatabaseBackupFile(String storageId, String fileName) { try { @@ -213,7 +216,7 @@ void testNotFoundOnRestore() { @Test void testTooManyRequestOnRestore() throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService() { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { @Override public String restoreDatabaseFromBackupFile(String storageId, String fileName) { try { @@ -247,6 +250,29 @@ public String restoreDatabaseFromBackupFile(String storageId, String fileName) { } // endregion + // region getStoragePathFromID + @Test + void testFileSystemNotFoundExceptionOnGetStoragePathFromID() { + String storageID = convertToHex("/void"); + assertThrowsExactly(FileSystemNotFoundException.class, () -> adminController.getStoragePathFromID(storageID)); + } + + @Test + void testGetStoragePathFromID(@TempDir Path tempDir) { + String storageID = convertToHex(tempDir.toString()); + assertEquals(tempDir.toString(), adminController.getStoragePathFromID(storageID)); + } + + private String convertToHex(String str) { + char[] chars = str.toCharArray(); + StringBuilder hex = new StringBuilder(); + for (char ch : chars) { + hex.append(Integer.toHexString(ch)); + } + return hex.toString(); + } + // endregion + private static Stream provideBadRequestParameters() { return Stream.of( Arguments.of(null, null), diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 0d6e904a..d81da7c5 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -22,7 +22,7 @@ class AdminServiceTests { - private final AdminService adminService = new AdminService(); + private final AdminService adminService = new AdminService("", "", ""); // region backup @Test From 6903d220088b9b1de93280b833e5393b51eb7723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 26 Oct 2023 16:35:32 +0200 Subject: [PATCH 21/64] Add the ability to trigger a backup via a dedicated endpoint --- CHANGELOG.md | 1 + .../com/iexec/sms/admin/AdminController.java | 19 ++++- .../com/iexec/sms/admin/AdminService.java | 75 ++++++++++++++++++- .../iexec/sms/admin/AdminControllerTests.java | 43 +++++------ .../iexec/sms/admin/AdminServiceTests.java | 64 +++++++++++++++- 5 files changed, 174 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9bb5be1..1a2bdb18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. - Add a security filter to activate an API Key mechanism on endpoints. (#207) - Create admin endpoints foundation. (#208 #209) +- Add the ability to trigger a backup via a dedicated endpoint (#) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index 9ac75dc1..465354fb 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -32,6 +32,20 @@ @RestController("/admin") public class AdminController { + /** + * The directory where the database backup file will be stored. + * The value of this constant should be a valid, existing directory path. + */ + private static final String BACKUP_STORAGE_LOCATION = "/work/"; + + /** + * The name of the database backup file. + */ + private static final String BACKUP_FILENAME = "backup.sql"; + + /** + * We want to perform one operation at a time. This ReentrantLock is used to set up the lock mechanism. + */ private final ReentrantLock rLock = new ReentrantLock(true); private final AdminService adminService; @@ -58,7 +72,10 @@ public AdminController(AdminService adminService) { public ResponseEntity createBackup() { try { if (tryToAcquireLock()) { - return ResponseEntity.ok(adminService.createDatabaseBackupFile()); + if (adminService.createDatabaseBackupFile(BACKUP_STORAGE_LOCATION, BACKUP_FILENAME)) { + return ResponseEntity.status(HttpStatus.CREATED).build(); + } + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); } else { return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); } diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 47302471..84913232 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -15,15 +15,86 @@ */ package com.iexec.sms.admin; + import lombok.extern.slf4j.Slf4j; +import org.h2.tools.Script; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import java.io.File; +import java.sql.SQLException; + @Slf4j @Service public class AdminService { - public String createDatabaseBackupFile() { - return "createDatabaseBackupFile is not implemented"; + + private final String datasourceUrl; + private final String datasourceUsername; + private final String datasourcePassword; + + public AdminService(@Value("${spring.datasource.url}") String datasourceUrl, + @Value("${spring.datasource.username}") String datasourceUsername, + @Value("${spring.datasource.password}") String datasourcePassword) { + this.datasourceUrl = datasourceUrl; + this.datasourceUsername = datasourceUsername; + this.datasourcePassword = datasourcePassword; + } + + /** + * Creates a backup of the H2 database and saves it to the specified location. + * + * @param storageLocation The location where the backup file will be saved, must be an existing directory. + * @param backupFileName The name of the backup file. + * @return {@code true} if the backup was successful; {@code false} if any error occurs. + */ + boolean createDatabaseBackupFile(String storageLocation, String backupFileName) { + // Check for missing or empty storageLocation parameter + if (storageLocation == null || storageLocation.isEmpty()) { + log.error("storageLocation must not be empty."); + return false; + } + // Check for missing or empty backupFileName parameter + if (backupFileName == null || backupFileName.isEmpty()) { + log.error("backupFileName must not be empty."); + return false; + } + // Ensure that storageLocation ends with a slash + if (!storageLocation.endsWith("/")) { + storageLocation += "/"; + } + + // Check if storageLocation is an existing directory, we don't want to create it. + File directory = new File(storageLocation); + if (!directory.exists() || !directory.isDirectory()) { + log.error("storageLocation must be an existing directory."); + return false; + } + + return databaseDump(storageLocation + backupFileName); + } + + /** + * @param fullBackupFileName complete fileName (location and filename) + * @return {@code true} if the backup was successful; {@code false} if any error occurs. + */ + boolean databaseDump(String fullBackupFileName) { + + if (fullBackupFileName == null || fullBackupFileName.isEmpty()) { + log.error("fullBackupFileName must not be empty."); + return false; + } + try { + log.info("Starting the backup process {}", fullBackupFileName); + final long start = System.currentTimeMillis(); + Script.process(datasourceUrl, datasourceUsername, datasourcePassword, fullBackupFileName, "DROP", ""); + final long stop = System.currentTimeMillis(); + log.info("Backup took {} ms", stop - start); + } catch (SQLException e) { + log.error("SQL error occurred during backup : " + e.getMessage()); + return false; + } + return true; } public String replicateDatabaseBackupFile(String storageID, String fileName) { diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index c836db1d..089ca549 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -24,6 +24,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -38,6 +39,7 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @Slf4j @@ -60,42 +62,41 @@ void init() { // region backup @Test - void testBackup() { - assertEquals(HttpStatus.OK, adminController.createBackup().getStatusCode()); + void shouldReturnCreatedWhenBackupSuccess() { + Mockito.doReturn(true).when(adminService).createDatabaseBackupFile(any(), any()); + assertEquals(HttpStatus.CREATED, adminController.createBackup().getStatusCode()); } @Test - void testInternalServerErrorOnBackup() { - when(adminService.createDatabaseBackupFile()).thenThrow(RuntimeException.class); + void shouldReturnErrorWhenBackupFail() { + Mockito.doReturn(false).when(adminService).createDatabaseBackupFile(any(), any()); assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.createBackup().getStatusCode()); } @Test - void testInterruptedThreadErrorOnBackup() throws InterruptedException { - ReflectionTestUtils.setField(adminController, "rLock", rLock); - when(rLock.tryLock(100, TimeUnit.MILLISECONDS)).thenThrow(InterruptedException.class); - assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.createBackup().getStatusCode()); - } - - @Test - void testTooManyRequestOnBackup() throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService() { + void shouldReturnTooManyRequestWhenBackupProcessIsAlreadyRunning() throws InterruptedException { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { @Override - public String createDatabaseBackupFile() { + public boolean createDatabaseBackupFile(String storageLocation, String backupFileName) { try { log.info("Long createDatabaseBackupFile action is running ..."); Thread.sleep(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - return adminService.createDatabaseBackupFile(); + return true; } }); final List> responses = Collections.synchronizedList(new ArrayList<>(3)); - Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.createBackup())); - Thread secondThread = new Thread(() -> responses.add(adminControllerWithLongAction.createBackup())); + Thread firstThread = new Thread(() -> { + responses.add(adminControllerWithLongAction.createBackup()); + }); + + Thread secondThread = new Thread(() -> { + responses.add(adminControllerWithLongAction.createBackup()); + }); firstThread.start(); secondThread.start(); @@ -105,10 +106,10 @@ public String createDatabaseBackupFile() { firstThread.join(); - long code200 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.OK).count(); + long code201 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.CREATED).count(); long code429 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS).count(); - assertEquals(1, code200); + assertEquals(1, code201); assertEquals(2, code429); } // endregion @@ -146,7 +147,7 @@ void testNotFoundOnReplicate() { @Test void testTooManyRequestOnReplicate() throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService() { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { @Override public String replicateDatabaseBackupFile(String storageId, String fileName) { try { @@ -213,7 +214,7 @@ void testNotFoundOnRestore() { @Test void testTooManyRequestOnRestore() throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService() { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { @Override public String restoreDatabaseFromBackupFile(String storageId, String fileName) { try { diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 0d6e904a..b5ec6f88 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -16,19 +16,75 @@ package com.iexec.sms.admin; + import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mockito; + +import java.io.File; -import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; class AdminServiceTests { - private final AdminService adminService = new AdminService(); + + private final AdminService adminService = new AdminService("", "", ""); + + @TempDir + File tempStorageLocation; // region backup @Test - void shouldReturnNotImplementedWhenCallingBackup() { - assertEquals("createDatabaseBackupFile is not implemented", adminService.createDatabaseBackupFile()); + void shouldReturnTrueWhenAllParametersAreValid() { + AdminService adminServiceSpy = Mockito.spy(adminService); + + Mockito.doReturn(true).when(adminServiceSpy).databaseDump(any()); + assertTrue(adminServiceSpy.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql")); + } + + @Test + void shouldReturnFalseWhenAllParametersAreValidButBackupFailed() { + AdminService adminServiceSpy = Mockito.spy(adminService); + + Mockito.doReturn(false).when(adminServiceSpy).databaseDump(any()); + assertFalse(adminServiceSpy.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql")); + } + + @ParameterizedTest + @NullSource + @ValueSource(strings = {""}) + void shouldReturnFalseWhenEmptyOrNUllStorageLocation(String location) { + assertFalse(adminService.createDatabaseBackupFile(location, "backup.sql")); + } + + @ParameterizedTest + @NullSource + @ValueSource(strings = {""}) + void shouldReturnFalseWhenEmptyOrNUllBackupFileName(String fileName) { + assertFalse(adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), fileName)); + } + + @Test + void shouldReturnFalseWhenStorageLocationDoesNotExist() { + assertFalse(adminService.createDatabaseBackupFile("/nonexistent/directory/", "backup.sql")); } + + @ParameterizedTest + @NullSource + @ValueSource(strings = {""}) + void shouldReturnFalseWhenEmptyOrNUllFullBackupFileName(String fullBackupFileName) { + assertFalse(adminService.databaseDump(fullBackupFileName)); + } + + @Test + void shouldReturnFalseWhenBackupFileNameDoesNotExist() { + assertFalse(adminService.databaseDump("/nonexistent/directory/backup.sql")); + } + // endregion // region replicate-backup From d3190d125ddcd413a48dd234b09ed90b60d05db8 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Thu, 26 Oct 2023 16:42:38 +0200 Subject: [PATCH 22/64] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9bb5be1..ac11f6ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. - Add a security filter to activate an API Key mechanism on endpoints. (#207) - Create admin endpoints foundation. (#208 #209) +- Add H2 database connection informations and storage ID decoding method. (#210) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 From 012f4a9376064e95a5e5308c0b8684a77a70495a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 26 Oct 2023 16:50:19 +0200 Subject: [PATCH 23/64] Update changelog with PR ID --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a2bdb18..814bafce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ All notable changes to this project will be documented in this file. - Add a security filter to activate an API Key mechanism on endpoints. (#207) - Create admin endpoints foundation. (#208 #209) -- Add the ability to trigger a backup via a dedicated endpoint (#) +- Add the ability to trigger a backup via a dedicated endpoint. (#211) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 From d593444dc06fd457e0292b5e20059b9398035158 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Thu, 26 Oct 2023 16:50:47 +0200 Subject: [PATCH 24/64] Fix spacing to avoid conflict --- src/main/java/com/iexec/sms/admin/AdminService.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index cf3d07be..5691135e 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -27,11 +27,9 @@ public class AdminService { private final String datasourceUsername; private final String datasourcePassword; - public AdminService( - @Value("${spring.datasource.url}") String datasourceUrl, - @Value("${spring.datasource.username}") String datasourceUsername, - @Value("${spring.datasource.password}") String datasourcePassword - ) { + public AdminService(@Value("${spring.datasource.url}") String datasourceUrl, + @Value("${spring.datasource.username}") String datasourceUsername, + @Value("${spring.datasource.password}") String datasourcePassword) { this.datasourceUrl = datasourceUrl; this.datasourceUsername = datasourceUsername; this.datasourcePassword = datasourcePassword; From 8d831aa89bcb142f6b77c8e6a2d317143f32af7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 26 Oct 2023 17:05:41 +0200 Subject: [PATCH 25/64] Fix typo and refactor backup method in controller --- .../java/com/iexec/sms/admin/AdminController.java | 11 +++++------ .../java/com/iexec/sms/admin/AdminServiceTests.java | 6 +++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index 465354fb..54e0ad2f 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -71,14 +71,13 @@ public AdminController(AdminService adminService) { @PostMapping("/backup") public ResponseEntity createBackup() { try { - if (tryToAcquireLock()) { - if (adminService.createDatabaseBackupFile(BACKUP_STORAGE_LOCATION, BACKUP_FILENAME)) { - return ResponseEntity.status(HttpStatus.CREATED).build(); - } - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); - } else { + if (!tryToAcquireLock()) { return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); } + if (adminService.createDatabaseBackupFile(BACKUP_STORAGE_LOCATION, BACKUP_FILENAME)) { + return ResponseEntity.status(HttpStatus.CREATED).build(); + } + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index b5ec6f88..319469fa 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -57,14 +57,14 @@ void shouldReturnFalseWhenAllParametersAreValidButBackupFailed() { @ParameterizedTest @NullSource @ValueSource(strings = {""}) - void shouldReturnFalseWhenEmptyOrNUllStorageLocation(String location) { + void shouldReturnFalseWhenEmptyOrNullStorageLocation(String location) { assertFalse(adminService.createDatabaseBackupFile(location, "backup.sql")); } @ParameterizedTest @NullSource @ValueSource(strings = {""}) - void shouldReturnFalseWhenEmptyOrNUllBackupFileName(String fileName) { + void shouldReturnFalseWhenEmptyOrNullBackupFileName(String fileName) { assertFalse(adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), fileName)); } @@ -76,7 +76,7 @@ void shouldReturnFalseWhenStorageLocationDoesNotExist() { @ParameterizedTest @NullSource @ValueSource(strings = {""}) - void shouldReturnFalseWhenEmptyOrNUllFullBackupFileName(String fullBackupFileName) { + void shouldReturnFalseWhenEmptyOrNullFullBackupFileName(String fullBackupFileName) { assertFalse(adminService.databaseDump(fullBackupFileName)); } From 0b0267c84c7cd6baf40baf081357107826af602a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 26 Oct 2023 17:46:17 +0200 Subject: [PATCH 26/64] Display of backup size and start date after a successful operation --- src/main/java/com/iexec/sms/admin/AdminService.java | 12 +++++++++--- .../java/com/iexec/sms/admin/AdminServiceTests.java | 3 +-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 84913232..9b8d20ca 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -23,12 +23,16 @@ import java.io.File; import java.sql.SQLException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; @Slf4j @Service public class AdminService { - + // Used to print formatted date in log + private final DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); private final String datasourceUrl; private final String datasourceUsername; private final String datasourcePassword; @@ -65,7 +69,7 @@ boolean createDatabaseBackupFile(String storageLocation, String backupFileName) } // Check if storageLocation is an existing directory, we don't want to create it. - File directory = new File(storageLocation); + final File directory = new File(storageLocation); if (!directory.exists() || !directory.isDirectory()) { log.error("storageLocation must be an existing directory."); return false; @@ -84,12 +88,14 @@ boolean databaseDump(String fullBackupFileName) { log.error("fullBackupFileName must not be empty."); return false; } + try { log.info("Starting the backup process {}", fullBackupFileName); final long start = System.currentTimeMillis(); Script.process(datasourceUrl, datasourceUsername, datasourcePassword, fullBackupFileName, "DROP", ""); final long stop = System.currentTimeMillis(); - log.info("Backup took {} ms", stop - start); + final long size = (new File(fullBackupFileName)).length(); + log.info("New backup created [timestamp {}, duration {} ms, size {}]", dateFormat.format(new Date(start)), stop - start, size); } catch (SQLException e) { log.error("SQL error occurred during backup : " + e.getMessage()); return false; diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 319469fa..1c3b0773 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -30,8 +30,7 @@ import static org.mockito.ArgumentMatchers.any; class AdminServiceTests { - - + private final AdminService adminService = new AdminService("", "", ""); @TempDir From 87155d38384ca1ba7d6ff10c53ceedcccce385db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Fri, 27 Oct 2023 11:16:20 +0200 Subject: [PATCH 27/64] taking account of the code review --- .../java/com/iexec/sms/admin/AdminController.java | 2 +- .../java/com/iexec/sms/admin/AdminService.java | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index 9bd5fac1..eb6d0b83 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -46,7 +46,7 @@ public class AdminController { private static final String BACKUP_FILENAME = "backup.sql"; /** - * We want to perform one operation at a time. This ReentrantLock is used to set up the lock mechanism. + * We want to perform one operation at a time. This ReentrantLock is used to set up the lock mechanism. */ private final ReentrantLock rLock = new ReentrantLock(true); diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 40a7d33c..7ee16ddd 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -16,9 +16,9 @@ package com.iexec.sms.admin; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.h2.tools.Script; import org.springframework.beans.factory.annotation.Value; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.File; @@ -54,12 +54,12 @@ public AdminService(@Value("${spring.datasource.url}") String datasourceUrl, */ boolean createDatabaseBackupFile(String storageLocation, String backupFileName) { // Check for missing or empty storageLocation parameter - if (storageLocation == null || storageLocation.isEmpty()) { + if (StringUtils.isBlank(storageLocation)) { log.error("storageLocation must not be empty."); return false; } // Check for missing or empty backupFileName parameter - if (backupFileName == null || backupFileName.isEmpty()) { + if (StringUtils.isBlank(backupFileName)) { log.error("backupFileName must not be empty."); return false; } @@ -71,7 +71,7 @@ boolean createDatabaseBackupFile(String storageLocation, String backupFileName) // Check if storageLocation is an existing directory, we don't want to create it. final File directory = new File(storageLocation); if (!directory.exists() || !directory.isDirectory()) { - log.error("storageLocation must be an existing directory."); + log.error("storageLocation must be an existing directory [storageLocation:{}]", storageLocation); return false; } @@ -90,14 +90,14 @@ boolean databaseDump(String fullBackupFileName) { } try { - log.info("Starting the backup process {}", fullBackupFileName); + log.info("Starting the backup process [fullBackupFileName:{}]", fullBackupFileName); final long start = System.currentTimeMillis(); Script.process(datasourceUrl, datasourceUsername, datasourcePassword, fullBackupFileName, "DROP", ""); final long stop = System.currentTimeMillis(); final long size = (new File(fullBackupFileName)).length(); - log.info("New backup created [timestamp {}, duration {} ms, size {}]", dateFormat.format(new Date(start)), stop - start, size); + log.info("New backup created [timestamp:{}, duration:{} ms, size:{}, fullBackupFileName:{}]", dateFormat.format(new Date(start)), stop - start, size, fullBackupFileName); } catch (SQLException e) { - log.error("SQL error occurred during backup : " + e.getMessage()); + log.error("SQL error occurred during backup", e); return false; } return true; From 15d69e34c977e1a6b98057914b1b813e3f677521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Fri, 27 Oct 2023 11:24:59 +0200 Subject: [PATCH 28/64] code simplification --- src/main/java/com/iexec/sms/admin/AdminService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 7ee16ddd..c06a1860 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -70,7 +70,7 @@ boolean createDatabaseBackupFile(String storageLocation, String backupFileName) // Check if storageLocation is an existing directory, we don't want to create it. final File directory = new File(storageLocation); - if (!directory.exists() || !directory.isDirectory()) { + if (!directory.isDirectory()) { log.error("storageLocation must be an existing directory [storageLocation:{}]", storageLocation); return false; } From 6527f57147a73a2e6415d3ef159dd11b32a24c10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Fri, 27 Oct 2023 13:54:08 +0200 Subject: [PATCH 29/64] taking account of the code review --- src/main/java/com/iexec/sms/admin/AdminService.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index c06a1860..acfb49fd 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -64,8 +64,8 @@ boolean createDatabaseBackupFile(String storageLocation, String backupFileName) return false; } // Ensure that storageLocation ends with a slash - if (!storageLocation.endsWith("/")) { - storageLocation += "/"; + if (!storageLocation.endsWith(File.separator)) { + storageLocation += File.separator; } // Check if storageLocation is an existing directory, we don't want to create it. @@ -84,7 +84,7 @@ boolean createDatabaseBackupFile(String storageLocation, String backupFileName) */ boolean databaseDump(String fullBackupFileName) { - if (fullBackupFileName == null || fullBackupFileName.isEmpty()) { + if (StringUtils.isBlank(fullBackupFileName)) { log.error("fullBackupFileName must not be empty."); return false; } @@ -94,7 +94,7 @@ boolean databaseDump(String fullBackupFileName) { final long start = System.currentTimeMillis(); Script.process(datasourceUrl, datasourceUsername, datasourcePassword, fullBackupFileName, "DROP", ""); final long stop = System.currentTimeMillis(); - final long size = (new File(fullBackupFileName)).length(); + final long size = new File(fullBackupFileName).length(); log.info("New backup created [timestamp:{}, duration:{} ms, size:{}, fullBackupFileName:{}]", dateFormat.format(new Date(start)), stop - start, size, fullBackupFileName); } catch (SQLException e) { log.error("SQL error occurred during backup", e); From 1fcf237b273f14044beefd8831463f30a5b492be Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 27 Oct 2023 16:06:34 +0200 Subject: [PATCH 30/64] Add the ability to trigger a database restore via a dedicated endpoint --- CHANGELOG.md | 1 + .../com/iexec/sms/admin/AdminController.java | 19 ++--- .../com/iexec/sms/admin/AdminService.java | 24 ++++++- .../iexec/sms/admin/AdminControllerTests.java | 72 ++++++++++--------- .../iexec/sms/admin/AdminServiceTests.java | 37 +++++++++- 5 files changed, 108 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bae2ef88..74b3d521 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file. - Create admin endpoints foundation. (#208 #209) - Add H2 database connection informations and storage ID decoding method. (#210) - Add the ability to trigger a backup via a dedicated endpoint. (#211) +- Add the ability to trigger a database restore via a dedicated endpoint. (#212) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index eb6d0b83..2b1c2d87 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -19,10 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.nio.file.FileSystemNotFoundException; import java.nio.file.Files; @@ -31,7 +28,8 @@ import java.util.concurrent.locks.ReentrantLock; @Slf4j -@RestController("/admin") +@RestController +@RequestMapping("/admin") public class AdminController { /** @@ -115,7 +113,8 @@ public ResponseEntity replicateBackup(@PathVariable String storageID, @R if (!tryToAcquireLock()) { return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); } - return ResponseEntity.ok(adminService.replicateDatabaseBackupFile(storageID, fileName)); + final String storagePath = getStoragePathFromID(storageID); + return ResponseEntity.ok(adminService.replicateDatabaseBackupFile(storagePath, fileName)); } catch (FileSystemNotFoundException e) { return ResponseEntity.notFound().build(); } catch (InterruptedException e) { @@ -146,7 +145,7 @@ public ResponseEntity replicateBackup(@PathVariable String storageID, @R * */ @PostMapping("/{storageID}/restore-backup") - public ResponseEntity restoreBackup(@PathVariable String storageID, @RequestParam String fileName) { + ResponseEntity restoreBackup(@PathVariable String storageID, @RequestParam String fileName) { try { if (StringUtils.isBlank(storageID) || StringUtils.isBlank(fileName)) { return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); @@ -154,7 +153,11 @@ public ResponseEntity restoreBackup(@PathVariable String storageID, @Req if (!tryToAcquireLock()) { return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); } - return ResponseEntity.ok(adminService.restoreDatabaseFromBackupFile(storageID, fileName)); + final String storagePath = getStoragePathFromID(storageID); + if (adminService.restoreDatabaseFromBackupFile(storagePath, fileName)) { + return ResponseEntity.ok().build(); + } + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); } catch (FileSystemNotFoundException e) { return ResponseEntity.notFound().build(); } catch (InterruptedException e) { diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index acfb49fd..9d7e3445 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -17,11 +17,14 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.h2.message.DbException; +import org.h2.tools.RunScript; import org.h2.tools.Script; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.File; +import java.nio.charset.Charset; import java.sql.SQLException; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -103,11 +106,26 @@ boolean databaseDump(String fullBackupFileName) { return true; } - public String replicateDatabaseBackupFile(String storageID, String fileName) { + public String replicateDatabaseBackupFile(String storagePath, String backupFileName) { return "replicateDatabaseBackupFile is not implemented"; } - public String restoreDatabaseFromBackupFile(String storageId, String fileName) { - return "restoreDatabaseFromBackupFile is not implemented"; + public boolean restoreDatabaseFromBackupFile(String storagePath, String backupFileName) { + try { + final String fullBackupFileName = storagePath + File.separator + backupFileName; + final long start = System.currentTimeMillis(); + RunScript.execute(datasourceUrl, datasourceUsername, datasourcePassword, + fullBackupFileName, Charset.defaultCharset(), true); + final long stop = System.currentTimeMillis(); + final long size = new File(fullBackupFileName).length(); + log.warn("Backup has been restored [timestamp:{}, duration:{} ms, size:{}, fullBackupFileName:{}]", + dateFormat.format(new Date(start)), stop - start, size, fullBackupFileName); + return true; + } catch (DbException e) { + log.error("RunScript error occurred during restore", e); + } catch (SQLException e) { + log.error("SQL error occurred during restore", e); + } + return false; } } diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index f74eabac..20376239 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -48,7 +48,7 @@ @Slf4j class AdminControllerTests { - private static final String STORAGE_ID = "storageID"; + private static final String STORAGE_PATH = "/storage"; private static final String FILE_NAME = "backup.sql"; @Mock @@ -114,8 +114,9 @@ public boolean createDatabaseBackupFile(String storageLocation, String backupFil // region replicate-backup @Test - void testReplicate() { - assertEquals(HttpStatus.OK, adminController.replicateBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + void testReplicate(@TempDir Path tempDir) { + final String storageID = convertToHex(tempDir.toString()); + assertEquals(HttpStatus.OK, adminController.replicateBackup(storageID, FILE_NAME).getStatusCode()); } @ParameterizedTest @@ -126,46 +127,48 @@ void testBadRequestOnReplicate(String storageID, String fileName) { @Test void testInternalServerErrorOnReplicate() { - when(adminService.replicateDatabaseBackupFile(STORAGE_ID, FILE_NAME)).thenThrow(RuntimeException.class); - assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.replicateBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + when(adminService.replicateDatabaseBackupFile(STORAGE_PATH, FILE_NAME)).thenThrow(RuntimeException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.replicateBackup(STORAGE_PATH, FILE_NAME).getStatusCode()); } @Test void testInterruptedThreadOnReplicate() throws InterruptedException { ReflectionTestUtils.setField(adminController, "rLock", rLock); when(rLock.tryLock(100, TimeUnit.MILLISECONDS)).thenThrow(InterruptedException.class); - assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.replicateBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.replicateBackup(STORAGE_PATH, FILE_NAME).getStatusCode()); } @Test void testNotFoundOnReplicate() { - when(adminService.replicateDatabaseBackupFile(STORAGE_ID, FILE_NAME)).thenThrow(FileSystemNotFoundException.class); - assertEquals(HttpStatus.NOT_FOUND, adminController.replicateBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + final String storageID = convertToHex(STORAGE_PATH); + when(adminService.replicateDatabaseBackupFile(STORAGE_PATH, FILE_NAME)).thenThrow(FileSystemNotFoundException.class); + assertEquals(HttpStatus.NOT_FOUND, adminController.replicateBackup(storageID, FILE_NAME).getStatusCode()); } @Test - void testTooManyRequestOnReplicate() throws InterruptedException { + void testTooManyRequestOnReplicate(@TempDir Path tempDir) throws InterruptedException { AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { @Override - public String replicateDatabaseBackupFile(String storageId, String fileName) { + public String replicateDatabaseBackupFile(String storagePath, String backupFileName) { try { log.info("Long replicateDatabaseBackupFile action is running ..."); Thread.sleep(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - return adminService.replicateDatabaseBackupFile(storageId, fileName); + return adminService.replicateDatabaseBackupFile(storagePath, backupFileName); } }); final List> responses = Collections.synchronizedList(new ArrayList<>(3)); + final String storageID = convertToHex(tempDir.toString()); - Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.replicateBackup(STORAGE_ID, FILE_NAME))); - Thread secondThread = new Thread(() -> responses.add(adminControllerWithLongAction.replicateBackup(STORAGE_ID, FILE_NAME))); + Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.replicateBackup(storageID, FILE_NAME))); + Thread secondThread = new Thread(() -> responses.add(adminControllerWithLongAction.replicateBackup(storageID, FILE_NAME))); firstThread.start(); secondThread.start(); - responses.add(adminControllerWithLongAction.replicateBackup(STORAGE_ID, FILE_NAME)); + responses.add(adminControllerWithLongAction.replicateBackup(storageID, FILE_NAME)); secondThread.join(); firstThread.join(); @@ -181,8 +184,10 @@ public String replicateDatabaseBackupFile(String storageId, String fileName) { // region restore-backup @Test - void testRestore() { - assertEquals(HttpStatus.OK, adminController.restoreBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + void testRestore(@TempDir Path tempDir) { + final String storageID = convertToHex(tempDir.toString()); + when(adminService.restoreDatabaseFromBackupFile(tempDir.toString(), FILE_NAME)).thenReturn(true); + assertEquals(HttpStatus.OK, adminController.restoreBackup(storageID, FILE_NAME).getStatusCode()); } @ParameterizedTest @@ -192,47 +197,50 @@ void testBadRequestOnRestore(String storageID, String fileName) { } @Test - void testInternalServerErrorOnRestore() { - when(adminService.restoreDatabaseFromBackupFile(STORAGE_ID, FILE_NAME)).thenThrow(RuntimeException.class); - assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.restoreBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + void testInternalServerErrorOnRestore(@TempDir Path tempDir) { + final String storageID = convertToHex(tempDir.toString()); + when(adminService.restoreDatabaseFromBackupFile(tempDir.toString(), FILE_NAME)).thenThrow(RuntimeException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.restoreBackup(storageID, FILE_NAME).getStatusCode()); } @Test - void testInterruptedThreadOnRestore() throws InterruptedException { + void testInterrupterThreadOnRestore() throws InterruptedException { ReflectionTestUtils.setField(adminController, "rLock", rLock); when(rLock.tryLock(100, TimeUnit.MILLISECONDS)).thenThrow(InterruptedException.class); - assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.restoreBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.restoreBackup(STORAGE_PATH, FILE_NAME).getStatusCode()); } @Test void testNotFoundOnRestore() { - when(adminService.restoreDatabaseFromBackupFile(STORAGE_ID, FILE_NAME)).thenThrow(FileSystemNotFoundException.class); - assertEquals(HttpStatus.NOT_FOUND, adminController.restoreBackup(STORAGE_ID, FILE_NAME).getStatusCode()); + final String storageID = convertToHex(STORAGE_PATH); + when(adminService.restoreDatabaseFromBackupFile(STORAGE_PATH, FILE_NAME)).thenThrow(FileSystemNotFoundException.class); + assertEquals(HttpStatus.NOT_FOUND, adminController.restoreBackup(storageID, FILE_NAME).getStatusCode()); } @Test - void testTooManyRequestOnRestore() throws InterruptedException { + void testTooManyRequestOnRestore(@TempDir Path tempDir) throws InterruptedException { AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { @Override - public String restoreDatabaseFromBackupFile(String storageId, String fileName) { + public boolean restoreDatabaseFromBackupFile(String storageId, String fileName) { try { log.info("Long restoreDatabaseFromBackupFile action is running ..."); Thread.sleep(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - return adminService.restoreDatabaseFromBackupFile(storageId, fileName); + return true; } }); - final List> responses = Collections.synchronizedList(new ArrayList<>(3)); + final List> responses = Collections.synchronizedList(new ArrayList<>(3)); + final String storageID = convertToHex(tempDir.toString()); - Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.restoreBackup(STORAGE_ID, FILE_NAME))); - Thread secondThread = new Thread(() -> responses.add(adminControllerWithLongAction.restoreBackup(STORAGE_ID, FILE_NAME))); + Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.restoreBackup(storageID, FILE_NAME))); + Thread secondThread = new Thread(() -> responses.add(adminControllerWithLongAction.restoreBackup(storageID, FILE_NAME))); firstThread.start(); secondThread.start(); - responses.add(adminControllerWithLongAction.restoreBackup(STORAGE_ID, FILE_NAME)); + responses.add(adminControllerWithLongAction.restoreBackup(storageID, FILE_NAME)); secondThread.join(); firstThread.join(); @@ -259,7 +267,7 @@ void testGetStoragePathFromID(@TempDir Path tempDir) { assertEquals(tempDir.toString(), adminController.getStoragePathFromID(storageID)); } - private String convertToHex(String str) { + private static String convertToHex(String str) { char[] chars = str.toCharArray(); StringBuilder hex = new StringBuilder(); for (char ch : chars) { @@ -274,7 +282,7 @@ private static Stream provideBadRequestParameters() { Arguments.of(null, null), Arguments.of("", ""), Arguments.of(" ", " "), - Arguments.of(STORAGE_ID, " "), + Arguments.of(STORAGE_PATH, " "), Arguments.of(" ", FILE_NAME) ); } diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 50651ecc..47acd648 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -17,20 +17,37 @@ package com.iexec.sms.admin; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.NullSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.boot.test.system.CapturedOutput; +import org.springframework.boot.test.system.OutputCaptureExtension; +import org.springframework.context.annotation.Bean; +import javax.sql.DataSource; import java.io.File; +import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; +@ExtendWith(OutputCaptureExtension.class) class AdminServiceTests { - private final AdminService adminService = new AdminService("", "", ""); + @Bean + public DataSource dataSource() { + return DataSourceBuilder.create() + .url("jdbc:h2:mem:test") + .username("sa") + .password("") + .build(); + } + + private final AdminService adminService = new AdminService("jdbc:h2:mem:test", "sa", ""); @TempDir File tempStorageLocation; @@ -95,7 +112,23 @@ void shouldReturnNotImplementedWhenCallingReplicate() { // region restore-backup @Test void shouldReturnNotImplementedWhenCallingRestore() { - assertEquals("restoreDatabaseFromBackupFile is not implemented", adminService.restoreDatabaseFromBackupFile("", "")); + final String backupFile = Path.of(tempStorageLocation.getPath(), "backup.sql").toString(); + adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql"); + assertTrue(new File(backupFile).exists()); + adminService.restoreDatabaseFromBackupFile(tempStorageLocation.getPath(), "backup.sql"); + } + + @Test + void withDbException(CapturedOutput output) { + adminService.restoreDatabaseFromBackupFile(tempStorageLocation.getPath(), "backup.sql"); + assertTrue(output.getOut().contains("RunScript error occurred during restore")); + } + + @Test + void withSQLException(CapturedOutput output) { + AdminService corruptAdminService = new AdminService("url", "username", "password"); + corruptAdminService.restoreDatabaseFromBackupFile(tempStorageLocation.getPath(), "backup.sql"); + assertTrue(output.getOut().contains("SQL error occurred during restore")); } // endregion } From eb17fb038a677a37420ffd0323862a873f4795b8 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 27 Oct 2023 18:17:01 +0200 Subject: [PATCH 31/64] Enforce a base storage folder for restoration, add several fixes --- .../java/com/iexec/sms/admin/AdminService.java | 18 ++++++++++++++++-- .../iexec/sms/admin/AdminControllerTests.java | 6 +++--- .../com/iexec/sms/admin/AdminServiceTests.java | 16 +++++++++++++--- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 9d7e3445..79323e22 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -25,6 +25,8 @@ import java.io.File; import java.nio.charset.Charset; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.Path; import java.sql.SQLException; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -39,13 +41,16 @@ public class AdminService { private final String datasourceUrl; private final String datasourceUsername; private final String datasourcePassword; + private final String storageFolder; public AdminService(@Value("${spring.datasource.url}") String datasourceUrl, @Value("${spring.datasource.username}") String datasourceUsername, - @Value("${spring.datasource.password}") String datasourcePassword) { + @Value("${spring.datasource.password}") String datasourcePassword, + @Value("${spring.datasource.storage-folder}") String storageFolder) { this.datasourceUrl = datasourceUrl; this.datasourceUsername = datasourceUsername; this.datasourcePassword = datasourcePassword; + this.storageFolder = storageFolder; } /** @@ -110,9 +115,12 @@ public String replicateDatabaseBackupFile(String storagePath, String backupFileN return "replicateDatabaseBackupFile is not implemented"; } - public boolean restoreDatabaseFromBackupFile(String storagePath, String backupFileName) { + boolean restoreDatabaseFromBackupFile(String storagePath, String backupFileName) { try { final String fullBackupFileName = storagePath + File.separator + backupFileName; + if (!isPathInBaseDirectory(fullBackupFileName)) { + throw new FileSystemNotFoundException("Backup file " + fullBackupFileName + " not found"); + } final long start = System.currentTimeMillis(); RunScript.execute(datasourceUrl, datasourceUsername, datasourcePassword, fullBackupFileName, Charset.defaultCharset(), true); @@ -128,4 +136,10 @@ public boolean restoreDatabaseFromBackupFile(String storagePath, String backupFi } return false; } + + boolean isPathInBaseDirectory(String pathToCheck) { + Path base = Path.of(storageFolder).toAbsolutePath().normalize(); + Path toCheck = Path.of(pathToCheck).toAbsolutePath().normalize(); + return toCheck.startsWith(base); + } } diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index 20376239..97b053b7 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -78,7 +78,7 @@ void shouldReturnErrorWhenBackupFail() { @Test void shouldReturnTooManyRequestWhenBackupProcessIsAlreadyRunning() throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { @Override public boolean createDatabaseBackupFile(String storageLocation, String backupFileName) { try { @@ -147,7 +147,7 @@ void testNotFoundOnReplicate() { @Test void testTooManyRequestOnReplicate(@TempDir Path tempDir) throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { @Override public String replicateDatabaseBackupFile(String storagePath, String backupFileName) { try { @@ -219,7 +219,7 @@ void testNotFoundOnRestore() { @Test void testTooManyRequestOnRestore(@TempDir Path tempDir) throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { @Override public boolean restoreDatabaseFromBackupFile(String storageId, String fileName) { try { diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 47acd648..5e6b9fb4 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -47,7 +47,7 @@ public DataSource dataSource() { .build(); } - private final AdminService adminService = new AdminService("jdbc:h2:mem:test", "sa", ""); + private final AdminService adminService = new AdminService("jdbc:h2:mem:test", "sa", "", "/tmp/"); @TempDir File tempStorageLocation; @@ -111,11 +111,12 @@ void shouldReturnNotImplementedWhenCallingReplicate() { // region restore-backup @Test - void shouldReturnNotImplementedWhenCallingRestore() { + void shouldRestoreBackup(CapturedOutput output) { final String backupFile = Path.of(tempStorageLocation.getPath(), "backup.sql").toString(); adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql"); assertTrue(new File(backupFile).exists()); adminService.restoreDatabaseFromBackupFile(tempStorageLocation.getPath(), "backup.sql"); + assertTrue(output.getOut().contains("Backup has been restored")); } @Test @@ -126,9 +127,18 @@ void withDbException(CapturedOutput output) { @Test void withSQLException(CapturedOutput output) { - AdminService corruptAdminService = new AdminService("url", "username", "password"); + AdminService corruptAdminService = new AdminService("url", "username", "password", "/tmp/"); corruptAdminService.restoreDatabaseFromBackupFile(tempStorageLocation.getPath(), "backup.sql"); assertTrue(output.getOut().contains("SQL error occurred during restore")); } // endregion + + @Test + void testIsPathInBaseDirectory() { + assertAll( + () -> assertFalse(adminService.isPathInBaseDirectory("/tmp/../../backup.sql")), + () -> assertTrue(adminService.isPathInBaseDirectory("/tmp/backup.sql")), + () -> assertTrue(adminService.isPathInBaseDirectory("/tmp/backup-copy.sql")) + ); + } } From f44efb114bd18cbf86de0927e78d04f93ab7c427 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 27 Oct 2023 18:44:45 +0200 Subject: [PATCH 32/64] Try other checks on backup file existence before restoration --- .../java/com/iexec/sms/admin/AdminService.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 79323e22..7201c386 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -24,8 +24,8 @@ import org.springframework.stereotype.Service; import java.io.File; +import java.io.IOException; import java.nio.charset.Charset; -import java.nio.file.FileSystemNotFoundException; import java.nio.file.Path; import java.sql.SQLException; import java.text.DateFormat; @@ -118,19 +118,26 @@ public String replicateDatabaseBackupFile(String storagePath, String backupFileN boolean restoreDatabaseFromBackupFile(String storagePath, String backupFileName) { try { final String fullBackupFileName = storagePath + File.separator + backupFileName; - if (!isPathInBaseDirectory(fullBackupFileName)) { - throw new FileSystemNotFoundException("Backup file " + fullBackupFileName + " not found"); + // checks on backup file to restore + final File backupFile = new File(fullBackupFileName); + final String backupFilePath = backupFile.getCanonicalPath(); + if (!backupFilePath.startsWith(storagePath)) { + throw new IOException("Backup file is outside of storage file system"); + } else if (!backupFile.exists()) { + throw new IOException("Backup file does not exist"); } + final long size = backupFilePath.length(); final long start = System.currentTimeMillis(); RunScript.execute(datasourceUrl, datasourceUsername, datasourcePassword, fullBackupFileName, Charset.defaultCharset(), true); final long stop = System.currentTimeMillis(); - final long size = new File(fullBackupFileName).length(); log.warn("Backup has been restored [timestamp:{}, duration:{} ms, size:{}, fullBackupFileName:{}]", dateFormat.format(new Date(start)), stop - start, size, fullBackupFileName); return true; } catch (DbException e) { log.error("RunScript error occurred during restore", e); + } catch (IOException e) { + log.error("Failed to read backup file size"); } catch (SQLException e) { log.error("SQL error occurred during restore", e); } From 370c06c7d35a06a1fd4afc2e1df24a1c62dde368 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 27 Oct 2023 19:01:10 +0200 Subject: [PATCH 33/64] Fix one test, disable another one --- src/test/java/com/iexec/sms/admin/AdminServiceTests.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 5e6b9fb4..b53d11c0 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -16,6 +16,7 @@ package com.iexec.sms.admin; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; @@ -120,6 +121,7 @@ void shouldRestoreBackup(CapturedOutput output) { } @Test + @Disabled void withDbException(CapturedOutput output) { adminService.restoreDatabaseFromBackupFile(tempStorageLocation.getPath(), "backup.sql"); assertTrue(output.getOut().contains("RunScript error occurred during restore")); @@ -127,7 +129,10 @@ void withDbException(CapturedOutput output) { @Test void withSQLException(CapturedOutput output) { + final String backupFile = Path.of(tempStorageLocation.getPath(), "backup.sql").toString(); AdminService corruptAdminService = new AdminService("url", "username", "password", "/tmp/"); + adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql"); + assertTrue(new File(backupFile).exists()); corruptAdminService.restoreDatabaseFromBackupFile(tempStorageLocation.getPath(), "backup.sql"); assertTrue(output.getOut().contains("SQL error occurred during restore")); } From 79b42490070694da80be5c3a43ab48217a02d1e4 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 27 Oct 2023 19:52:01 +0200 Subject: [PATCH 34/64] Add tests in `AdminServiceTests` --- .../java/com/iexec/sms/admin/AdminService.java | 4 ++-- .../com/iexec/sms/admin/AdminServiceTests.java | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 7201c386..8eeb44b4 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -121,7 +121,7 @@ boolean restoreDatabaseFromBackupFile(String storagePath, String backupFileName) // checks on backup file to restore final File backupFile = new File(fullBackupFileName); final String backupFilePath = backupFile.getCanonicalPath(); - if (!backupFilePath.startsWith(storagePath)) { + if (!backupFilePath.startsWith(storageFolder)) { throw new IOException("Backup file is outside of storage file system"); } else if (!backupFile.exists()) { throw new IOException("Backup file does not exist"); @@ -137,7 +137,7 @@ boolean restoreDatabaseFromBackupFile(String storagePath, String backupFileName) } catch (DbException e) { log.error("RunScript error occurred during restore", e); } catch (IOException e) { - log.error("Failed to read backup file size"); + log.error("Failed to read backup file size", e); } catch (SQLException e) { log.error("SQL error occurred during restore", e); } diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index b53d11c0..585dafea 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -120,6 +120,22 @@ void shouldRestoreBackup(CapturedOutput output) { assertTrue(output.getOut().contains("Backup has been restored")); } + @Test + void shouldFailToRestoreWithBackupFileMissing(CapturedOutput output) { + assertAll( + () -> assertFalse(adminService.restoreDatabaseFromBackupFile(tempStorageLocation.getPath(), "backup.sql")), + () -> assertTrue(output.getOut().contains("Backup file does not exist")) + ); + } + + @Test + void shouldFailToRestoreWithBackupFileOutOfStorage(CapturedOutput output) { + assertAll( + () -> assertFalse(adminService.restoreDatabaseFromBackupFile("/backup", "backup.sql")), + () -> assertTrue(output.getOut().contains("Backup file is outside of storage file system")) + ); + } + @Test @Disabled void withDbException(CapturedOutput output) { From be39fbf7b045591f22617b2f3d8bc903c3394ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Mon, 30 Oct 2023 17:58:49 +0100 Subject: [PATCH 35/64] add the ability to trigger a delete of backup file via a dedicated endpoint --- CHANGELOG.md | 1 + .../com/iexec/sms/admin/AdminController.java | 48 +++++++- .../com/iexec/sms/admin/AdminService.java | 101 ++++++++++++++-- .../iexec/sms/admin/AdminControllerTests.java | 79 +++++++++++- .../iexec/sms/admin/AdminServiceTests.java | 114 +++++++++++++++++- 5 files changed, 318 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bae2ef88..22d47e28 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file. - Create admin endpoints foundation. (#208 #209) - Add H2 database connection informations and storage ID decoding method. (#210) - Add the ability to trigger a backup via a dedicated endpoint. (#211) +- Add the ability to trigger a delete via a dedicated endpoint. (#) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index eb6d0b83..ebf11ccd 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -19,10 +19,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import java.nio.file.FileSystemNotFoundException; import java.nio.file.Files; @@ -167,6 +164,49 @@ public ResponseEntity restoreBackup(@PathVariable String storageID, @Req } } + /** + * Endpoint to delete a database backup. + *

+ * This method allows the client to initiate the deletion of a database backup + * from a specified dump file, identified by the {@code fileName}, located in a location specified by the {@code storageID}. + * + * @param storageID The unique identifier for the storage location of the dump in hexadecimal. + * @param fileName The name of the dump file to be deleted. + * @return A response entity indicating the status and details of the delete operation. + *

    + *
  • HTTP 200 (OK) - If the backup has been successfully deleted. + *
  • HTTP 400 (Bad Request) - If {@code fileName} is missing or {@code storageID} does not match an existing directory. + *
  • HTTP 404 (Not Found) - If the backup file specified by {@code fileName} does not exist. + *
  • HTTP 429 (Too Many Requests) - If another operation (backup/restore/replicate) is already in progress. + *
  • HTTP 500 (Internal Server Error) - If an unexpected error occurs during the restore process. + *
+ */ + @DeleteMapping("/{storageID}/delete-backup") + ResponseEntity deleteBackup(@PathVariable String storageID, @RequestParam String fileName) { + try { + if (StringUtils.isBlank(storageID) || StringUtils.isBlank(fileName)) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); + } + if (!tryToAcquireLock()) { + return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); + } + final String storagePath = getStoragePathFromID(storageID); + if (adminService.deleteBackupFileFromStorage(storagePath, fileName)) { + return ResponseEntity.ok().build(); + } + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); + } catch (FileSystemNotFoundException e) { + return ResponseEntity.notFound().build(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); + } catch (Exception e) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); + } finally { + tryToReleaseLock(); + } + } + /** * Converts {@code storageID} to an ascii string and checks if it is an existing folder. * diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index acfb49fd..6e0654d2 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -22,6 +22,10 @@ import org.springframework.stereotype.Service; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.sql.SQLException; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -36,13 +40,16 @@ public class AdminService { private final String datasourceUrl; private final String datasourceUsername; private final String datasourcePassword; + private final String storageFolder; public AdminService(@Value("${spring.datasource.url}") String datasourceUrl, @Value("${spring.datasource.username}") String datasourceUsername, - @Value("${spring.datasource.password}") String datasourcePassword) { + @Value("${spring.datasource.password}") String datasourcePassword, + @Value("${spring.datasource.storage-folder}") String storageFolder) { this.datasourceUrl = datasourceUrl; this.datasourceUsername = datasourceUsername; this.datasourcePassword = datasourcePassword; + this.storageFolder = storageFolder; } /** @@ -53,21 +60,13 @@ public AdminService(@Value("${spring.datasource.url}") String datasourceUrl, * @return {@code true} if the backup was successful; {@code false} if any error occurs. */ boolean createDatabaseBackupFile(String storageLocation, String backupFileName) { - // Check for missing or empty storageLocation parameter - if (StringUtils.isBlank(storageLocation)) { - log.error("storageLocation must not be empty."); - return false; - } - // Check for missing or empty backupFileName parameter - if (StringUtils.isBlank(backupFileName)) { - log.error("backupFileName must not be empty."); + // Ensure that storageLocation and backupFileName are not blanks + boolean validation = commonsParametersValidation(storageLocation, backupFileName); + if (!validation) { return false; } // Ensure that storageLocation ends with a slash - if (!storageLocation.endsWith(File.separator)) { - storageLocation += File.separator; - } - + storageLocation = normalizePathWithSeparator(storageLocation); // Check if storageLocation is an existing directory, we don't want to create it. final File directory = new File(storageLocation); if (!directory.isDirectory()) { @@ -110,4 +109,80 @@ public String replicateDatabaseBackupFile(String storageID, String fileName) { public String restoreDatabaseFromBackupFile(String storageId, String fileName) { return "restoreDatabaseFromBackupFile is not implemented"; } + + /** + * Delete a backup of the H2 database from a location + * + * @param storageLocation The location of the backup file. + * @param backupFileName The name of the backup file. + * @return {@code true} if the deletion was successful; {@code false} if any error occurs. + */ + public boolean deleteBackupFileFromStorage(String storageLocation, String backupFileName) { + + // Ensure that storageLocation and backupFileName are not blanks + boolean validation = commonsParametersValidation(storageLocation, backupFileName); + if (!validation) { + return false; + } + // Ensure that storageLocation correspond to an authorised area + if (!isPathInBaseDirectory(storageLocation)) { + log.error("Backup file is outside of storage file system [storageLocation:{}]", storageLocation); + return false; + } + // Ensure that storageLocation ends with a slash + storageLocation = normalizePathWithSeparator(storageLocation); + + String fullBackupFileName = storageLocation + backupFileName; + try { + Path fileToDeletePath = Paths.get(fullBackupFileName); + if (!fileToDeletePath.toFile().exists()) { + log.error("Backup file does not exist[fullBackupFileName:{}]", fullBackupFileName); + return false; + } + log.info("Starting the delete process [fullBackupFileName:{}]", fullBackupFileName); + final long start = System.currentTimeMillis(); + Files.delete(fileToDeletePath); + final long stop = System.currentTimeMillis(); + log.info("Successfully deleted backup [timestamp:{}, fullBackupFileName:{}, duration:{} ms]", dateFormat.format(new Date(start)), fullBackupFileName, stop - start); + return true; + } catch (IOException e) { + log.error("An error occurred while deleting backup [fullBackupFileName:{}]", fullBackupFileName, e); + } + return false; + } + + + boolean commonsParametersValidation(String storageLocation, String backupFileName) { + // Check for missing or empty storageLocation parameter + if (StringUtils.isBlank(storageLocation)) { + log.error("storageLocation must not be empty."); + return false; + } + // Check for missing or empty backupFileName parameter + if (StringUtils.isBlank(backupFileName)) { + log.error("backupFileName must not be empty."); + return false; + } + return true; + } + + boolean isPathInBaseDirectory(String pathToCheck) { + Path base = Path.of(storageFolder).toAbsolutePath().normalize(); + Path toCheck = Path.of(pathToCheck).toAbsolutePath().normalize(); + return toCheck.startsWith(base); + } + + /** + * Ensures that a given path string ends with a file separator by adding one if it's missing. + * + * @param path The path string to normalize. + * @return The normalized path with a trailing file separator. + */ + String normalizePathWithSeparator(String path) { + if (!path.endsWith(File.separator)) { + path += File.separator; + } + + return path; + } } diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index f74eabac..141fbbe7 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -49,6 +49,7 @@ class AdminControllerTests { private static final String STORAGE_ID = "storageID"; + private static final String STORAGE_PATH = "/storage"; private static final String FILE_NAME = "backup.sql"; @Mock @@ -78,7 +79,7 @@ void shouldReturnErrorWhenBackupFail() { @Test void shouldReturnTooManyRequestWhenBackupProcessIsAlreadyRunning() throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { @Override public boolean createDatabaseBackupFile(String storageLocation, String backupFileName) { try { @@ -145,7 +146,7 @@ void testNotFoundOnReplicate() { @Test void testTooManyRequestOnReplicate() throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { @Override public String replicateDatabaseBackupFile(String storageId, String fileName) { try { @@ -212,7 +213,7 @@ void testNotFoundOnRestore() { @Test void testTooManyRequestOnRestore() throws InterruptedException { - AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "") { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { @Override public String restoreDatabaseFromBackupFile(String storageId, String fileName) { try { @@ -269,6 +270,78 @@ private String convertToHex(String str) { } // endregion + // region delete-backup + @Test + void testDelete(@TempDir Path tempDir) { + final String storageID = convertToHex(tempDir.toString()); + when(adminService.deleteBackupFileFromStorage(tempDir.toString(), FILE_NAME)).thenReturn(true); + assertEquals(HttpStatus.OK, adminController.restoreBackup(storageID, FILE_NAME).getStatusCode()); + } + + @ParameterizedTest + @MethodSource("provideBadRequestParameters") + void testBadRequestOnDelete(String storageID, String fileName) { + assertEquals(HttpStatus.BAD_REQUEST, adminController.deleteBackup(storageID, fileName).getStatusCode()); + } + + @Test + void testInternalServerErrorOnDelete(@TempDir Path tempDir) { + final String storageID = convertToHex(tempDir.toString()); + when(adminService.deleteBackupFileFromStorage(tempDir.toString(), FILE_NAME)).thenThrow(RuntimeException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.deleteBackup(storageID, FILE_NAME).getStatusCode()); + } + + @Test + void testInterrupterThreadOnDelete() throws InterruptedException { + ReflectionTestUtils.setField(adminController, "rLock", rLock); + when(rLock.tryLock(100, TimeUnit.MILLISECONDS)).thenThrow(InterruptedException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.deleteBackup(STORAGE_PATH, FILE_NAME).getStatusCode()); + } + + @Test + void testNotFoundOnDelete() { + final String storageID = convertToHex(STORAGE_PATH); + when(adminService.deleteBackupFileFromStorage(STORAGE_PATH, FILE_NAME)).thenThrow(FileSystemNotFoundException.class); + assertEquals(HttpStatus.NOT_FOUND, adminController.deleteBackup(storageID, FILE_NAME).getStatusCode()); + } + + @Test + void testTooManyRequestOnDelete(@TempDir Path tempDir) throws InterruptedException { + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { + @Override + public boolean deleteBackupFileFromStorage(String storageLocation, String backupFileName) { + try { + log.info("Long restoreDatabaseFromBackupFile action is running ..."); + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return true; + } + }); + + final List> responses = Collections.synchronizedList(new ArrayList<>(3)); + final String storageID = convertToHex(tempDir.toString()); + + Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.deleteBackup(storageID, FILE_NAME))); + Thread secondThread = new Thread(() -> responses.add(adminControllerWithLongAction.deleteBackup(storageID, FILE_NAME))); + + firstThread.start(); + secondThread.start(); + responses.add(adminControllerWithLongAction.deleteBackup(storageID, FILE_NAME)); + + secondThread.join(); + firstThread.join(); + + + long code200 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.OK).count(); + long code429 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS).count(); + + assertEquals(1, code200); + assertEquals(2, code429); + } + // endregion + private static Stream provideBadRequestParameters() { return Stream.of( Arguments.of(null, null), diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 50651ecc..45d7507d 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -16,34 +16,53 @@ package com.iexec.sms.admin; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.NullSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.boot.test.system.CapturedOutput; +import org.springframework.boot.test.system.OutputCaptureExtension; +import org.springframework.context.annotation.Bean; +import javax.sql.DataSource; import java.io.File; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; +@ExtendWith(OutputCaptureExtension.class) class AdminServiceTests { - private final AdminService adminService = new AdminService("", "", ""); + @Bean + public DataSource dataSource() { + return DataSourceBuilder.create() + .url("jdbc:h2:mem:test") + .username("sa") + .password("") + .build(); + } + private final AdminService adminService = new AdminService("jdbc:h2:mem:test", "sa", "", "/tmp/"); @TempDir File tempStorageLocation; // region backup @Test void shouldReturnTrueWhenAllParametersAreValid() { - AdminService adminServiceSpy = Mockito.spy(adminService); - - Mockito.doReturn(true).when(adminServiceSpy).databaseDump(any()); - assertTrue(adminServiceSpy.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql")); + assertTrue(adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql")); } + @Test void shouldReturnFalseWhenAllParametersAreValidButBackupFailed() { AdminService adminServiceSpy = Mockito.spy(adminService); @@ -98,4 +117,89 @@ void shouldReturnNotImplementedWhenCallingRestore() { assertEquals("restoreDatabaseFromBackupFile is not implemented", adminService.restoreDatabaseFromBackupFile("", "")); } // endregion + + // region delete-backup + @Test + void shouldDeleteBackup(CapturedOutput output) throws IOException { + final String backupFileName = "backup.sql"; + final Path tmpFile = Files.createFile(tempStorageLocation.toPath().resolve(backupFileName)); + assertAll( + () -> assertTrue(adminService.deleteBackupFileFromStorage(tempStorageLocation.getPath(), backupFileName)), + () -> assertFalse(tmpFile.toFile().exists()), + () -> assertTrue(output.getOut().contains("Successfully deleted backup")) + ); + } + + @Test + void shouldFailedDeleteWithBackupFileMissing(CapturedOutput output) { + final String backupFileName = "backup.sql"; + assertAll( + () -> assertFalse(adminService.deleteBackupFileFromStorage(tempStorageLocation.getPath(), backupFileName)), + () -> assertFalse(adminService.deleteBackupFileFromStorage(tempStorageLocation.getPath(), "")), + () -> assertFalse(adminService.deleteBackupFileFromStorage("", backupFileName)), + () -> assertTrue(output.getOut().contains("Backup file does not exist")) + ); + } + + @Test + @Disabled("Can't lock file on linux") + void shouldFailedDeleteWithBackupFileCantDelete(CapturedOutput output) throws IOException { + final String backupFileName = "backupLock.sql"; + final Path tmpFile = Files.createFile(tempStorageLocation.toPath().resolve(backupFileName)); + + + try (FileChannel ignored = FileChannel.open(tmpFile, StandardOpenOption.WRITE)) { + assertAll( + () -> assertFalse(adminService.deleteBackupFileFromStorage(tempStorageLocation.getPath(), backupFileName)) + ); + } + } + + @Test + void shouldFailedToDeleteWithBackupFileOutOfStorage(CapturedOutput output) { + assertAll( + () -> assertFalse(adminService.deleteBackupFileFromStorage("/backup", "backup.sql")), + () -> assertTrue(output.getOut().contains("Backup file is outside of storage file system")) + ); + } + + // endregion + + //region utils + + @Test + void testIsPathInBaseDirectory() { + assertAll( + () -> assertFalse(adminService.isPathInBaseDirectory("/tmp/../../backup.sql")), + () -> assertTrue(adminService.isPathInBaseDirectory("/tmp/backup.sql")), + () -> assertTrue(adminService.isPathInBaseDirectory("/tmp/backup-copy.sql")) + ); + } + + @Test + void testNormalizePathWithSeparator() { + String path1 = "test/path/"; + String path2 = "test/path"; + assertAll( + () -> assertEquals(path1, adminService.normalizePathWithSeparator(path1)), + () -> assertEquals(path1, adminService.normalizePathWithSeparator(path2)) + ); + } + + @Test + void testCommonsParametersValidation() { + // Valid case + final String validStorageLocation = "test/path/"; + final String validBackupFileName = "backup.sql"; + final String emptyStorageLocation = ""; + final String emptyBackupFileName = ""; + + assertAll( + () -> assertTrue(adminService.commonsParametersValidation(validStorageLocation, validBackupFileName)), + () -> assertFalse(adminService.commonsParametersValidation(emptyStorageLocation, validBackupFileName)), + () -> assertFalse(adminService.commonsParametersValidation(validStorageLocation, emptyBackupFileName)), + () -> assertFalse(adminService.commonsParametersValidation(emptyStorageLocation, emptyBackupFileName)) + ); + } + // endregion } From c3cd7210f9b8baf3ee97cd5d66becf703ca52236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Tue, 31 Oct 2023 08:50:57 +0100 Subject: [PATCH 36/64] Add controle on input user data --- src/main/java/com/iexec/sms/admin/AdminService.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 6e0654d2..9a34b5b9 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -124,15 +124,17 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup if (!validation) { return false; } + + // Ensure that storageLocation ends with a slash + storageLocation = normalizePathWithSeparator(storageLocation); + String fullBackupFileName = storageLocation + backupFileName; + // Ensure that storageLocation correspond to an authorised area - if (!isPathInBaseDirectory(storageLocation)) { + if (!fullBackupFileName.startsWith(storageFolder) || !isPathInBaseDirectory(storageLocation)) { log.error("Backup file is outside of storage file system [storageLocation:{}]", storageLocation); return false; } - // Ensure that storageLocation ends with a slash - storageLocation = normalizePathWithSeparator(storageLocation); - String fullBackupFileName = storageLocation + backupFileName; try { Path fileToDeletePath = Paths.get(fullBackupFileName); if (!fileToDeletePath.toFile().exists()) { From 913f03541f6f3247d8505fb9c007e0eb16aa16bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Tue, 31 Oct 2023 09:14:49 +0100 Subject: [PATCH 37/64] Add controle on input user data --- .../com/iexec/sms/admin/AdminService.java | 38 +++++++++---------- .../iexec/sms/admin/AdminServiceTests.java | 17 --------- 2 files changed, 17 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 9a34b5b9..439183ee 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -118,29 +118,25 @@ public String restoreDatabaseFromBackupFile(String storageId, String fileName) { * @return {@code true} if the deletion was successful; {@code false} if any error occurs. */ public boolean deleteBackupFileFromStorage(String storageLocation, String backupFileName) { - - // Ensure that storageLocation and backupFileName are not blanks - boolean validation = commonsParametersValidation(storageLocation, backupFileName); - if (!validation) { - return false; - } - - // Ensure that storageLocation ends with a slash - storageLocation = normalizePathWithSeparator(storageLocation); - String fullBackupFileName = storageLocation + backupFileName; - - // Ensure that storageLocation correspond to an authorised area - if (!fullBackupFileName.startsWith(storageFolder) || !isPathInBaseDirectory(storageLocation)) { - log.error("Backup file is outside of storage file system [storageLocation:{}]", storageLocation); - return false; - } - try { - Path fileToDeletePath = Paths.get(fullBackupFileName); - if (!fileToDeletePath.toFile().exists()) { - log.error("Backup file does not exist[fullBackupFileName:{}]", fullBackupFileName); + // Ensure that storageLocation and backupFileName are not blanks + boolean validation = commonsParametersValidation(storageLocation, backupFileName); + if (!validation) { return false; } + // Ensure that storageLocation ends with a slash + storageLocation = normalizePathWithSeparator(storageLocation); + String fullBackupFileName = storageLocation + backupFileName; + + final File backupFile = new File(fullBackupFileName); + final String backupFilePath = backupFile.getCanonicalPath(); + // Ensure that storageLocation correspond to an authorised area + if (!backupFilePath.startsWith(storageFolder) || !isPathInBaseDirectory(backupFilePath)) { + throw new IOException("Backup file is outside of storage file system"); + } else if (!backupFile.exists()) { + throw new IOException("Backup file does not exist"); + } + Path fileToDeletePath = Paths.get(fullBackupFileName); log.info("Starting the delete process [fullBackupFileName:{}]", fullBackupFileName); final long start = System.currentTimeMillis(); Files.delete(fileToDeletePath); @@ -148,7 +144,7 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup log.info("Successfully deleted backup [timestamp:{}, fullBackupFileName:{}, duration:{} ms]", dateFormat.format(new Date(start)), fullBackupFileName, stop - start); return true; } catch (IOException e) { - log.error("An error occurred while deleting backup [fullBackupFileName:{}]", fullBackupFileName, e); + log.error("An error occurred while deleting backup", e); } return false; } diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 45d7507d..81bef011 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -16,7 +16,6 @@ package com.iexec.sms.admin; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; @@ -32,10 +31,8 @@ import javax.sql.DataSource; import java.io.File; import java.io.IOException; -import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.StandardOpenOption; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; @@ -141,20 +138,6 @@ void shouldFailedDeleteWithBackupFileMissing(CapturedOutput output) { ); } - @Test - @Disabled("Can't lock file on linux") - void shouldFailedDeleteWithBackupFileCantDelete(CapturedOutput output) throws IOException { - final String backupFileName = "backupLock.sql"; - final Path tmpFile = Files.createFile(tempStorageLocation.toPath().resolve(backupFileName)); - - - try (FileChannel ignored = FileChannel.open(tmpFile, StandardOpenOption.WRITE)) { - assertAll( - () -> assertFalse(adminService.deleteBackupFileFromStorage(tempStorageLocation.getPath(), backupFileName)) - ); - } - } - @Test void shouldFailedToDeleteWithBackupFileOutOfStorage(CapturedOutput output) { assertAll( From 67e0e2575f0b87a5e52075837524a476b3c8e1ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 2 Nov 2023 10:13:46 +0100 Subject: [PATCH 38/64] deleting a redundant check --- src/main/java/com/iexec/sms/admin/AdminService.java | 8 +------- src/test/java/com/iexec/sms/admin/AdminServiceTests.java | 9 --------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 439183ee..3db51714 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -131,7 +131,7 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup final File backupFile = new File(fullBackupFileName); final String backupFilePath = backupFile.getCanonicalPath(); // Ensure that storageLocation correspond to an authorised area - if (!backupFilePath.startsWith(storageFolder) || !isPathInBaseDirectory(backupFilePath)) { + if (!backupFilePath.startsWith(storageFolder)) { throw new IOException("Backup file is outside of storage file system"); } else if (!backupFile.exists()) { throw new IOException("Backup file does not exist"); @@ -164,12 +164,6 @@ boolean commonsParametersValidation(String storageLocation, String backupFileNam return true; } - boolean isPathInBaseDirectory(String pathToCheck) { - Path base = Path.of(storageFolder).toAbsolutePath().normalize(); - Path toCheck = Path.of(pathToCheck).toAbsolutePath().normalize(); - return toCheck.startsWith(base); - } - /** * Ensures that a given path string ends with a file separator by adding one if it's missing. * diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 81bef011..deeb80f3 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -150,15 +150,6 @@ void shouldFailedToDeleteWithBackupFileOutOfStorage(CapturedOutput output) { //region utils - @Test - void testIsPathInBaseDirectory() { - assertAll( - () -> assertFalse(adminService.isPathInBaseDirectory("/tmp/../../backup.sql")), - () -> assertTrue(adminService.isPathInBaseDirectory("/tmp/backup.sql")), - () -> assertTrue(adminService.isPathInBaseDirectory("/tmp/backup-copy.sql")) - ); - } - @Test void testNormalizePathWithSeparator() { String path1 = "test/path/"; From 5a682183a2a13dafdffa6f3a87d9f87d4011534d Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Thu, 2 Nov 2023 10:39:45 +0100 Subject: [PATCH 39/64] Cleanup, rename configuration properties and add documentation --- README.md | 2 + .../com/iexec/sms/admin/AdminService.java | 44 +++++++++---------- .../iexec/sms/config/ApiKeyFilterConfig.java | 4 +- src/main/resources/application.yml | 5 ++- .../iexec/sms/admin/AdminServiceTests.java | 29 ++++-------- .../sms/config/ApiKeyFilterConfigTests.java | 4 +- 6 files changed, 40 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 482dfc7c..f5e7c0fe 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ To support: | `IEXEC_SMS_H2_URL` | JDBC URL of the database. | URL | `jdbc:h2:file:/tmp/h2/sms-h2` | `jdbc:h2:file:/tmp/h2/sms-h2` | | `IEXEC_SMS_H2_CONSOLE` | Whether to enable the H2 console. | Boolean | `false` | `false` | | `IEXEC_SMS_STORAGE_ENCRYPTION_AES_KEY_PATH` | Path to the key created and used to encrypt secrets. | String | `src/main/resources/iexec-sms-aes.key` | `src/main/resources/iexec-sms-aes.key` | +| `IEXEC_SMS_ADMIN_API_KEY` | API key used to authorize calls to `/admin` endpoints. | String | | | +| `IEXEC_SMS_ADMIN_STORAGE_LOCATION` | Storage location where to persist replicated backups. It must be an absolute directory path. | String | `/backup` | `/backup` | | `IEXEC_CHAIN_ID` | Chain ID of the blockchain network to connect. | Positive integer | `134` | `134` | | `IEXEC_IS_SIDECHAIN` | Define whether iExec on-chain protocol is built on top of token (`false`) or native currency (`true`). | Boolean | `true` | `true` | | `IEXEC_SMS_BLOCKCHAIN_NODE_ADDRESS` | URL to connect to the blockchain node. | URL | `https://bellecour.iex.ec` | `https://bellecour.iex.ec` | diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 8eeb44b4..fbada54d 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -17,7 +17,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.h2.message.DbException; import org.h2.tools.RunScript; import org.h2.tools.Script; import org.springframework.beans.factory.annotation.Value; @@ -26,6 +25,7 @@ import java.io.File; import java.io.IOException; import java.nio.charset.Charset; +import java.nio.file.FileSystemNotFoundException; import java.nio.file.Path; import java.sql.SQLException; import java.text.DateFormat; @@ -41,16 +41,16 @@ public class AdminService { private final String datasourceUrl; private final String datasourceUsername; private final String datasourcePassword; - private final String storageFolder; + private final String adminStorageLocation; public AdminService(@Value("${spring.datasource.url}") String datasourceUrl, @Value("${spring.datasource.username}") String datasourceUsername, @Value("${spring.datasource.password}") String datasourcePassword, - @Value("${spring.datasource.storage-folder}") String storageFolder) { + @Value("${admin.storage-location}") String adminStorageLocation) { this.datasourceUrl = datasourceUrl; this.datasourceUsername = datasourceUsername; this.datasourcePassword = datasourcePassword; - this.storageFolder = storageFolder; + this.adminStorageLocation = adminStorageLocation; } /** @@ -115,38 +115,38 @@ public String replicateDatabaseBackupFile(String storagePath, String backupFileN return "replicateDatabaseBackupFile is not implemented"; } - boolean restoreDatabaseFromBackupFile(String storagePath, String backupFileName) { + /** + * Restores a backup from provided inputs. + *

+ * The location is checked against a configuration property value provided by an admin. + * + * @param storageLocation Where to find the backup file + * @param backupFileName The file to restore + * @return {@code true} if the restoration was successful, {@code false} otherwise. + */ + boolean restoreDatabaseFromBackupFile(String storageLocation, String backupFileName) { try { - final String fullBackupFileName = storagePath + File.separator + backupFileName; // checks on backup file to restore - final File backupFile = new File(fullBackupFileName); - final String backupFilePath = backupFile.getCanonicalPath(); - if (!backupFilePath.startsWith(storageFolder)) { + final File backupFile = new File(storageLocation + File.separator + backupFileName); + final String backupFileLocation = backupFile.getCanonicalPath(); + if (!backupFileLocation.startsWith(adminStorageLocation)) { throw new IOException("Backup file is outside of storage file system"); } else if (!backupFile.exists()) { - throw new IOException("Backup file does not exist"); + throw new FileSystemNotFoundException("Backup file does not exist"); } - final long size = backupFilePath.length(); + final long size = backupFileLocation.length(); final long start = System.currentTimeMillis(); RunScript.execute(datasourceUrl, datasourceUsername, datasourcePassword, - fullBackupFileName, Charset.defaultCharset(), true); + backupFileLocation, Charset.defaultCharset(), true); final long stop = System.currentTimeMillis(); log.warn("Backup has been restored [timestamp:{}, duration:{} ms, size:{}, fullBackupFileName:{}]", - dateFormat.format(new Date(start)), stop - start, size, fullBackupFileName); + dateFormat.format(new Date(start)), stop - start, size, backupFileLocation); return true; - } catch (DbException e) { - log.error("RunScript error occurred during restore", e); } catch (IOException e) { - log.error("Failed to read backup file size", e); + log.error("Invalid backup file location", e); } catch (SQLException e) { log.error("SQL error occurred during restore", e); } return false; } - - boolean isPathInBaseDirectory(String pathToCheck) { - Path base = Path.of(storageFolder).toAbsolutePath().normalize(); - Path toCheck = Path.of(pathToCheck).toAbsolutePath().normalize(); - return toCheck.startsWith(base); - } } diff --git a/src/main/java/com/iexec/sms/config/ApiKeyFilterConfig.java b/src/main/java/com/iexec/sms/config/ApiKeyFilterConfig.java index d6df1e50..501c41e3 100644 --- a/src/main/java/com/iexec/sms/config/ApiKeyFilterConfig.java +++ b/src/main/java/com/iexec/sms/config/ApiKeyFilterConfig.java @@ -25,12 +25,12 @@ @Configuration @ConditionalOnExpression( - "T(org.apache.commons.lang3.StringUtils).isNotEmpty('${server.admin-api-key:}')" + "T(org.apache.commons.lang3.StringUtils).isNotEmpty('${admin.api-key:}')" ) public class ApiKeyFilterConfig { @Bean - public FilterRegistrationBean filterRegistrationBean(@Value("${server.admin-api-key}") String apiKey) { + public FilterRegistrationBean filterRegistrationBean(@Value("${admin.api-key}") String apiKey) { FilterRegistrationBean registrationBean = new FilterRegistrationBean<>(); ApiKeyRequestFilter apiKeyRequestFilter = new ApiKeyRequestFilter(apiKey); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 7dedea5e..b15f47af 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -1,6 +1,5 @@ server: port: ${IEXEC_SMS_PORT:13300} - admin-api-key: ${IEXEC_SMS_ADMIN_API_KEY:} # Embedded H2 inside JVM spring: @@ -24,6 +23,10 @@ spring: enabled: ${IEXEC_SMS_H2_CONSOLE:false} # http://localhost:13300/h2-console/ settings.web-allow-others: ${IEXEC_SMS_H2_CONSOLE:false} # Get console if Docker run +admin: + api-key: ${IEXEC_SMS_ADMIN_API_KEY:} + storage-location: ${IEXEC_SMS_ADMIN_STORAGE_LOCATION:/backup} + encryption: # Will get previous key or else create one on this path # this file shouldn't be clearly readable outside the enclave (but encrypted content could be copied outside) diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 585dafea..c8f85ab7 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -16,7 +16,6 @@ package com.iexec.sms.admin; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.io.TempDir; @@ -31,6 +30,8 @@ import javax.sql.DataSource; import java.io.File; +import java.io.IOException; +import java.nio.file.FileSystemNotFoundException; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; @@ -121,10 +122,12 @@ void shouldRestoreBackup(CapturedOutput output) { } @Test - void shouldFailToRestoreWithBackupFileMissing(CapturedOutput output) { - assertAll( - () -> assertFalse(adminService.restoreDatabaseFromBackupFile(tempStorageLocation.getPath(), "backup.sql")), - () -> assertTrue(output.getOut().contains("Backup file does not exist")) + void shouldFailToRestoreWithBackupFileMissing() throws IOException { + final String backupStorageLocation = tempStorageLocation.getCanonicalPath(); + assertThrows( + FileSystemNotFoundException.class, + () -> adminService.restoreDatabaseFromBackupFile(backupStorageLocation, "backup.sql"), + "Backup file does not exist" ); } @@ -136,13 +139,6 @@ void shouldFailToRestoreWithBackupFileOutOfStorage(CapturedOutput output) { ); } - @Test - @Disabled - void withDbException(CapturedOutput output) { - adminService.restoreDatabaseFromBackupFile(tempStorageLocation.getPath(), "backup.sql"); - assertTrue(output.getOut().contains("RunScript error occurred during restore")); - } - @Test void withSQLException(CapturedOutput output) { final String backupFile = Path.of(tempStorageLocation.getPath(), "backup.sql").toString(); @@ -153,13 +149,4 @@ void withSQLException(CapturedOutput output) { assertTrue(output.getOut().contains("SQL error occurred during restore")); } // endregion - - @Test - void testIsPathInBaseDirectory() { - assertAll( - () -> assertFalse(adminService.isPathInBaseDirectory("/tmp/../../backup.sql")), - () -> assertTrue(adminService.isPathInBaseDirectory("/tmp/backup.sql")), - () -> assertTrue(adminService.isPathInBaseDirectory("/tmp/backup-copy.sql")) - ); - } } diff --git a/src/test/java/com/iexec/sms/config/ApiKeyFilterConfigTests.java b/src/test/java/com/iexec/sms/config/ApiKeyFilterConfigTests.java index 6ef93836..4f6c079e 100644 --- a/src/test/java/com/iexec/sms/config/ApiKeyFilterConfigTests.java +++ b/src/test/java/com/iexec/sms/config/ApiKeyFilterConfigTests.java @@ -29,13 +29,13 @@ class ApiKeyFilterConfigTests { @Test void shouldCreateFilterWhenAPIKeyIsFilled() { - runner.withPropertyValues("server.admin-api-key=879") + runner.withPropertyValues("admin.api-key=879") .run(context -> assertThat(context).hasSingleBean(ApiKeyFilterConfig.class)); } @Test void shouldNotCreateFilterWhenAPIKeyIsNotFilled() { - runner.withPropertyValues("server.admin-api-key=") + runner.withPropertyValues("admin.api-key=") .run(context -> assertThat(context).doesNotHaveBean(ApiKeyFilterConfig.class)); } } From 77772829b7815218ee346486b88ebb71efa6d37a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 2 Nov 2023 14:45:14 +0100 Subject: [PATCH 40/64] Fix bad test --- src/test/java/com/iexec/sms/admin/AdminControllerTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index cec74225..5c201897 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -282,7 +282,7 @@ private static String convertToHex(String str) { void testDelete(@TempDir Path tempDir) { final String storageID = convertToHex(tempDir.toString()); when(adminService.deleteBackupFileFromStorage(tempDir.toString(), FILE_NAME)).thenReturn(true); - assertEquals(HttpStatus.OK, adminController.restoreBackup(storageID, FILE_NAME).getStatusCode()); + assertEquals(HttpStatus.OK, adminController.deleteBackup(storageID, FILE_NAME).getStatusCode()); } @ParameterizedTest From 290e82478c034e2e240956a32007d617a4375cee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 2 Nov 2023 15:21:34 +0100 Subject: [PATCH 41/64] Fix test for FileNotFound --- .../java/com/iexec/sms/admin/AdminService.java | 4 ++-- .../com/iexec/sms/admin/AdminServiceTests.java | 15 +++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 51c335fb..d3f5b33b 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -26,8 +26,8 @@ import java.io.IOException; import java.nio.charset.Charset; import java.nio.file.FileSystemNotFoundException; -import java.nio.file.Path; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.sql.SQLException; import java.text.DateFormat; @@ -168,7 +168,7 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup if (!backupFilePath.startsWith(adminStorageLocation)) { throw new IOException("Backup file is outside of storage file system"); } else if (!backupFile.exists()) { - throw new IOException("Backup file does not exist"); + throw new FileSystemNotFoundException("Backup file does not exist"); } Path fileToDeletePath = Paths.get(fullBackupFileName); log.info("Starting the delete process [fullBackupFileName:{}]", fullBackupFileName); diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 91a5a918..018c11fb 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -32,8 +32,8 @@ import java.io.File; import java.io.IOException; import java.nio.file.FileSystemNotFoundException; -import java.nio.file.Path; import java.nio.file.Files; +import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; @@ -161,13 +161,12 @@ void shouldDeleteBackup(CapturedOutput output) throws IOException { } @Test - void shouldFailedDeleteWithBackupFileMissing(CapturedOutput output) { - final String backupFileName = "backup.sql"; - assertAll( - () -> assertFalse(adminService.deleteBackupFileFromStorage(tempStorageLocation.getPath(), backupFileName)), - () -> assertFalse(adminService.deleteBackupFileFromStorage(tempStorageLocation.getPath(), "")), - () -> assertFalse(adminService.deleteBackupFileFromStorage("", backupFileName)), - () -> assertTrue(output.getOut().contains("Backup file does not exist")) + void shouldFailedDeleteWithBackupFileMissing() throws IOException { + final String backupStorageLocation = tempStorageLocation.getCanonicalPath(); + assertThrows( + FileSystemNotFoundException.class, + () -> adminService.deleteBackupFileFromStorage(backupStorageLocation, "backup.sql"), + "Backup file does not exist" ); } From 24111cfabe8ee328bb9470a357c6f4a6abb02120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 2 Nov 2023 17:37:11 +0100 Subject: [PATCH 42/64] Refactor to avoid code redundancy --- CHANGELOG.md | 2 +- .../com/iexec/sms/admin/AdminController.java | 112 ++++++++---------- .../com/iexec/sms/admin/AdminService.java | 63 ++++------ .../iexec/sms/admin/AdminControllerTests.java | 11 +- .../iexec/sms/admin/AdminServiceTests.java | 20 +--- 5 files changed, 90 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da200974..2f090814 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ All notable changes to this project will be documented in this file. - Add H2 database connection informations and storage ID decoding method. (#210) - Add the ability to trigger a backup via a dedicated endpoint. (#211) - Add the ability to trigger a database restore via a dedicated endpoint. (#212) -- Add the ability to trigger a delete via a dedicated endpoint. (#) +- Add the ability to trigger a delete via a dedicated endpoint. (#213) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index 70239143..4c15ba4e 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -32,6 +32,13 @@ @RequestMapping("/admin") public class AdminController { + /** + * Enum representing different types of backup operations: BACKUP, DELETE, REPLICATE and RESTORE. + */ + private enum BackupAction { + BACKUP, DELETE, REPLICATE, RESTORE; + } + /** * The directory where the database backup file will be stored. * The value of this constant should be a valid, existing directory path. @@ -69,23 +76,8 @@ public AdminController(AdminService adminService) { * */ @PostMapping("/backup") - public ResponseEntity createBackup() { - try { - if (!tryToAcquireLock()) { - return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); - } - if (adminService.createDatabaseBackupFile(BACKUP_STORAGE_LOCATION, BACKUP_FILENAME)) { - return ResponseEntity.status(HttpStatus.CREATED).build(); - } - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); - } catch (Exception e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); - } finally { - tryToReleaseLock(); - } + public ResponseEntity createBackup() { + return performOperation("", "", BackupAction.BACKUP); } /** @@ -105,26 +97,8 @@ public ResponseEntity createBackup() { * */ @PostMapping("/{storageID}/replicate-backup") - public ResponseEntity replicateBackup(@PathVariable String storageID, @RequestParam String fileName) { - try { - if (StringUtils.isBlank(storageID) || StringUtils.isBlank(fileName)) { - return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); - } - if (!tryToAcquireLock()) { - return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); - } - final String storagePath = getStoragePathFromID(storageID); - return ResponseEntity.ok(adminService.replicateDatabaseBackupFile(storagePath, fileName)); - } catch (FileSystemNotFoundException e) { - return ResponseEntity.notFound().build(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); - } catch (Exception e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); - } finally { - tryToReleaseLock(); - } + public ResponseEntity replicateBackup(@PathVariable String storageID, @RequestParam String fileName) { + return performOperation(storageID, fileName, BackupAction.REPLICATE); } /** @@ -146,28 +120,7 @@ public ResponseEntity replicateBackup(@PathVariable String storageID, @R */ @PostMapping("/{storageID}/restore-backup") ResponseEntity restoreBackup(@PathVariable String storageID, @RequestParam String fileName) { - try { - if (StringUtils.isBlank(storageID) || StringUtils.isBlank(fileName)) { - return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); - } - if (!tryToAcquireLock()) { - return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); - } - final String storagePath = getStoragePathFromID(storageID); - if (adminService.restoreDatabaseFromBackupFile(storagePath, fileName)) { - return ResponseEntity.ok().build(); - } - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); - } catch (FileSystemNotFoundException e) { - return ResponseEntity.notFound().build(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); - } catch (Exception e) { - return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); - } finally { - tryToReleaseLock(); - } + return performOperation(storageID, fileName, BackupAction.RESTORE); } /** @@ -189,17 +142,54 @@ ResponseEntity restoreBackup(@PathVariable String storageID, @RequestParam */ @DeleteMapping("/{storageID}/delete-backup") ResponseEntity deleteBackup(@PathVariable String storageID, @RequestParam String fileName) { + return performOperation(storageID, fileName, BackupAction.DELETE); + } + + /** + * Common method for database backup operations. + * + * @param storageID The unique identifier for the storage location of the dump in hexadecimal. + * @param fileName The name of the dump file to be operated on. + * @param operationType The type of operation {{@link BackupAction}. + * @return A response entity indicating the status and details of the operation. + */ + private ResponseEntity performOperation(String storageID, String fileName, BackupAction operationType) { try { - if (StringUtils.isBlank(storageID) || StringUtils.isBlank(fileName)) { + if ((StringUtils.isBlank(storageID) || StringUtils.isBlank(fileName)) && operationType != BackupAction.BACKUP) { return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); } if (!tryToAcquireLock()) { return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); } + final String storagePath = getStoragePathFromID(storageID); - if (adminService.deleteBackupFileFromStorage(storagePath, fileName)) { + + boolean operationSuccessful = false; + + switch (operationType) { + case BACKUP: + operationSuccessful = adminService.createDatabaseBackupFile(BACKUP_STORAGE_LOCATION, BACKUP_FILENAME); + break; + case RESTORE: + operationSuccessful = adminService.restoreDatabaseFromBackupFile(storagePath, fileName); + break; + case DELETE: + operationSuccessful = adminService.deleteBackupFileFromStorage(storagePath, fileName); + break; + case REPLICATE: + operationSuccessful = adminService.replicateDatabaseBackupFile(storagePath, fileName); + break; + default: + break; + } + + if (operationSuccessful) { + if (operationType == BackupAction.BACKUP) { + return ResponseEntity.status(HttpStatus.CREATED).build(); + } return ResponseEntity.ok().build(); } + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build(); } catch (FileSystemNotFoundException e) { return ResponseEntity.notFound().build(); diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index d3f5b33b..3a02b622 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -63,21 +63,26 @@ public AdminService(@Value("${spring.datasource.url}") String datasourceUrl, * @return {@code true} if the backup was successful; {@code false} if any error occurs. */ boolean createDatabaseBackupFile(String storageLocation, String backupFileName) { - // Ensure that storageLocation and backupFileName are not blanks - boolean validation = commonsParametersValidation(storageLocation, backupFileName); - if (!validation) { - return false; - } - // Ensure that storageLocation ends with a slash - storageLocation = normalizePathWithSeparator(storageLocation); - // Check if storageLocation is an existing directory, we don't want to create it. - final File directory = new File(storageLocation); - if (!directory.isDirectory()) { - log.error("storageLocation must be an existing directory [storageLocation:{}]", storageLocation); - return false; - } + try { + // Ensure that storageLocation and backupFileName are not blanks + boolean validation = checkCommonParameters(storageLocation, backupFileName); + if (!validation) { + return false; + } + // Check if storageLocation is an existing directory, we don't want to create it. + final File directory = new File(storageLocation); + if (!directory.isDirectory()) { + log.error("storageLocation must be an existing directory [storageLocation:{}]", storageLocation); + return false; + } + final File backupFile = new File(storageLocation + File.separator + backupFileName); + final String backupFileLocation = backupFile.getCanonicalPath(); - return databaseDump(storageLocation + backupFileName); + return databaseDump(backupFileLocation); + } catch (IOException e) { + log.error("An error occurred while creating backup", e); + } + return false; } /** @@ -105,8 +110,8 @@ boolean databaseDump(String fullBackupFileName) { return true; } - public String replicateDatabaseBackupFile(String storagePath, String backupFileName) { - return "replicateDatabaseBackupFile is not implemented"; + boolean replicateDatabaseBackupFile(String storagePath, String backupFileName) { + return false; } /** @@ -154,18 +159,15 @@ boolean restoreDatabaseFromBackupFile(String storageLocation, String backupFileN public boolean deleteBackupFileFromStorage(String storageLocation, String backupFileName) { try { // Ensure that storageLocation and backupFileName are not blanks - boolean validation = commonsParametersValidation(storageLocation, backupFileName); + boolean validation = checkCommonParameters(storageLocation, backupFileName); if (!validation) { return false; } - // Ensure that storageLocation ends with a slash - storageLocation = normalizePathWithSeparator(storageLocation); - String fullBackupFileName = storageLocation + backupFileName; - + String fullBackupFileName = storageLocation + File.separator + backupFileName; final File backupFile = new File(fullBackupFileName); - final String backupFilePath = backupFile.getCanonicalPath(); + final String backupFileLocation = backupFile.getCanonicalPath(); // Ensure that storageLocation correspond to an authorised area - if (!backupFilePath.startsWith(adminStorageLocation)) { + if (!backupFileLocation.startsWith(adminStorageLocation)) { throw new IOException("Backup file is outside of storage file system"); } else if (!backupFile.exists()) { throw new FileSystemNotFoundException("Backup file does not exist"); @@ -184,7 +186,7 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup } - boolean commonsParametersValidation(String storageLocation, String backupFileName) { + boolean checkCommonParameters(String storageLocation, String backupFileName) { // Check for missing or empty storageLocation parameter if (StringUtils.isBlank(storageLocation)) { log.error("storageLocation must not be empty."); @@ -198,17 +200,4 @@ boolean commonsParametersValidation(String storageLocation, String backupFileNam return true; } - /** - * Ensures that a given path string ends with a file separator by adding one if it's missing. - * - * @param path The path string to normalize. - * @return The normalized path with a trailing file separator. - */ - String normalizePathWithSeparator(String path) { - if (!path.endsWith(File.separator)) { - path += File.separator; - } - - return path; - } } diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index 5c201897..e41c159c 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -18,6 +18,7 @@ import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -91,7 +92,7 @@ public boolean createDatabaseBackupFile(String storageLocation, String backupFil } }); - final List> responses = Collections.synchronizedList(new ArrayList<>(3)); + final List> responses = Collections.synchronizedList(new ArrayList<>(3)); Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.createBackup())); Thread secondThread = new Thread(() -> responses.add(adminControllerWithLongAction.createBackup())); @@ -115,8 +116,9 @@ public boolean createDatabaseBackupFile(String storageLocation, String backupFil // region replicate-backup @Test void testReplicate(@TempDir Path tempDir) { + // Test to change when the methode replicateDatabaseBackupFile will be implemented final String storageID = convertToHex(tempDir.toString()); - assertEquals(HttpStatus.OK, adminController.replicateBackup(storageID, FILE_NAME).getStatusCode()); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.replicateBackup(storageID, FILE_NAME).getStatusCode()); } @ParameterizedTest @@ -146,10 +148,11 @@ void testNotFoundOnReplicate() { } @Test + @Disabled("Test to change when the methode replicateDatabaseBackupFile will be implemented") void testTooManyRequestOnReplicate(@TempDir Path tempDir) throws InterruptedException { AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { @Override - public String replicateDatabaseBackupFile(String storagePath, String backupFileName) { + public boolean replicateDatabaseBackupFile(String storagePath, String backupFileName) { try { log.info("Long replicateDatabaseBackupFile action is running ..."); Thread.sleep(2000); @@ -160,7 +163,7 @@ public String replicateDatabaseBackupFile(String storagePath, String backupFileN } }); - final List> responses = Collections.synchronizedList(new ArrayList<>(3)); + final List> responses = Collections.synchronizedList(new ArrayList<>(3)); final String storageID = convertToHex(tempDir.toString()); Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.replicateBackup(storageID, FILE_NAME))); diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 018c11fb..bdac4eb4 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -105,7 +105,7 @@ void shouldReturnFalseWhenBackupFileNameDoesNotExist() { // region replicate-backup @Test void shouldReturnNotImplementedWhenCallingReplicate() { - assertEquals("replicateDatabaseBackupFile is not implemented", adminService.replicateDatabaseBackupFile("", "")); + assertFalse(adminService.replicateDatabaseBackupFile("", "")); } // endregion @@ -182,16 +182,6 @@ void shouldFailedToDeleteWithBackupFileOutOfStorage(CapturedOutput output) { //region utils - @Test - void testNormalizePathWithSeparator() { - String path1 = "test/path/"; - String path2 = "test/path"; - assertAll( - () -> assertEquals(path1, adminService.normalizePathWithSeparator(path1)), - () -> assertEquals(path1, adminService.normalizePathWithSeparator(path2)) - ); - } - @Test void testCommonsParametersValidation() { // Valid case @@ -201,10 +191,10 @@ void testCommonsParametersValidation() { final String emptyBackupFileName = ""; assertAll( - () -> assertTrue(adminService.commonsParametersValidation(validStorageLocation, validBackupFileName)), - () -> assertFalse(adminService.commonsParametersValidation(emptyStorageLocation, validBackupFileName)), - () -> assertFalse(adminService.commonsParametersValidation(validStorageLocation, emptyBackupFileName)), - () -> assertFalse(adminService.commonsParametersValidation(emptyStorageLocation, emptyBackupFileName)) + () -> assertTrue(adminService.checkCommonParameters(validStorageLocation, validBackupFileName)), + () -> assertFalse(adminService.checkCommonParameters(emptyStorageLocation, validBackupFileName)), + () -> assertFalse(adminService.checkCommonParameters(validStorageLocation, emptyBackupFileName)), + () -> assertFalse(adminService.checkCommonParameters(emptyStorageLocation, emptyBackupFileName)) ); } // endregion From 4b66a16ffaf13dcb42b1abf7387ad02405c72aa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Fri, 3 Nov 2023 09:26:52 +0100 Subject: [PATCH 43/64] Deleting unnecessary variables --- src/main/java/com/iexec/sms/admin/AdminService.java | 8 +++----- src/test/java/com/iexec/sms/admin/AdminServiceTests.java | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 3a02b622..d8730190 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -27,7 +27,6 @@ import java.nio.charset.Charset; import java.nio.file.FileSystemNotFoundException; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.sql.SQLException; import java.text.DateFormat; @@ -172,12 +171,11 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup } else if (!backupFile.exists()) { throw new FileSystemNotFoundException("Backup file does not exist"); } - Path fileToDeletePath = Paths.get(fullBackupFileName); - log.info("Starting the delete process [fullBackupFileName:{}]", fullBackupFileName); + log.info("Starting the delete process [backupFileLocation:{}]", backupFileLocation); final long start = System.currentTimeMillis(); - Files.delete(fileToDeletePath); + Files.delete(Paths.get(backupFileLocation)); final long stop = System.currentTimeMillis(); - log.info("Successfully deleted backup [timestamp:{}, fullBackupFileName:{}, duration:{} ms]", dateFormat.format(new Date(start)), fullBackupFileName, stop - start); + log.info("Successfully deleted backup [timestamp:{}, backupFileLocation:{}, duration:{} ms]", dateFormat.format(new Date(start)), backupFileLocation, stop - start); return true; } catch (IOException e) { log.error("An error occurred while deleting backup", e); diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index bdac4eb4..3b17e6ae 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -183,7 +183,7 @@ void shouldFailedToDeleteWithBackupFileOutOfStorage(CapturedOutput output) { //region utils @Test - void testCommonsParametersValidation() { + void testCheckCommonParametersValidation() { // Valid case final String validStorageLocation = "test/path/"; final String validBackupFileName = "backup.sql"; From 4a00b6879a6757f1426f14cc4e60d3c4948dedcb Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 3 Nov 2023 12:06:08 +0100 Subject: [PATCH 44/64] Add the ability to trigger a backup replication via a dedicated endpoint --- CHANGELOG.md | 1 + .../com/iexec/sms/admin/AdminController.java | 6 +- .../com/iexec/sms/admin/AdminService.java | 51 +++++++++++---- .../iexec/sms/admin/AdminControllerTests.java | 17 +++-- .../iexec/sms/admin/AdminServiceTests.java | 64 +++++++++++++++++-- 5 files changed, 111 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f090814..058696ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file. - Add the ability to trigger a backup via a dedicated endpoint. (#211) - Add the ability to trigger a database restore via a dedicated endpoint. (#212) - Add the ability to trigger a delete via a dedicated endpoint. (#213) +- Add the ability to trigger a backup replication via a dedicated endpoint. (#214) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index 4c15ba4e..c02dd626 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.iexec.sms.admin; import lombok.extern.slf4j.Slf4j; @@ -97,7 +98,7 @@ public ResponseEntity createBackup() { * */ @PostMapping("/{storageID}/replicate-backup") - public ResponseEntity replicateBackup(@PathVariable String storageID, @RequestParam String fileName) { + ResponseEntity replicateBackup(@PathVariable String storageID, @RequestParam String fileName) { return performOperation(storageID, fileName, BackupAction.REPLICATE); } @@ -177,7 +178,8 @@ private ResponseEntity performOperation(String storageID, String fileName, operationSuccessful = adminService.deleteBackupFileFromStorage(storagePath, fileName); break; case REPLICATE: - operationSuccessful = adminService.replicateDatabaseBackupFile(storagePath, fileName); + operationSuccessful = adminService.replicateDatabaseBackupFile( + BACKUP_STORAGE_LOCATION, BACKUP_FILENAME, storagePath, fileName); break; default: break; diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index d8730190..099a3df0 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.iexec.sms.admin; import lombok.extern.slf4j.Slf4j; @@ -25,9 +26,7 @@ import java.io.File; import java.io.IOException; import java.nio.charset.Charset; -import java.nio.file.FileSystemNotFoundException; -import java.nio.file.Files; -import java.nio.file.Paths; +import java.nio.file.*; import java.sql.SQLException; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -59,7 +58,7 @@ public AdminService(@Value("${spring.datasource.url}") String datasourceUrl, * * @param storageLocation The location where the backup file will be saved, must be an existing directory. * @param backupFileName The name of the backup file. - * @return {@code true} if the backup was successful; {@code false} if any error occurs. + * @return {@code true} if the backup was successful, {@code false} if any error occurs. */ boolean createDatabaseBackupFile(String storageLocation, String backupFileName) { try { @@ -86,7 +85,7 @@ boolean createDatabaseBackupFile(String storageLocation, String backupFileName) /** * @param fullBackupFileName complete fileName (location and filename) - * @return {@code true} if the backup was successful; {@code false} if any error occurs. + * @return {@code true} if the backup was successful, {@code false} if any error occurs. */ boolean databaseDump(String fullBackupFileName) { @@ -101,7 +100,8 @@ boolean databaseDump(String fullBackupFileName) { Script.process(datasourceUrl, datasourceUsername, datasourcePassword, fullBackupFileName, "DROP", ""); final long stop = System.currentTimeMillis(); final long size = new File(fullBackupFileName).length(); - log.info("New backup created [timestamp:{}, duration:{} ms, size:{}, fullBackupFileName:{}]", dateFormat.format(new Date(start)), stop - start, size, fullBackupFileName); + log.info("New backup created [fullBackupFileName:{}, timestamp:{}, duration:{} ms, size:{}]", + fullBackupFileName, dateFormat.format(new Date(start)), stop - start, size); } catch (SQLException e) { log.error("SQL error occurred during backup", e); return false; @@ -109,7 +109,35 @@ boolean databaseDump(String fullBackupFileName) { return true; } - boolean replicateDatabaseBackupFile(String storagePath, String backupFileName) { + /** + * Replicates a backup to the persistent storage. + *

+ * The {@code backupStorageLocation/backupFileName} is replicated to {@code replicateStorageLocation/replicateFileName}. + * + * @param backupStorageLocation Location of backup to replicate + * @param backupFileName Name of backup file to replicate + * @param replicateStorageLocation Location of replicated backup + * @param replicateFileName Name of replicated backup file + * @return {@code true} if the replication was successful, {@code false} if any error occurs. + */ + boolean replicateDatabaseBackupFile(String backupStorageLocation, String backupFileName, String replicateStorageLocation, String replicateFileName) { + try { + final Path fullBackupFilePath = Path.of(backupStorageLocation, backupFileName).toRealPath(); + final Path backupFileLocation = Path.of(replicateStorageLocation, replicateFileName).toAbsolutePath(); + if (!backupFileLocation.startsWith(adminStorageLocation)) { + throw new IOException("Replicated backup file destination is outside of storage file system"); + } + final long size = fullBackupFilePath.toFile().length(); + log.info("Starting the replicate process [backupFileLocation:{}]", backupFileLocation); + final long start = System.currentTimeMillis(); + Files.copy(fullBackupFilePath, backupFileLocation, StandardCopyOption.COPY_ATTRIBUTES); + final long stop = System.currentTimeMillis(); + log.info("Backup has been replicated [backupFileLocation:{}, timestamp:{}, duration:{} ms, size:{}]", + backupFileLocation, dateFormat.format(start), stop - start, size); + return true; + } catch (IOException e) { + log.error("Error occurred during copy", e); + } return false; } @@ -120,7 +148,7 @@ boolean replicateDatabaseBackupFile(String storagePath, String backupFileName) { * * @param storageLocation Where to find the backup file * @param backupFileName The file to restore - * @return {@code true} if the restoration was successful, {@code false} otherwise. + * @return {@code true} if the restoration was successful, {@code false} if any error occurs. */ boolean restoreDatabaseFromBackupFile(String storageLocation, String backupFileName) { try { @@ -133,12 +161,13 @@ boolean restoreDatabaseFromBackupFile(String storageLocation, String backupFileN throw new FileSystemNotFoundException("Backup file does not exist"); } final long size = backupFileLocation.length(); + log.info("Starting the restore process [backupFileLocation:{}]", backupFileLocation); final long start = System.currentTimeMillis(); RunScript.execute(datasourceUrl, datasourceUsername, datasourcePassword, backupFileLocation, Charset.defaultCharset(), true); final long stop = System.currentTimeMillis(); - log.warn("Backup has been restored [timestamp:{}, duration:{} ms, size:{}, fullBackupFileName:{}]", - dateFormat.format(new Date(start)), stop - start, size, backupFileLocation); + log.warn("Backup has been restored [backupFileLocation:{}, timestamp:{}, duration:{} ms, size:{}]", + backupFileLocation, dateFormat.format(new Date(start)), stop - start, size); return true; } catch (IOException e) { log.error("Invalid backup file location", e); @@ -153,7 +182,7 @@ boolean restoreDatabaseFromBackupFile(String storageLocation, String backupFileN * * @param storageLocation The location of the backup file. * @param backupFileName The name of the backup file. - * @return {@code true} if the deletion was successful; {@code false} if any error occurs. + * @return {@code true} if the deletion was successful, {@code false} if any error occurs. */ public boolean deleteBackupFileFromStorage(String storageLocation, String backupFileName) { try { diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index e41c159c..ffd9a23b 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -18,7 +18,6 @@ import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -118,7 +117,8 @@ public boolean createDatabaseBackupFile(String storageLocation, String backupFil void testReplicate(@TempDir Path tempDir) { // Test to change when the methode replicateDatabaseBackupFile will be implemented final String storageID = convertToHex(tempDir.toString()); - assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.replicateBackup(storageID, FILE_NAME).getStatusCode()); + when(adminService.replicateDatabaseBackupFile("/work/", FILE_NAME, tempDir.toString(), FILE_NAME)).thenReturn(true); + assertEquals(HttpStatus.OK, adminController.replicateBackup(storageID, FILE_NAME).getStatusCode()); } @ParameterizedTest @@ -129,7 +129,7 @@ void testBadRequestOnReplicate(String storageID, String fileName) { @Test void testInternalServerErrorOnReplicate() { - when(adminService.replicateDatabaseBackupFile(STORAGE_PATH, FILE_NAME)).thenThrow(RuntimeException.class); + when(adminService.replicateDatabaseBackupFile(STORAGE_PATH, FILE_NAME, STORAGE_PATH, FILE_NAME)).thenThrow(RuntimeException.class); assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.replicateBackup(STORAGE_PATH, FILE_NAME).getStatusCode()); } @@ -143,23 +143,22 @@ void testInterruptedThreadOnReplicate() throws InterruptedException { @Test void testNotFoundOnReplicate() { final String storageID = convertToHex(STORAGE_PATH); - when(adminService.replicateDatabaseBackupFile(STORAGE_PATH, FILE_NAME)).thenThrow(FileSystemNotFoundException.class); + when(adminService.replicateDatabaseBackupFile(STORAGE_PATH, FILE_NAME, STORAGE_PATH, FILE_NAME)).thenThrow(FileSystemNotFoundException.class); assertEquals(HttpStatus.NOT_FOUND, adminController.replicateBackup(storageID, FILE_NAME).getStatusCode()); } @Test - @Disabled("Test to change when the methode replicateDatabaseBackupFile will be implemented") void testTooManyRequestOnReplicate(@TempDir Path tempDir) throws InterruptedException { AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { @Override - public boolean replicateDatabaseBackupFile(String storagePath, String backupFileName) { + public boolean replicateDatabaseBackupFile(String backupStoragePath, String backupFileName, String replicateStoragePath, String replicateFileName) { try { log.info("Long replicateDatabaseBackupFile action is running ..."); Thread.sleep(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } - return adminService.replicateDatabaseBackupFile(storagePath, backupFileName); + return true; } }); @@ -207,7 +206,7 @@ void testInternalServerErrorOnRestore(@TempDir Path tempDir) { } @Test - void testInterrupterThreadOnRestore() throws InterruptedException { + void testInterruptedThreadOnRestore() throws InterruptedException { ReflectionTestUtils.setField(adminController, "rLock", rLock); when(rLock.tryLock(100, TimeUnit.MILLISECONDS)).thenThrow(InterruptedException.class); assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.restoreBackup(STORAGE_PATH, FILE_NAME).getStatusCode()); @@ -302,7 +301,7 @@ void testInternalServerErrorOnDelete(@TempDir Path tempDir) { } @Test - void testInterrupterThreadOnDelete() throws InterruptedException { + void testInterruptedThreadOnDelete() throws InterruptedException { ReflectionTestUtils.setField(adminController, "rLock", rLock); when(rLock.tryLock(100, TimeUnit.MILLISECONDS)).thenThrow(InterruptedException.class); assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.deleteBackup(STORAGE_PATH, FILE_NAME).getStatusCode()); diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 3b17e6ae..5d2543fd 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -104,8 +104,47 @@ void shouldReturnFalseWhenBackupFileNameDoesNotExist() { // region replicate-backup @Test - void shouldReturnNotImplementedWhenCallingReplicate() { - assertFalse(adminService.replicateDatabaseBackupFile("", "")); + void shouldReplicateBackup() { + adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql"); + adminService.replicateDatabaseBackupFile( + tempStorageLocation.getPath(), "backup.sql", + tempStorageLocation.getPath(), "backup-copy.sql"); + assertTrue(Path.of(tempStorageLocation.getPath(), "backup-copy.sql").toFile().isFile()); + } + + @Test + void shouldFailToReplicateWithBackupFileMissing(CapturedOutput output) { + adminService.replicateDatabaseBackupFile( + tempStorageLocation.getPath(), "backup.sql", + tempStorageLocation.getPath(), "backup-copy.sql"); + assertAll( + () -> assertFalse(Path.of(tempStorageLocation.getPath(), "backup-copy.sql").toFile().isFile()), + () -> assertTrue(output.getOut().contains("NoSuchFileException")) + ); + } + + @Test + void shouldFailToReplicateWithInvalidStorageLocation(CapturedOutput output) { + adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql"); + boolean replicateStatus = adminService.replicateDatabaseBackupFile( + tempStorageLocation.getPath(), "backup.sql", + "/tmp/nonexistent/directory", "backup-copy.sql"); + assertAll( + () -> assertFalse(replicateStatus), + () -> assertTrue(output.getAll().contains("NoSuchFileException")) + ); + } + + @Test + void shouldFailToReplicateWithOutOfStorageLocation(CapturedOutput output) { + adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql"); + boolean replicateStatus = adminService.replicateDatabaseBackupFile( + tempStorageLocation.getPath(), "backup.sql", + "/opt", "backup-copy.sql"); + assertAll( + () -> assertFalse(replicateStatus), + () -> assertTrue(output.getAll().contains("Replicated backup file destination is outside of storage file system")) + ); } // endregion @@ -120,7 +159,7 @@ void shouldRestoreBackup(CapturedOutput output) { } @Test - void shouldFailToRestoreWithBackupFileMissing() throws IOException { + void shouldFailToRestoreWhenBackupFileMissing() throws IOException { final String backupStorageLocation = tempStorageLocation.getCanonicalPath(); assertThrows( FileSystemNotFoundException.class, @@ -130,7 +169,7 @@ void shouldFailToRestoreWithBackupFileMissing() throws IOException { } @Test - void shouldFailToRestoreWithBackupFileOutOfStorage(CapturedOutput output) { + void shouldFailToRestoreWhenBackupFileOutOfStorage(CapturedOutput output) { assertAll( () -> assertFalse(adminService.restoreDatabaseFromBackupFile("/backup", "backup.sql")), () -> assertTrue(output.getOut().contains("Backup file is outside of storage file system")) @@ -161,7 +200,7 @@ void shouldDeleteBackup(CapturedOutput output) throws IOException { } @Test - void shouldFailedDeleteWithBackupFileMissing() throws IOException { + void shouldFailToDeleteWhenBackupFileMissing() throws IOException { final String backupStorageLocation = tempStorageLocation.getCanonicalPath(); assertThrows( FileSystemNotFoundException.class, @@ -171,13 +210,26 @@ void shouldFailedDeleteWithBackupFileMissing() throws IOException { } @Test - void shouldFailedToDeleteWithBackupFileOutOfStorage(CapturedOutput output) { + void shouldFailToDeleteWhenBackupFileOutOfStorage(CapturedOutput output) { assertAll( () -> assertFalse(adminService.deleteBackupFileFromStorage("/backup", "backup.sql")), () -> assertTrue(output.getOut().contains("Backup file is outside of storage file system")) ); } + @Test + void shouldFailToDeleteWhenInvalidParameters() { + final String validStorageLocation = "test/path/"; + final String validBackupFileName = "backup.sql"; + final String emptyStorageLocation = ""; + final String emptyBackupFileName = ""; + + assertAll( + () -> assertFalse(adminService.deleteBackupFileFromStorage(emptyStorageLocation, validBackupFileName)), + () -> assertFalse(adminService.deleteBackupFileFromStorage(validStorageLocation, emptyBackupFileName)), + () -> assertFalse(adminService.deleteBackupFileFromStorage(emptyStorageLocation, emptyBackupFileName)) + ); + } // endregion //region utils From 206951fddfd808141e79a54c6d2dfe44b778f176 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 3 Nov 2023 13:02:01 +0100 Subject: [PATCH 45/64] Gather backup file location check against configured admin storage in a dedicated method --- .../com/iexec/sms/admin/AdminService.java | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 099a3df0..1aa8c0a2 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -152,14 +152,7 @@ boolean replicateDatabaseBackupFile(String backupStorageLocation, String backupF */ boolean restoreDatabaseFromBackupFile(String storageLocation, String backupFileName) { try { - // checks on backup file to restore - final File backupFile = new File(storageLocation + File.separator + backupFileName); - final String backupFileLocation = backupFile.getCanonicalPath(); - if (!backupFileLocation.startsWith(adminStorageLocation)) { - throw new IOException("Backup file is outside of storage file system"); - } else if (!backupFile.exists()) { - throw new FileSystemNotFoundException("Backup file does not exist"); - } + final String backupFileLocation = checkBackupFileLocation(storageLocation + File.separator + backupFileName); final long size = backupFileLocation.length(); log.info("Starting the restore process [backupFileLocation:{}]", backupFileLocation); final long start = System.currentTimeMillis(); @@ -191,20 +184,13 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup if (!validation) { return false; } - String fullBackupFileName = storageLocation + File.separator + backupFileName; - final File backupFile = new File(fullBackupFileName); - final String backupFileLocation = backupFile.getCanonicalPath(); - // Ensure that storageLocation correspond to an authorised area - if (!backupFileLocation.startsWith(adminStorageLocation)) { - throw new IOException("Backup file is outside of storage file system"); - } else if (!backupFile.exists()) { - throw new FileSystemNotFoundException("Backup file does not exist"); - } + final String backupFileLocation = checkBackupFileLocation(storageLocation + File.separator + backupFileName); log.info("Starting the delete process [backupFileLocation:{}]", backupFileLocation); final long start = System.currentTimeMillis(); Files.delete(Paths.get(backupFileLocation)); final long stop = System.currentTimeMillis(); - log.info("Successfully deleted backup [timestamp:{}, backupFileLocation:{}, duration:{} ms]", dateFormat.format(new Date(start)), backupFileLocation, stop - start); + log.info("Successfully deleted backup [backupFileLocation:{}, timestamp:{}, duration:{} ms]", + backupFileLocation, dateFormat.format(new Date(start)), stop - start); return true; } catch (IOException e) { log.error("An error occurred while deleting backup", e); @@ -212,6 +198,18 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup return false; } + String checkBackupFileLocation(String fullBackupFileName) throws IOException { + final File backupFile = new File(fullBackupFileName); + final String backupFileLocation = backupFile.getCanonicalPath(); + // Ensure that storageLocation correspond to an authorised area + if (!backupFileLocation.startsWith(adminStorageLocation)) { + throw new IOException("Backup file is outside of storage file system"); + } else if (!backupFile.exists()) { + throw new FileSystemNotFoundException("Backup file does not exist"); + } + return backupFileLocation; + } + boolean checkCommonParameters(String storageLocation, String backupFileName) { // Check for missing or empty storageLocation parameter From 8ba6518cf0e5875dd547264c81b79411636c1c12 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 3 Nov 2023 13:49:29 +0100 Subject: [PATCH 46/64] Fix vulnerability in backup replication method --- .../com/iexec/sms/admin/AdminService.java | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 1aa8c0a2..a9007148 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -26,7 +26,10 @@ import java.io.File; import java.io.IOException; import java.nio.charset.Charset; -import java.nio.file.*; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.sql.SQLException; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -123,14 +126,13 @@ boolean databaseDump(String fullBackupFileName) { boolean replicateDatabaseBackupFile(String backupStorageLocation, String backupFileName, String replicateStorageLocation, String replicateFileName) { try { final Path fullBackupFilePath = Path.of(backupStorageLocation, backupFileName).toRealPath(); - final Path backupFileLocation = Path.of(replicateStorageLocation, replicateFileName).toAbsolutePath(); - if (!backupFileLocation.startsWith(adminStorageLocation)) { - throw new IOException("Replicated backup file destination is outside of storage file system"); - } + final String backupFileLocation = checkBackupFileLocation( + replicateStorageLocation + File.separator + replicateFileName, + "Replicated backup file destination is outside of storage file system"); final long size = fullBackupFilePath.toFile().length(); log.info("Starting the replicate process [backupFileLocation:{}]", backupFileLocation); final long start = System.currentTimeMillis(); - Files.copy(fullBackupFilePath, backupFileLocation, StandardCopyOption.COPY_ATTRIBUTES); + Files.copy(fullBackupFilePath, Path.of(backupFileLocation), StandardCopyOption.COPY_ATTRIBUTES); final long stop = System.currentTimeMillis(); log.info("Backup has been replicated [backupFileLocation:{}, timestamp:{}, duration:{} ms, size:{}]", backupFileLocation, dateFormat.format(start), stop - start, size); @@ -152,7 +154,12 @@ boolean replicateDatabaseBackupFile(String backupStorageLocation, String backupF */ boolean restoreDatabaseFromBackupFile(String storageLocation, String backupFileName) { try { - final String backupFileLocation = checkBackupFileLocation(storageLocation + File.separator + backupFileName); + final String backupFileLocation = checkBackupFileLocation( + storageLocation + File.separator + backupFileName, + "Backup file is outside of storage file system"); + if (!Path.of(backupFileLocation).toFile().exists()) { + throw new FileSystemNotFoundException("Backup file does not exist"); + } final long size = backupFileLocation.length(); log.info("Starting the restore process [backupFileLocation:{}]", backupFileLocation); final long start = System.currentTimeMillis(); @@ -184,10 +191,16 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup if (!validation) { return false; } - final String backupFileLocation = checkBackupFileLocation(storageLocation + File.separator + backupFileName); + final String backupFileLocation = checkBackupFileLocation( + storageLocation + File.separator + backupFileName, + "Backup file is outside of storage file system"); + final Path backupFileLocationPath = Path.of(backupFileLocation); + if (!backupFileLocationPath.toFile().exists()) { + throw new FileSystemNotFoundException("Backup file does not exist"); + } log.info("Starting the delete process [backupFileLocation:{}]", backupFileLocation); final long start = System.currentTimeMillis(); - Files.delete(Paths.get(backupFileLocation)); + Files.delete(backupFileLocationPath); final long stop = System.currentTimeMillis(); log.info("Successfully deleted backup [backupFileLocation:{}, timestamp:{}, duration:{} ms]", backupFileLocation, dateFormat.format(new Date(start)), stop - start); @@ -198,14 +211,12 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup return false; } - String checkBackupFileLocation(String fullBackupFileName) throws IOException { + String checkBackupFileLocation(String fullBackupFileName, String errorMessage) throws IOException { final File backupFile = new File(fullBackupFileName); final String backupFileLocation = backupFile.getCanonicalPath(); // Ensure that storageLocation correspond to an authorised area if (!backupFileLocation.startsWith(adminStorageLocation)) { - throw new IOException("Backup file is outside of storage file system"); - } else if (!backupFile.exists()) { - throw new FileSystemNotFoundException("Backup file does not exist"); + throw new IOException(errorMessage); } return backupFileLocation; } From aae20b70fb3ab658c4b24b719b6f6b9522360d49 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 3 Nov 2023 14:32:18 +0100 Subject: [PATCH 47/64] Align methods visibility in `AdminService` --- src/main/java/com/iexec/sms/admin/AdminService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index a9007148..d5319dac 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -184,7 +184,7 @@ boolean restoreDatabaseFromBackupFile(String storageLocation, String backupFileN * @param backupFileName The name of the backup file. * @return {@code true} if the deletion was successful, {@code false} if any error occurs. */ - public boolean deleteBackupFileFromStorage(String storageLocation, String backupFileName) { + boolean deleteBackupFileFromStorage(String storageLocation, String backupFileName) { try { // Ensure that storageLocation and backupFileName are not blanks boolean validation = checkCommonParameters(storageLocation, backupFileName); From 61eb3794ce0124159685f60c83a9999057048c4f Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 3 Nov 2023 16:14:45 +0100 Subject: [PATCH 48/64] Create backups in admin storage `work` sub-folder --- CHANGELOG.md | 2 +- .../java/com/iexec/sms/admin/AdminController.java | 9 ++++++--- .../com/iexec/sms/admin/AdminControllerTests.java | 12 ++++++------ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 058696ae..b859f078 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ All notable changes to this project will be documented in this file. - Add a security filter to activate an API Key mechanism on endpoints. (#207) - Create admin endpoints foundation. (#208 #209) - Add H2 database connection informations and storage ID decoding method. (#210) -- Add the ability to trigger a backup via a dedicated endpoint. (#211) +- Add the ability to trigger a backup via a dedicated endpoint. (#211, #215) - Add the ability to trigger a database restore via a dedicated endpoint. (#212) - Add the ability to trigger a delete via a dedicated endpoint. (#213) - Add the ability to trigger a backup replication via a dedicated endpoint. (#214) diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index c02dd626..27242609 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -18,6 +18,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; @@ -57,9 +58,11 @@ private enum BackupAction { private final ReentrantLock rLock = new ReentrantLock(true); private final AdminService adminService; + private final String adminStorageLocation; - public AdminController(AdminService adminService) { + public AdminController(AdminService adminService, @Value("${admin.storage-location}") String adminStorageLocation) { this.adminService = adminService; + this.adminStorageLocation = adminStorageLocation; } /** @@ -169,7 +172,7 @@ private ResponseEntity performOperation(String storageID, String fileName, switch (operationType) { case BACKUP: - operationSuccessful = adminService.createDatabaseBackupFile(BACKUP_STORAGE_LOCATION, BACKUP_FILENAME); + operationSuccessful = adminService.createDatabaseBackupFile(adminStorageLocation + BACKUP_STORAGE_LOCATION, BACKUP_FILENAME); break; case RESTORE: operationSuccessful = adminService.restoreDatabaseFromBackupFile(storagePath, fileName); @@ -179,7 +182,7 @@ private ResponseEntity performOperation(String storageID, String fileName, break; case REPLICATE: operationSuccessful = adminService.replicateDatabaseBackupFile( - BACKUP_STORAGE_LOCATION, BACKUP_FILENAME, storagePath, fileName); + adminStorageLocation + BACKUP_STORAGE_LOCATION, BACKUP_FILENAME, storagePath, fileName); break; default: break; diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index ffd9a23b..7cd1e7ad 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -89,7 +89,7 @@ public boolean createDatabaseBackupFile(String storageLocation, String backupFil } return true; } - }); + }, ""); final List> responses = Collections.synchronizedList(new ArrayList<>(3)); @@ -115,9 +115,9 @@ public boolean createDatabaseBackupFile(String storageLocation, String backupFil // region replicate-backup @Test void testReplicate(@TempDir Path tempDir) { - // Test to change when the methode replicateDatabaseBackupFile will be implemented final String storageID = convertToHex(tempDir.toString()); - when(adminService.replicateDatabaseBackupFile("/work/", FILE_NAME, tempDir.toString(), FILE_NAME)).thenReturn(true); + ReflectionTestUtils.setField(adminController, "adminStorageLocation", tempDir.toString()); + when(adminService.replicateDatabaseBackupFile(tempDir + "/work/", FILE_NAME, tempDir.toString(), FILE_NAME)).thenReturn(true); assertEquals(HttpStatus.OK, adminController.replicateBackup(storageID, FILE_NAME).getStatusCode()); } @@ -160,7 +160,7 @@ public boolean replicateDatabaseBackupFile(String backupStoragePath, String back } return true; } - }); + }, ""); final List> responses = Collections.synchronizedList(new ArrayList<>(3)); final String storageID = convertToHex(tempDir.toString()); @@ -232,7 +232,7 @@ public boolean restoreDatabaseFromBackupFile(String storageId, String fileName) } return true; } - }); + }, ""); final List> responses = Collections.synchronizedList(new ArrayList<>(3)); final String storageID = convertToHex(tempDir.toString()); @@ -327,7 +327,7 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup } return true; } - }); + }, ""); final List> responses = Collections.synchronizedList(new ArrayList<>(3)); final String storageID = convertToHex(tempDir.toString()); From 0f0211cba48090bf97ba3697f8fdc738a4a4e938 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Mon, 13 Nov 2023 13:26:31 +0100 Subject: [PATCH 49/64] Use `jenkins-library` 2.7.4 --- CHANGELOG.md | 4 ++++ Jenkinsfile | 14 ++++---------- build.gradle | 25 ++++++++++--------------- iexec-sms-library/build.gradle | 2 ++ 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b859f078..e7cf538a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ All notable changes to this project will be documented in this file. - Add the ability to trigger a delete via a dedicated endpoint. (#213) - Add the ability to trigger a backup replication via a dedicated endpoint. (#214) +### Dependency Upgrades + +- Upgrade to `jenkins-library` 2.7.4. (#216) + ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 ### Bug Fixes diff --git a/Jenkinsfile b/Jenkinsfile index 18ad1ded..0b96b4da 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,8 +1,10 @@ -@Library('global-jenkins-library@2.7.3') _ +@Library('global-jenkins-library@2.7.4') _ String repositoryName = 'iexec-sms' -buildInfo = getBuildInfo() +buildInfo = buildJavaProject( + shouldPublishJars: true, + shouldPublishDockerImages: true) // add parameters for non-PR builds when branch is not develop or production branch boolean addParameters = !buildInfo.isPullRequestBuild && !buildInfo.isDevelopBranch && !buildInfo.isProductionBranch @@ -15,14 +17,6 @@ if (addParameters) { ]) } -buildJavaProject( - buildInfo: buildInfo, - integrationTestsEnvVars: [], - shouldPublishJars: true, - shouldPublishDockerImages: true, - dockerfileDir: '.', - buildContext: '.') - // BUILD_TEE parameter only exists if addParameters is true // If BUILD_TEE is false, TEE builds won't be executed and we return here if (addParameters && !params.BUILD_TEE) { diff --git a/build.gradle b/build.gradle index 5c76c9a7..ae2c3006 100644 --- a/build.gradle +++ b/build.gradle @@ -37,8 +37,10 @@ allprojects { } java { toolchain { - languageVersion.set(JavaLanguageVersion.of(11)) + languageVersion.set(JavaLanguageVersion.of(17)) } + sourceCompatibility = "11" + targetCompatibility = "11" } } @@ -145,28 +147,21 @@ publishing { } } -ext.jarPathForOCI = relativePath(tasks.bootJar.outputs.files.singleFile) +ext.jarPathForOCI = relativePath(tasks.bootJar.outputs.files.singleFile) ext.gitShortCommit = 'git rev-parse --short=8 HEAD'.execute().text.trim() -ext.ociImageName = 'local/' + ['bash', '-c', 'basename $(git config --get remote.origin.url) .git'].execute().text.trim() +ext.ociImageName = 'local/' + ['bash', '-c', 'basename $(git config --get remote.origin.url) .git'].execute().text.trim() tasks.register('buildImage', Exec) { - group 'Build' + group 'Build' description 'Builds an OCI image from a Dockerfile.' - dependsOn bootJar - commandLine ('sh', '-c', "docker build --build-arg jar=$jarPathForOCI" - + " -t $ociImageName:$gitShortCommit . && docker tag $ociImageName:$gitShortCommit $ociImageName:dev") - standardOutput = new ByteArrayOutputStream() - - ext.output = { - println standardOutput - return standardOutput.toString() - } + dependsOn bootJar + commandLine 'docker', 'build', '--build-arg', 'jar=' + jarPathForOCI, '-t', ociImageName + ':dev', '.' } tasks.register('buildSconeImage', Exec) { - group "Build" + group "Build" description "Build an OCI image compatible with scontain TEE framework" - dependsOn buildImage + dependsOn buildImage commandLine "docker/sconify.sh" environment "IMG_FROM", "$ociImageName:dev" environment "IMG_TO", "$ociImageName-unlocked:dev" diff --git a/iexec-sms-library/build.gradle b/iexec-sms-library/build.gradle index e66849c5..ffa99122 100644 --- a/iexec-sms-library/build.gradle +++ b/iexec-sms-library/build.gradle @@ -15,6 +15,8 @@ dependencies { } java { + sourceCompatibility = "11" + targetCompatibility = "11" withJavadocJar() withSourcesJar() } From 7bc7d313ef3ef87ee03f71744ddb6f1b81cf0fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Tue, 21 Nov 2023 11:42:38 +0100 Subject: [PATCH 50/64] Add the ability to trigger a backup copy via a dedicated endpoint --- CHANGELOG.md | 1 + .../com/iexec/sms/admin/AdminController.java | 61 ++++++++++---- .../com/iexec/sms/admin/AdminService.java | 58 ++++++++++++- .../iexec/sms/admin/AdminControllerTests.java | 82 +++++++++++++++++++ .../iexec/sms/admin/AdminServiceTests.java | 44 ++++++++++ 5 files changed, 227 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7cf538a..1a7abe7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file. - Add the ability to trigger a database restore via a dedicated endpoint. (#212) - Add the ability to trigger a delete via a dedicated endpoint. (#213) - Add the ability to trigger a backup replication via a dedicated endpoint. (#214) +- Add the ability to trigger a backup copy via a dedicated endpoint. (#217) ### Dependency Upgrades diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index 27242609..66b2d48d 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -35,10 +35,10 @@ public class AdminController { /** - * Enum representing different types of backup operations: BACKUP, DELETE, REPLICATE and RESTORE. + * Enum representing different types of backup operations: BACKUP, COPY,DELETE, REPLICATE and RESTORE. */ private enum BackupAction { - BACKUP, DELETE, REPLICATE, RESTORE; + BACKUP, COPY, DELETE, REPLICATE, RESTORE; } /** @@ -81,7 +81,7 @@ public AdminController(AdminService adminService, @Value("${admin.storage-locati */ @PostMapping("/backup") public ResponseEntity createBackup() { - return performOperation("", "", BackupAction.BACKUP); + return performOperation(StringUtils.EMPTY, StringUtils.EMPTY, StringUtils.EMPTY, StringUtils.EMPTY, BackupAction.BACKUP); } /** @@ -102,7 +102,7 @@ public ResponseEntity createBackup() { */ @PostMapping("/{storageID}/replicate-backup") ResponseEntity replicateBackup(@PathVariable String storageID, @RequestParam String fileName) { - return performOperation(storageID, fileName, BackupAction.REPLICATE); + return performOperation(storageID, fileName, StringUtils.EMPTY, StringUtils.EMPTY, BackupAction.REPLICATE); } /** @@ -124,7 +124,7 @@ ResponseEntity replicateBackup(@PathVariable String storageID, @RequestPar */ @PostMapping("/{storageID}/restore-backup") ResponseEntity restoreBackup(@PathVariable String storageID, @RequestParam String fileName) { - return performOperation(storageID, fileName, BackupAction.RESTORE); + return performOperation(storageID, fileName, StringUtils.EMPTY, StringUtils.EMPTY, BackupAction.RESTORE); } /** @@ -146,27 +146,53 @@ ResponseEntity restoreBackup(@PathVariable String storageID, @RequestParam */ @DeleteMapping("/{storageID}/delete-backup") ResponseEntity deleteBackup(@PathVariable String storageID, @RequestParam String fileName) { - return performOperation(storageID, fileName, BackupAction.DELETE); + return performOperation(storageID, fileName, StringUtils.EMPTY, StringUtils.EMPTY, BackupAction.DELETE); + } + + /** + * Endpoint to copy a database backup. + *

+ * This method allows the copy of the backup toward another storage. + * + * @param sourceStorageID The unique identifier for the source storage location of the dump in hexadecimal. + * @param sourceFileName The name of the source file to copy from the source storage location. + * @param destinationStorageID The unique identifier for the destination storage location of the dump in hexadecimal. + * @param destinationFileName The name of the destination file, can be empty. + * @return A response entity indicating the status and details of the copy operation. + *

    + *
  • HTTP 200 (OK) - If the copy has been successfully replicated. + *
  • HTTP 400 (Bad Request) - If {@code sourceFileName} is missing or {@code sourceStorageID} or {@code destinationStorageID} does not match an existing directory. + *
  • HTTP 404 (Not Found) - If the backup file specified by {@code fileName} does not exist. + *
  • HTTP 429 (Too Many Requests) - If another operation (backup/restore/delete/replicate/copy) is already in progress. + *
  • HTTP 500 (Internal Server Error) - If an unexpected error occurs during the copy process. + *
+ */ + @PostMapping("/{sourceStorageID}/copy-to/{destinationStorageID}") + ResponseEntity copyBackup(@PathVariable String sourceStorageID, @PathVariable String destinationStorageID, @RequestParam String sourceFileName, @RequestParam(required = false) String destinationFileName) { + return performOperation(sourceStorageID, sourceFileName, destinationStorageID, destinationFileName, BackupAction.COPY); } /** * Common method for database backup operations. * - * @param storageID The unique identifier for the storage location of the dump in hexadecimal. - * @param fileName The name of the dump file to be operated on. - * @param operationType The type of operation {{@link BackupAction}. + * @param sourceStorageID The unique identifier for the storage location of the dump in hexadecimal. + * @param sourceFileName The name of the dump file to be operated on. + * @param operationType The type of operation {{@link BackupAction}. * @return A response entity indicating the status and details of the operation. */ - private ResponseEntity performOperation(String storageID, String fileName, BackupAction operationType) { + private ResponseEntity performOperation(String sourceStorageID, String sourceFileName, String destinationStorageID, String destinationFileName, BackupAction operationType) { try { - if ((StringUtils.isBlank(storageID) || StringUtils.isBlank(fileName)) && operationType != BackupAction.BACKUP) { + if ((StringUtils.isBlank(sourceStorageID) || StringUtils.isBlank(sourceFileName)) && operationType != BackupAction.BACKUP) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); + } + if (StringUtils.isBlank(destinationStorageID) && operationType == BackupAction.COPY) { return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); } if (!tryToAcquireLock()) { return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); } - final String storagePath = getStoragePathFromID(storageID); + final String sourceStoragePath = getStoragePathFromID(sourceStorageID); boolean operationSuccessful = false; @@ -175,14 +201,19 @@ private ResponseEntity performOperation(String storageID, String fileName, operationSuccessful = adminService.createDatabaseBackupFile(adminStorageLocation + BACKUP_STORAGE_LOCATION, BACKUP_FILENAME); break; case RESTORE: - operationSuccessful = adminService.restoreDatabaseFromBackupFile(storagePath, fileName); + operationSuccessful = adminService.restoreDatabaseFromBackupFile(sourceStoragePath, sourceFileName); break; case DELETE: - operationSuccessful = adminService.deleteBackupFileFromStorage(storagePath, fileName); + operationSuccessful = adminService.deleteBackupFileFromStorage(sourceStoragePath, sourceFileName); break; case REPLICATE: operationSuccessful = adminService.replicateDatabaseBackupFile( - adminStorageLocation + BACKUP_STORAGE_LOCATION, BACKUP_FILENAME, storagePath, fileName); + adminStorageLocation + BACKUP_STORAGE_LOCATION, BACKUP_FILENAME, sourceStoragePath, sourceFileName); + break; + case COPY: + final String destinationStoragePath = getStoragePathFromID(destinationStorageID); + operationSuccessful = adminService.copyBackupFileFromStorageToStorage( + sourceStoragePath, sourceFileName, destinationStoragePath, destinationFileName); break; default: break; diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index d5319dac..9f633f75 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -41,6 +41,10 @@ public class AdminService { // Used to print formatted date in log private final DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); + + public static final String ERR_BACKUP_FILE_OUTSIDE_STORAGE = "Backup file is outside of storage file system"; + public static final String ERR_BACKUP_FILE_NOT_EXIST = "Backup file does not exist"; + public static final String ERR_FILE_ALREADY_EXIST = "A file already exists at the destination"; private final String datasourceUrl; private final String datasourceUsername; private final String datasourcePassword; @@ -156,9 +160,9 @@ boolean restoreDatabaseFromBackupFile(String storageLocation, String backupFileN try { final String backupFileLocation = checkBackupFileLocation( storageLocation + File.separator + backupFileName, - "Backup file is outside of storage file system"); + ERR_BACKUP_FILE_OUTSIDE_STORAGE); if (!Path.of(backupFileLocation).toFile().exists()) { - throw new FileSystemNotFoundException("Backup file does not exist"); + throw new FileSystemNotFoundException(ERR_BACKUP_FILE_NOT_EXIST); } final long size = backupFileLocation.length(); log.info("Starting the restore process [backupFileLocation:{}]", backupFileLocation); @@ -193,10 +197,10 @@ boolean deleteBackupFileFromStorage(String storageLocation, String backupFileNam } final String backupFileLocation = checkBackupFileLocation( storageLocation + File.separator + backupFileName, - "Backup file is outside of storage file system"); + ERR_BACKUP_FILE_OUTSIDE_STORAGE); final Path backupFileLocationPath = Path.of(backupFileLocation); if (!backupFileLocationPath.toFile().exists()) { - throw new FileSystemNotFoundException("Backup file does not exist"); + throw new FileSystemNotFoundException(ERR_BACKUP_FILE_NOT_EXIST); } log.info("Starting the delete process [backupFileLocation:{}]", backupFileLocation); final long start = System.currentTimeMillis(); @@ -211,6 +215,52 @@ boolean deleteBackupFileFromStorage(String storageLocation, String backupFileNam return false; } + /** + * Copy a backup from a location to another with the possibility of renaming the file + * + * @param sourceStorageLocation The location of the source backup file. + * @param sourceBackupFileName The name of the source backup file. + * @param destinationStorageLocation The location of destination the backup file. + * @param destinationBackupFileName The name of the destination backup file. + * @return {@code true} if the copy was successful, {@code false} if any error occurs. + */ + boolean copyBackupFileFromStorageToStorage(String sourceStorageLocation, String sourceBackupFileName, String destinationStorageLocation, String destinationBackupFileName) { + try { + // Check that we want to copy an authorised file + final Path sourceBackupFileLocation = Path.of(checkBackupFileLocation( + sourceStorageLocation + File.separator + sourceBackupFileName, + ERR_BACKUP_FILE_OUTSIDE_STORAGE)); + // File must exist + if (!sourceBackupFileLocation.toFile().exists()) { + throw new FileSystemNotFoundException(ERR_BACKUP_FILE_NOT_EXIST); + } + // No renaming, keep the original name + if (StringUtils.isBlank(destinationBackupFileName)) { + destinationBackupFileName = sourceBackupFileName; + } + // Check that we want to copy into authorized location + final Path destinationBackupFileLocation = Path.of(checkBackupFileLocation( + destinationStorageLocation + File.separator + destinationBackupFileName, + ERR_BACKUP_FILE_OUTSIDE_STORAGE)); + + // Check that we are not trying to overwrite a file + if (destinationBackupFileLocation.toFile().exists()) { + throw new IOException(ERR_FILE_ALREADY_EXIST); + } + + log.info("Starting the copy process [sourceBackupFileLocation:{}, destinationBackupFileLocation:{}]", sourceBackupFileLocation, destinationBackupFileLocation); + final long start = System.currentTimeMillis(); + Files.copy(sourceBackupFileLocation, destinationBackupFileLocation, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + final long stop = System.currentTimeMillis(); + log.info("Backup has been copied [sourceBackupFileLocation:{}, destinationBackupFileLocation:{}, timestamp:{}, duration:{} ms]", + sourceBackupFileLocation, destinationBackupFileLocation, dateFormat.format(start), stop - start); + return true; + } catch (IOException e) { + log.error("An error occurred while copying backup", e); + } + return false; + } + String checkBackupFileLocation(String fullBackupFileName, String errorMessage) throws IOException { final File backupFile = new File(fullBackupFileName); final String backupFileLocation = backupFile.getCanonicalPath(); diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index 7cd1e7ad..9e081d9f 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -351,6 +351,80 @@ public boolean deleteBackupFileFromStorage(String storageLocation, String backup } // endregion + // region replicate-backup + @Test + void testCopy(@TempDir Path tempDir) { + final String sourceStorageID = convertToHex(tempDir.toString()); + + ReflectionTestUtils.setField(adminController, "adminStorageLocation", tempDir.toString()); + when(adminService.copyBackupFileFromStorageToStorage(tempDir.toString(), FILE_NAME, tempDir.toString(), "backup2.sql")).thenReturn(true); + assertEquals(HttpStatus.OK, adminController.copyBackup(sourceStorageID, sourceStorageID, FILE_NAME, "backup2.sql").getStatusCode()); + } + + @Test + void testInterruptedThreadOnCopy() throws InterruptedException { + ReflectionTestUtils.setField(adminController, "rLock", rLock); + when(rLock.tryLock(100, TimeUnit.MILLISECONDS)).thenThrow(InterruptedException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.copyBackup(STORAGE_PATH, STORAGE_PATH, FILE_NAME, FILE_NAME).getStatusCode()); + } + + @Test + void testInternalServerErrorOnCopy(@TempDir Path tempDir) { + final String sourceStorageID = convertToHex(tempDir.toString()); + when(adminService.copyBackupFileFromStorageToStorage(tempDir.toString(), FILE_NAME, tempDir.toString(), "backup2.sql")).thenThrow(RuntimeException.class); + assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.copyBackup(sourceStorageID, sourceStorageID, FILE_NAME, "backup2.sql").getStatusCode()); + } + + @Test + void testTooManyRequestOnCopy(@TempDir Path tempDir) throws InterruptedException { + + AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { + @Override + public boolean copyBackupFileFromStorageToStorage(String sourceStorageLocation, String sourceBackupFileName, String destinationStorageLocation, String destinationBackupFileName) { + try { + log.info("Long copyBackupFileFromStorageToStorage action is running ..."); + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return true; + } + }, ""); + + final List> responses = Collections.synchronizedList(new ArrayList<>(3)); + final String sourceStorageID = convertToHex(tempDir.toString()); + + Thread firstThread = new Thread(() -> responses.add(adminControllerWithLongAction.copyBackup(sourceStorageID, sourceStorageID, FILE_NAME, "backup2.sql"))); + Thread secondThread = new Thread(() -> responses.add(adminControllerWithLongAction.copyBackup(sourceStorageID, sourceStorageID, FILE_NAME, "backup2.sql"))); + + firstThread.start(); + secondThread.start(); + responses.add(adminControllerWithLongAction.copyBackup(sourceStorageID, sourceStorageID, FILE_NAME, "backup2.sql")); + + secondThread.join(); + firstThread.join(); + + + long code200 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.OK).count(); + long code429 = responses.stream().filter(element -> element.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS).count(); + + assertEquals(1, code200); + assertEquals(2, code429); + } + + @ParameterizedTest + @MethodSource("provideBadRequestParameters") + void testBadRequestOnCopyOnCommonsAdminControls(String storageID, String fileName) { + assertEquals(HttpStatus.BAD_REQUEST, adminController.copyBackup(storageID, "NOT_CONCERNED", fileName, "NOT_CONCERNED").getStatusCode()); + } + + @ParameterizedTest + @MethodSource("provideBadRequestParametersForCopy") + void testBadRequestOnCopyOnSpecificsCopyControls(String sourceStorageID, String sourceFileName, String destinationStorageID) { + assertEquals(HttpStatus.BAD_REQUEST, adminController.copyBackup(sourceStorageID, destinationStorageID, sourceFileName, "NOT_CONCERNED").getStatusCode()); + } + // endregion + private static Stream provideBadRequestParameters() { return Stream.of( Arguments.of(null, null), @@ -360,4 +434,12 @@ private static Stream provideBadRequestParameters() { Arguments.of(" ", FILE_NAME) ); } + + private static Stream provideBadRequestParametersForCopy() { + return Stream.of( + Arguments.of(STORAGE_PATH, FILE_NAME, ""), + Arguments.of(STORAGE_PATH, FILE_NAME, " "), + Arguments.of(STORAGE_PATH, FILE_NAME, null) + ); + } } diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 5d2543fd..9387700b 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -232,6 +232,50 @@ void shouldFailToDeleteWhenInvalidParameters() { } // endregion + //region copy-backup + + @Test + void shouldCopy() { + final String validStorageLocation = tempStorageLocation.getPath(); + final String validBackupFileName = "backup.sql"; + adminService.createDatabaseBackupFile(validStorageLocation, validBackupFileName); + assertTrue(adminService.copyBackupFileFromStorageToStorage(validStorageLocation, validBackupFileName, validStorageLocation, "backup2.sql")); + assertTrue(new File(validStorageLocation + File.separator + "backup2.sql").exists()); + } + + @Test + void shouldFailToCopyWhenInvalidParameters(CapturedOutput output) { + final String validStorageLocation = tempStorageLocation.getPath(); + final String validBackupFileName = "backup.sql"; + final String emptyStorageLocation = ""; + adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), validBackupFileName); + + assertAll( + + () -> assertAll( + () -> assertFalse(adminService.copyBackupFileFromStorageToStorage(validStorageLocation, validBackupFileName, validStorageLocation, validBackupFileName)), + () -> assertTrue(output.getOut().contains(AdminService.ERR_FILE_ALREADY_EXIST)) + ), + + () -> assertAll( + () -> assertFalse(adminService.copyBackupFileFromStorageToStorage(validStorageLocation, validBackupFileName, "", "")), + () -> assertTrue(output.getOut().contains(AdminService.ERR_BACKUP_FILE_OUTSIDE_STORAGE)) + ), + + () -> assertAll( + () -> assertFalse(adminService.copyBackupFileFromStorageToStorage(emptyStorageLocation, validBackupFileName, "", "")), + () -> assertTrue(output.getOut().contains(AdminService.ERR_BACKUP_FILE_OUTSIDE_STORAGE)) + ), + + () -> assertThrows( + FileSystemNotFoundException.class, + () -> adminService.copyBackupFileFromStorageToStorage(validStorageLocation, "backup2.sql", "", ""), + AdminService.ERR_BACKUP_FILE_NOT_EXIST + ) + ); + } + // endregion + //region utils @Test From fa373598a34aa070fe7d632e3ffd51375aae6e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Tue, 21 Nov 2023 18:14:26 +0100 Subject: [PATCH 51/64] Refactor to manage copying and replicating as a single operation in the service layer --- .../com/iexec/sms/admin/AdminController.java | 37 ++++-- .../com/iexec/sms/admin/AdminService.java | 52 ++------ .../iexec/sms/admin/AdminControllerTests.java | 33 +++-- .../iexec/sms/admin/AdminServiceTests.java | 123 +++++++----------- 4 files changed, 108 insertions(+), 137 deletions(-) diff --git a/src/main/java/com/iexec/sms/admin/AdminController.java b/src/main/java/com/iexec/sms/admin/AdminController.java index 66b2d48d..977e31b0 100644 --- a/src/main/java/com/iexec/sms/admin/AdminController.java +++ b/src/main/java/com/iexec/sms/admin/AdminController.java @@ -102,7 +102,7 @@ public ResponseEntity createBackup() { */ @PostMapping("/{storageID}/replicate-backup") ResponseEntity replicateBackup(@PathVariable String storageID, @RequestParam String fileName) { - return performOperation(storageID, fileName, StringUtils.EMPTY, StringUtils.EMPTY, BackupAction.REPLICATE); + return performOperation(StringUtils.EMPTY, StringUtils.EMPTY, storageID, fileName, BackupAction.REPLICATE); } /** @@ -169,30 +169,32 @@ ResponseEntity deleteBackup(@PathVariable String storageID, @RequestParam */ @PostMapping("/{sourceStorageID}/copy-to/{destinationStorageID}") ResponseEntity copyBackup(@PathVariable String sourceStorageID, @PathVariable String destinationStorageID, @RequestParam String sourceFileName, @RequestParam(required = false) String destinationFileName) { + destinationFileName = StringUtils.isNotBlank(destinationFileName) ? destinationFileName : sourceFileName; return performOperation(sourceStorageID, sourceFileName, destinationStorageID, destinationFileName, BackupAction.COPY); } /** * Common method for database backup operations. * - * @param sourceStorageID The unique identifier for the storage location of the dump in hexadecimal. - * @param sourceFileName The name of the dump file to be operated on. - * @param operationType The type of operation {{@link BackupAction}. + * @param sourceStorageID The unique identifier for the storage location of the dump in hexadecimal. + * @param sourceFileName The name of the dump file to be operated on. + * @param destinationStorageID The unique identifier for the destination storage location of the dump in hexadecimal. + * @param destinationFileName The name of the destination file, can be empty. + * @param operationType The type of operation {{@link BackupAction}. * @return A response entity indicating the status and details of the operation. */ private ResponseEntity performOperation(String sourceStorageID, String sourceFileName, String destinationStorageID, String destinationFileName, BackupAction operationType) { try { - if ((StringUtils.isBlank(sourceStorageID) || StringUtils.isBlank(sourceFileName)) && operationType != BackupAction.BACKUP) { - return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); - } - if (StringUtils.isBlank(destinationStorageID) && operationType == BackupAction.COPY) { + if (invalidSource(sourceStorageID, sourceFileName, operationType) || invalidDestination(destinationStorageID, destinationFileName, operationType)) { return ResponseEntity.status(HttpStatus.BAD_REQUEST).build(); } + if (!tryToAcquireLock()) { return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); } final String sourceStoragePath = getStoragePathFromID(sourceStorageID); + String destinationStoragePath = ""; boolean operationSuccessful = false; @@ -207,12 +209,13 @@ private ResponseEntity performOperation(String sourceStorageID, String sou operationSuccessful = adminService.deleteBackupFileFromStorage(sourceStoragePath, sourceFileName); break; case REPLICATE: - operationSuccessful = adminService.replicateDatabaseBackupFile( - adminStorageLocation + BACKUP_STORAGE_LOCATION, BACKUP_FILENAME, sourceStoragePath, sourceFileName); + destinationStoragePath = getStoragePathFromID(destinationStorageID); + operationSuccessful = adminService.copyBackupFile( + adminStorageLocation + BACKUP_STORAGE_LOCATION, BACKUP_FILENAME, destinationStoragePath, destinationFileName); break; case COPY: - final String destinationStoragePath = getStoragePathFromID(destinationStorageID); - operationSuccessful = adminService.copyBackupFileFromStorageToStorage( + destinationStoragePath = getStoragePathFromID(destinationStorageID); + operationSuccessful = adminService.copyBackupFile( sourceStoragePath, sourceFileName, destinationStoragePath, destinationFileName); break; default: @@ -258,6 +261,15 @@ String getStoragePathFromID(String storageID) { return output; } + private boolean invalidSource(String sourceStorageID, String sourceFileName, BackupAction operationType) { + return (StringUtils.isBlank(sourceStorageID) || StringUtils.isBlank(sourceFileName)) + && operationType != BackupAction.REPLICATE && operationType != BackupAction.BACKUP; + } + + private boolean invalidDestination(String destinationStorageID, String destinationFileName, BackupAction operationType) { + return (StringUtils.isBlank(destinationStorageID) || StringUtils.isBlank(destinationFileName)) && (operationType == BackupAction.COPY || operationType == BackupAction.REPLICATE); + } + private boolean tryToAcquireLock() throws InterruptedException { return rLock.tryLock(100, TimeUnit.MILLISECONDS); } @@ -267,5 +279,4 @@ private void tryToReleaseLock() { rLock.unlock(); } } - } diff --git a/src/main/java/com/iexec/sms/admin/AdminService.java b/src/main/java/com/iexec/sms/admin/AdminService.java index 9f633f75..6c5c5492 100644 --- a/src/main/java/com/iexec/sms/admin/AdminService.java +++ b/src/main/java/com/iexec/sms/admin/AdminService.java @@ -44,6 +44,7 @@ public class AdminService { public static final String ERR_BACKUP_FILE_OUTSIDE_STORAGE = "Backup file is outside of storage file system"; public static final String ERR_BACKUP_FILE_NOT_EXIST = "Backup file does not exist"; + public static final String ERR_REPLICATE_OR_COPY_FILE_OUTSIDE_STORAGE = "Replicated or Copied backup file destination is outside of storage file system"; public static final String ERR_FILE_ALREADY_EXIST = "A file already exists at the destination"; private final String datasourceUrl; private final String datasourceUsername; @@ -116,37 +117,6 @@ boolean databaseDump(String fullBackupFileName) { return true; } - /** - * Replicates a backup to the persistent storage. - *

- * The {@code backupStorageLocation/backupFileName} is replicated to {@code replicateStorageLocation/replicateFileName}. - * - * @param backupStorageLocation Location of backup to replicate - * @param backupFileName Name of backup file to replicate - * @param replicateStorageLocation Location of replicated backup - * @param replicateFileName Name of replicated backup file - * @return {@code true} if the replication was successful, {@code false} if any error occurs. - */ - boolean replicateDatabaseBackupFile(String backupStorageLocation, String backupFileName, String replicateStorageLocation, String replicateFileName) { - try { - final Path fullBackupFilePath = Path.of(backupStorageLocation, backupFileName).toRealPath(); - final String backupFileLocation = checkBackupFileLocation( - replicateStorageLocation + File.separator + replicateFileName, - "Replicated backup file destination is outside of storage file system"); - final long size = fullBackupFilePath.toFile().length(); - log.info("Starting the replicate process [backupFileLocation:{}]", backupFileLocation); - final long start = System.currentTimeMillis(); - Files.copy(fullBackupFilePath, Path.of(backupFileLocation), StandardCopyOption.COPY_ATTRIBUTES); - final long stop = System.currentTimeMillis(); - log.info("Backup has been replicated [backupFileLocation:{}, timestamp:{}, duration:{} ms, size:{}]", - backupFileLocation, dateFormat.format(start), stop - start, size); - return true; - } catch (IOException e) { - log.error("Error occurred during copy", e); - } - return false; - } - /** * Restores a backup from provided inputs. *

@@ -217,6 +187,8 @@ boolean deleteBackupFileFromStorage(String storageLocation, String backupFileNam /** * Copy a backup from a location to another with the possibility of renaming the file + *

+ * The {@code sourceStorageLocation/sourceBackupFileName} is replicated to {@code destinationStorageLocation/destinationBackupFileName}. * * @param sourceStorageLocation The location of the source backup file. * @param sourceBackupFileName The name of the source backup file. @@ -224,36 +196,34 @@ boolean deleteBackupFileFromStorage(String storageLocation, String backupFileNam * @param destinationBackupFileName The name of the destination backup file. * @return {@code true} if the copy was successful, {@code false} if any error occurs. */ - boolean copyBackupFileFromStorageToStorage(String sourceStorageLocation, String sourceBackupFileName, String destinationStorageLocation, String destinationBackupFileName) { + boolean copyBackupFile(String sourceStorageLocation, String sourceBackupFileName, String destinationStorageLocation, String destinationBackupFileName) { try { // Check that we want to copy an authorised file final Path sourceBackupFileLocation = Path.of(checkBackupFileLocation( sourceStorageLocation + File.separator + sourceBackupFileName, ERR_BACKUP_FILE_OUTSIDE_STORAGE)); + // File must exist if (!sourceBackupFileLocation.toFile().exists()) { throw new FileSystemNotFoundException(ERR_BACKUP_FILE_NOT_EXIST); } - // No renaming, keep the original name - if (StringUtils.isBlank(destinationBackupFileName)) { - destinationBackupFileName = sourceBackupFileName; - } + // Check that we want to copy into authorized location final Path destinationBackupFileLocation = Path.of(checkBackupFileLocation( destinationStorageLocation + File.separator + destinationBackupFileName, - ERR_BACKUP_FILE_OUTSIDE_STORAGE)); + ERR_REPLICATE_OR_COPY_FILE_OUTSIDE_STORAGE)); // Check that we are not trying to overwrite a file if (destinationBackupFileLocation.toFile().exists()) { throw new IOException(ERR_FILE_ALREADY_EXIST); } - + final long size = sourceBackupFileLocation.toFile().length(); log.info("Starting the copy process [sourceBackupFileLocation:{}, destinationBackupFileLocation:{}]", sourceBackupFileLocation, destinationBackupFileLocation); final long start = System.currentTimeMillis(); - Files.copy(sourceBackupFileLocation, destinationBackupFileLocation, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + Files.copy(sourceBackupFileLocation, destinationBackupFileLocation, StandardCopyOption.COPY_ATTRIBUTES); final long stop = System.currentTimeMillis(); - log.info("Backup has been copied [sourceBackupFileLocation:{}, destinationBackupFileLocation:{}, timestamp:{}, duration:{} ms]", - sourceBackupFileLocation, destinationBackupFileLocation, dateFormat.format(start), stop - start); + log.info("Backup has been copied [sourceBackupFileLocation:{}, destinationBackupFileLocation:{}, timestamp:{}, duration:{} ms, size:{}]", + sourceBackupFileLocation, destinationBackupFileLocation, dateFormat.format(start), stop - start, size); return true; } catch (IOException e) { log.error("An error occurred while copying backup", e); diff --git a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java index 9e081d9f..c774a7e5 100644 --- a/src/test/java/com/iexec/sms/admin/AdminControllerTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminControllerTests.java @@ -31,8 +31,11 @@ import org.springframework.http.ResponseEntity; import org.springframework.test.util.ReflectionTestUtils; +import java.io.IOException; import java.nio.file.FileSystemNotFoundException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -117,7 +120,7 @@ public boolean createDatabaseBackupFile(String storageLocation, String backupFil void testReplicate(@TempDir Path tempDir) { final String storageID = convertToHex(tempDir.toString()); ReflectionTestUtils.setField(adminController, "adminStorageLocation", tempDir.toString()); - when(adminService.replicateDatabaseBackupFile(tempDir + "/work/", FILE_NAME, tempDir.toString(), FILE_NAME)).thenReturn(true); + when(adminService.copyBackupFile(tempDir + "/work/", FILE_NAME, tempDir.toString(), FILE_NAME)).thenReturn(true); assertEquals(HttpStatus.OK, adminController.replicateBackup(storageID, FILE_NAME).getStatusCode()); } @@ -129,7 +132,7 @@ void testBadRequestOnReplicate(String storageID, String fileName) { @Test void testInternalServerErrorOnReplicate() { - when(adminService.replicateDatabaseBackupFile(STORAGE_PATH, FILE_NAME, STORAGE_PATH, FILE_NAME)).thenThrow(RuntimeException.class); + when(adminService.copyBackupFile(STORAGE_PATH, FILE_NAME, STORAGE_PATH, FILE_NAME)).thenThrow(RuntimeException.class); assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.replicateBackup(STORAGE_PATH, FILE_NAME).getStatusCode()); } @@ -143,7 +146,7 @@ void testInterruptedThreadOnReplicate() throws InterruptedException { @Test void testNotFoundOnReplicate() { final String storageID = convertToHex(STORAGE_PATH); - when(adminService.replicateDatabaseBackupFile(STORAGE_PATH, FILE_NAME, STORAGE_PATH, FILE_NAME)).thenThrow(FileSystemNotFoundException.class); + when(adminService.copyBackupFile(STORAGE_PATH, FILE_NAME, STORAGE_PATH, FILE_NAME)).thenThrow(FileSystemNotFoundException.class); assertEquals(HttpStatus.NOT_FOUND, adminController.replicateBackup(storageID, FILE_NAME).getStatusCode()); } @@ -151,9 +154,9 @@ void testNotFoundOnReplicate() { void testTooManyRequestOnReplicate(@TempDir Path tempDir) throws InterruptedException { AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { @Override - public boolean replicateDatabaseBackupFile(String backupStoragePath, String backupFileName, String replicateStoragePath, String replicateFileName) { + public boolean copyBackupFile(String backupStoragePath, String backupFileName, String replicateStoragePath, String replicateFileName) { try { - log.info("Long replicateDatabaseBackupFile action is running ..."); + log.info("Long copyOrReplicateBackupFileFromStorageToStorage action is running ..."); Thread.sleep(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -357,10 +360,22 @@ void testCopy(@TempDir Path tempDir) { final String sourceStorageID = convertToHex(tempDir.toString()); ReflectionTestUtils.setField(adminController, "adminStorageLocation", tempDir.toString()); - when(adminService.copyBackupFileFromStorageToStorage(tempDir.toString(), FILE_NAME, tempDir.toString(), "backup2.sql")).thenReturn(true); + when(adminService.copyBackupFile(tempDir.toString(), FILE_NAME, tempDir.toString(), "backup2.sql")).thenReturn(true); assertEquals(HttpStatus.OK, adminController.copyBackup(sourceStorageID, sourceStorageID, FILE_NAME, "backup2.sql").getStatusCode()); } + @Test + void testCopyWithTheSameName(@TempDir Path tempDir) throws IOException { + final String sourceStorageID = convertToHex(tempDir.toString()); + final Path dailyDir = Paths.get(tempDir.toString(), "daily"); + final String dailyDirString = Files.createDirectories(dailyDir).toFile().getAbsolutePath(); + final String destinationStorageID = convertToHex(dailyDirString); + + ReflectionTestUtils.setField(adminController, "adminStorageLocation", tempDir.toString()); + when(adminService.copyBackupFile(tempDir.toString(), FILE_NAME, dailyDirString, FILE_NAME)).thenReturn(true); + assertEquals(HttpStatus.OK, adminController.copyBackup(sourceStorageID, destinationStorageID, FILE_NAME, "").getStatusCode()); + } + @Test void testInterruptedThreadOnCopy() throws InterruptedException { ReflectionTestUtils.setField(adminController, "rLock", rLock); @@ -371,7 +386,7 @@ void testInterruptedThreadOnCopy() throws InterruptedException { @Test void testInternalServerErrorOnCopy(@TempDir Path tempDir) { final String sourceStorageID = convertToHex(tempDir.toString()); - when(adminService.copyBackupFileFromStorageToStorage(tempDir.toString(), FILE_NAME, tempDir.toString(), "backup2.sql")).thenThrow(RuntimeException.class); + when(adminService.copyBackupFile(tempDir.toString(), FILE_NAME, tempDir.toString(), "backup2.sql")).thenThrow(RuntimeException.class); assertEquals(HttpStatus.INTERNAL_SERVER_ERROR, adminController.copyBackup(sourceStorageID, sourceStorageID, FILE_NAME, "backup2.sql").getStatusCode()); } @@ -380,9 +395,9 @@ void testTooManyRequestOnCopy(@TempDir Path tempDir) throws InterruptedException AdminController adminControllerWithLongAction = new AdminController(new AdminService("", "", "", "") { @Override - public boolean copyBackupFileFromStorageToStorage(String sourceStorageLocation, String sourceBackupFileName, String destinationStorageLocation, String destinationBackupFileName) { + public boolean copyBackupFile(String sourceStorageLocation, String sourceBackupFileName, String destinationStorageLocation, String destinationBackupFileName) { try { - log.info("Long copyBackupFileFromStorageToStorage action is running ..."); + log.info("Long copyOrReplicateBackupFileFromStorageToStorage action is running ..."); Thread.sleep(2000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java index 9387700b..da151948 100644 --- a/src/test/java/com/iexec/sms/admin/AdminServiceTests.java +++ b/src/test/java/com/iexec/sms/admin/AdminServiceTests.java @@ -102,52 +102,6 @@ void shouldReturnFalseWhenBackupFileNameDoesNotExist() { // endregion - // region replicate-backup - @Test - void shouldReplicateBackup() { - adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql"); - adminService.replicateDatabaseBackupFile( - tempStorageLocation.getPath(), "backup.sql", - tempStorageLocation.getPath(), "backup-copy.sql"); - assertTrue(Path.of(tempStorageLocation.getPath(), "backup-copy.sql").toFile().isFile()); - } - - @Test - void shouldFailToReplicateWithBackupFileMissing(CapturedOutput output) { - adminService.replicateDatabaseBackupFile( - tempStorageLocation.getPath(), "backup.sql", - tempStorageLocation.getPath(), "backup-copy.sql"); - assertAll( - () -> assertFalse(Path.of(tempStorageLocation.getPath(), "backup-copy.sql").toFile().isFile()), - () -> assertTrue(output.getOut().contains("NoSuchFileException")) - ); - } - - @Test - void shouldFailToReplicateWithInvalidStorageLocation(CapturedOutput output) { - adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql"); - boolean replicateStatus = adminService.replicateDatabaseBackupFile( - tempStorageLocation.getPath(), "backup.sql", - "/tmp/nonexistent/directory", "backup-copy.sql"); - assertAll( - () -> assertFalse(replicateStatus), - () -> assertTrue(output.getAll().contains("NoSuchFileException")) - ); - } - - @Test - void shouldFailToReplicateWithOutOfStorageLocation(CapturedOutput output) { - adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), "backup.sql"); - boolean replicateStatus = adminService.replicateDatabaseBackupFile( - tempStorageLocation.getPath(), "backup.sql", - "/opt", "backup-copy.sql"); - assertAll( - () -> assertFalse(replicateStatus), - () -> assertTrue(output.getAll().contains("Replicated backup file destination is outside of storage file system")) - ); - } - // endregion - // region restore-backup @Test void shouldRestoreBackup(CapturedOutput output) { @@ -232,52 +186,73 @@ void shouldFailToDeleteWhenInvalidParameters() { } // endregion - //region copy-backup - + // region copy-backup @Test void shouldCopy() { final String validStorageLocation = tempStorageLocation.getPath(); final String validBackupFileName = "backup.sql"; adminService.createDatabaseBackupFile(validStorageLocation, validBackupFileName); - assertTrue(adminService.copyBackupFileFromStorageToStorage(validStorageLocation, validBackupFileName, validStorageLocation, "backup2.sql")); - assertTrue(new File(validStorageLocation + File.separator + "backup2.sql").exists()); + assertTrue(adminService.copyBackupFile(validStorageLocation, validBackupFileName, validStorageLocation, "backup-copy.sql")); + assertTrue(new File(validStorageLocation + File.separator + "backup-copy.sql").exists()); } @Test - void shouldFailToCopyWhenInvalidParameters(CapturedOutput output) { + void shouldFailToCopyWhenDestinationFileAlreadyExist(CapturedOutput output) { final String validStorageLocation = tempStorageLocation.getPath(); final String validBackupFileName = "backup.sql"; - final String emptyStorageLocation = ""; adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), validBackupFileName); - assertAll( + assertFalse(adminService.copyBackupFile(validStorageLocation, validBackupFileName, validStorageLocation, validBackupFileName)); + assertTrue(output.getOut().contains(AdminService.ERR_FILE_ALREADY_EXIST)); + } - () -> assertAll( - () -> assertFalse(adminService.copyBackupFileFromStorageToStorage(validStorageLocation, validBackupFileName, validStorageLocation, validBackupFileName)), - () -> assertTrue(output.getOut().contains(AdminService.ERR_FILE_ALREADY_EXIST)) - ), - - () -> assertAll( - () -> assertFalse(adminService.copyBackupFileFromStorageToStorage(validStorageLocation, validBackupFileName, "", "")), - () -> assertTrue(output.getOut().contains(AdminService.ERR_BACKUP_FILE_OUTSIDE_STORAGE)) - ), - - () -> assertAll( - () -> assertFalse(adminService.copyBackupFileFromStorageToStorage(emptyStorageLocation, validBackupFileName, "", "")), - () -> assertTrue(output.getOut().contains(AdminService.ERR_BACKUP_FILE_OUTSIDE_STORAGE)) - ), - - () -> assertThrows( - FileSystemNotFoundException.class, - () -> adminService.copyBackupFileFromStorageToStorage(validStorageLocation, "backup2.sql", "", ""), - AdminService.ERR_BACKUP_FILE_NOT_EXIST - ) + @ParameterizedTest + @ValueSource(strings = {"", "/opt"}) + void shouldFailToCopyWhenSourceIsOutsideStorage(String location, CapturedOutput output) { + final String validStorageLocation = tempStorageLocation.getPath(); + final String validBackupFileName = "backup.sql"; + adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), validBackupFileName); + + assertFalse(adminService.copyBackupFile(location, validBackupFileName, validStorageLocation, validBackupFileName)); + assertTrue(output.getOut().contains(AdminService.ERR_BACKUP_FILE_OUTSIDE_STORAGE)); + } + + @ParameterizedTest + @ValueSource(strings = {"", "/opt"}) + void shouldFailToCopyWhenDestinationIsOutsideStorage(String location, CapturedOutput output) { + final String validStorageLocation = tempStorageLocation.getPath(); + final String validBackupFileName = "backup.sql"; + adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), validBackupFileName); + + assertFalse(adminService.copyBackupFile(validStorageLocation, validBackupFileName, location, "")); + assertTrue(output.getOut().contains(AdminService.ERR_REPLICATE_OR_COPY_FILE_OUTSIDE_STORAGE)); + } + + @Test + void shouldFailToCopyWhenBackupFileDoesNotExist() { + final String validStorageLocation = tempStorageLocation.getPath(); + final String validBackupFileName = "backup.sql"; + adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), validBackupFileName); + + assertThrows( + FileSystemNotFoundException.class, + () -> adminService.copyBackupFile(validStorageLocation, "backup2.sql", "", ""), + AdminService.ERR_BACKUP_FILE_NOT_EXIST ); } + + @Test + void shouldFailToCopyWhenDestinationStorageDoesNotExist(CapturedOutput output) { + final String validStorageLocation = tempStorageLocation.getPath(); + final String validBackupFileName = "backup.sql"; + adminService.createDatabaseBackupFile(tempStorageLocation.getPath(), validBackupFileName); + + assertFalse(adminService.copyBackupFile(validStorageLocation, validBackupFileName, "/tmp/nonexistent", validBackupFileName)); + assertTrue(output.getAll().contains("NoSuchFileException")); + } // endregion //region utils - @Test void testCheckCommonParametersValidation() { // Valid case From 2179c3a2efe0c87666abedbd2f665fb83b6accd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Tue, 28 Nov 2023 11:19:54 +0100 Subject: [PATCH 52/64] Upgrade to Spring Boot 2.7.17 and Spring Dependency Management Plugin 1.1.4 --- CHANGELOG.md | 2 ++ build.gradle | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a7abe7b..9c744714 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ All notable changes to this project will be documented in this file. ### Dependency Upgrades +- Upgrade to Spring Boot 2.7.17. (#218) +- Upgrade to Spring Dependency Management Plugin 1.1.4. (#218) - Upgrade to `jenkins-library` 2.7.4. (#216) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 diff --git a/build.gradle b/build.gradle index ae2c3006..c72ba0e4 100644 --- a/build.gradle +++ b/build.gradle @@ -1,8 +1,8 @@ plugins { id 'java' id 'io.freefair.lombok' version '8.2.2' - id 'org.springframework.boot' version '2.7.14' - id 'io.spring.dependency-management' version '1.1.3' + id 'org.springframework.boot' version '2.7.17' + id 'io.spring.dependency-management' version '1.1.4' id 'jacoco' id 'org.sonarqube' version '4.2.1.3168' id 'maven-publish' @@ -103,7 +103,7 @@ springBoot { tasks.named("bootJar") { manifest { attributes("Implementation-Title": "iExec Secret Management Service", - "Implementation-Version": project.version) + "Implementation-Version": project.version) } } @@ -112,7 +112,7 @@ test { } tasks.register('itest', Test) { - group 'Verification' + group 'Verification' description 'Runs the integration tests.' testClassesDirs = sourceSets.integrationTest.output.classesDirs classpath = sourceSets.integrationTest.runtimeClasspath @@ -126,7 +126,7 @@ jacocoTestReport { xml.required = true } } -tasks.test.finalizedBy tasks.jacocoTestReport +tasks.test.finalizedBy tasks.jacocoTestReport tasks.sonarqube.dependsOn tasks.jacocoTestReport publishing { From d26e42583f866066fb343a28cade9f6342beb545 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Wed, 29 Nov 2023 13:51:44 +0100 Subject: [PATCH 53/64] Upgrade to `eclipse-temurin:11.0.21_9-jre-focal` --- CHANGELOG.md | 1 + Dockerfile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c744714..6778a4c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ All notable changes to this project will be documented in this file. ### Dependency Upgrades +- Upgrade to `eclipse-temurin:11.0.21_9-jre-focal`. (#219) - Upgrade to Spring Boot 2.7.17. (#218) - Upgrade to Spring Dependency Management Plugin 1.1.4. (#218) - Upgrade to `jenkins-library` 2.7.4. (#216) diff --git a/Dockerfile b/Dockerfile index b9a61a46..2c71112a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM eclipse-temurin:11.0.20_8-jre-focal +FROM eclipse-temurin:11.0.21_9-jre-focal ARG jar From e3b52a16f6759b4e2c12666fcc2b27104cb30f68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Mon, 4 Dec 2023 16:02:44 +0100 Subject: [PATCH 54/64] Expose version through prometheus endpoint and through VersionController --- CHANGELOG.md | 2 + build.gradle | 2 +- .../com/iexec/sms/config/OpenApiConfig.java | 16 ++--- .../iexec/sms/version/VersionController.java | 27 ++++++-- .../com/iexec/sms/version/VersionService.java | 35 ----------- .../iexec/sms/config/OpenApiConfigTests.java | 56 +++++++++++++++++ .../sms/version/VersionControllerTests.java | 63 ++++++++++++++----- .../sms/version/VersionServiceTests.java | 49 --------------- 8 files changed, 139 insertions(+), 111 deletions(-) delete mode 100644 src/main/java/com/iexec/sms/version/VersionService.java create mode 100644 src/test/java/com/iexec/sms/config/OpenApiConfigTests.java delete mode 100644 src/test/java/com/iexec/sms/version/VersionServiceTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 6778a4c7..379e37ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,12 +14,14 @@ All notable changes to this project will be documented in this file. - Add the ability to trigger a delete via a dedicated endpoint. (#213) - Add the ability to trigger a backup replication via a dedicated endpoint. (#214) - Add the ability to trigger a backup copy via a dedicated endpoint. (#217) +- Expose version through prometheus endpoint and through VersionController. (#220) ### Dependency Upgrades - Upgrade to `eclipse-temurin:11.0.21_9-jre-focal`. (#219) - Upgrade to Spring Boot 2.7.17. (#218) - Upgrade to Spring Dependency Management Plugin 1.1.4. (#218) +- Upgrade to Spring Doc Openapi 1.7.0. (#220) - Upgrade to `jenkins-library` 2.7.4. (#216) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 diff --git a/build.gradle b/build.gradle index c72ba0e4..db35bc13 100644 --- a/build.gradle +++ b/build.gradle @@ -76,7 +76,7 @@ dependencies { implementation 'com.h2database:h2:2.2.222' // Spring Doc - implementation 'org.springdoc:springdoc-openapi-ui:1.6.3' + implementation 'org.springdoc:springdoc-openapi-ui:1.7.0' //ssl implementation 'org.apache.httpcomponents:httpclient' diff --git a/src/main/java/com/iexec/sms/config/OpenApiConfig.java b/src/main/java/com/iexec/sms/config/OpenApiConfig.java index ded0a120..d21e9aa2 100644 --- a/src/main/java/com/iexec/sms/config/OpenApiConfig.java +++ b/src/main/java/com/iexec/sms/config/OpenApiConfig.java @@ -16,19 +16,20 @@ package com.iexec.sms.config; -import com.iexec.sms.version.VersionService; import io.swagger.v3.oas.models.OpenAPI; import io.swagger.v3.oas.models.info.Info; +import org.springframework.boot.info.BuildProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @Configuration public class OpenApiConfig { + public static final String TITLE = "iExec SMS"; - private final VersionService versionService; + private final BuildProperties buildProperties; - public OpenApiConfig(VersionService versionService) { - this.versionService = versionService; + public OpenApiConfig(BuildProperties buildProperties) { + this.buildProperties = buildProperties; } /* @@ -38,7 +39,8 @@ public OpenApiConfig(VersionService versionService) { public OpenAPI api() { return new OpenAPI().info( new Info() - .title("iExec SMS") - .version(versionService.getVersion()) + .title(TITLE) + .version(buildProperties.getVersion()) ); - }} + } +} diff --git a/src/main/java/com/iexec/sms/version/VersionController.java b/src/main/java/com/iexec/sms/version/VersionController.java index 62437b2b..89286598 100644 --- a/src/main/java/com/iexec/sms/version/VersionController.java +++ b/src/main/java/com/iexec/sms/version/VersionController.java @@ -16,21 +16,40 @@ package com.iexec.sms.version; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Metrics; +import org.springframework.boot.info.BuildProperties; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; +import javax.annotation.PostConstruct; + @RestController public class VersionController { - private final VersionService versionService; + public static final String METRIC_INFO_GAUGE_NAME = "iexec.version.info"; + public static final String METRIC_INFO_GAUGE_DESC = "A metric to expose version and application name."; + public static final String METRIC_INFO_LABEL_APP_NAME = "iexecAppName"; + public static final String METRIC_INFO_LABEL_APP_VERSION = "iexecAppVersion"; + + private final BuildProperties buildProperties; + + public VersionController(BuildProperties buildProperties) { + this.buildProperties = buildProperties; + } - public VersionController(VersionService versionService) { - this.versionService = versionService; + @PostConstruct + void initializeGaugeVersion() { + Gauge.builder(METRIC_INFO_GAUGE_NAME, 1.0, n -> n) + .description(METRIC_INFO_GAUGE_DESC) + .tags(METRIC_INFO_LABEL_APP_VERSION, buildProperties.getVersion(), + METRIC_INFO_LABEL_APP_NAME, buildProperties.getName()) + .register(Metrics.globalRegistry); } @GetMapping("/version") public ResponseEntity getVersion() { - return ResponseEntity.ok(versionService.getVersion()); + return ResponseEntity.ok(buildProperties.getVersion()); } } diff --git a/src/main/java/com/iexec/sms/version/VersionService.java b/src/main/java/com/iexec/sms/version/VersionService.java deleted file mode 100644 index ef6d85cc..00000000 --- a/src/main/java/com/iexec/sms/version/VersionService.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2020-2023 IEXEC BLOCKCHAIN TECH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.iexec.sms.version; - -import org.springframework.boot.info.BuildProperties; -import org.springframework.stereotype.Service; - -@Service -public class VersionService { - - private final BuildProperties buildProperties; - - VersionService(BuildProperties buildProperties) { - this.buildProperties = buildProperties; - } - - public String getVersion() { - return buildProperties.getVersion(); - } - -} diff --git a/src/test/java/com/iexec/sms/config/OpenApiConfigTests.java b/src/test/java/com/iexec/sms/config/OpenApiConfigTests.java new file mode 100644 index 00000000..f7597d79 --- /dev/null +++ b/src/test/java/com/iexec/sms/config/OpenApiConfigTests.java @@ -0,0 +1,56 @@ +/* + * Copyright 2023-2023 IEXEC BLOCKCHAIN TECH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.iexec.sms.config; + +import io.swagger.v3.oas.models.OpenAPI; +import io.swagger.v3.oas.models.info.Info; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.info.ProjectInfoAutoConfiguration; +import org.springframework.boot.info.BuildProperties; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(SpringExtension.class) +@Import(ProjectInfoAutoConfiguration.class) +class OpenApiConfigTest { + + @Autowired + private BuildProperties buildProperties; + + private OpenApiConfig openApiConfig; + + @BeforeEach + void setUp() { + openApiConfig = new OpenApiConfig(buildProperties); + } + + @Test + void shouldReturnOpenAPIObjectWithCorrectInfo() { + OpenAPI api = openApiConfig.api(); + assertThat(api).isNotNull(). + extracting(OpenAPI::getInfo).isNotNull(). + extracting( + Info::getVersion, + Info::getTitle + ) + .containsExactly(buildProperties.getVersion(), OpenApiConfig.TITLE); + } +} diff --git a/src/test/java/com/iexec/sms/version/VersionControllerTests.java b/src/test/java/com/iexec/sms/version/VersionControllerTests.java index 4b2d5c4c..10c04006 100644 --- a/src/test/java/com/iexec/sms/version/VersionControllerTests.java +++ b/src/test/java/com/iexec/sms/version/VersionControllerTests.java @@ -16,32 +16,65 @@ package com.iexec.sms.version; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.info.ProjectInfoAutoConfiguration; +import org.springframework.boot.info.BuildProperties; +import org.springframework.context.annotation.Import; import org.springframework.http.ResponseEntity; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.when; +@ExtendWith(SpringExtension.class) +@Import(ProjectInfoAutoConfiguration.class) class VersionControllerTests { - @Mock - private VersionService versionService; - @InjectMocks + private VersionController versionController; + @Autowired + private BuildProperties buildProperties; + + @BeforeAll + static void initRegistry() { + Metrics.globalRegistry.add(new SimpleMeterRegistry()); + } + @BeforeEach void init() { - MockitoAnnotations.openMocks(this); + versionController = new VersionController(buildProperties); + versionController.initializeGaugeVersion(); + } + + @AfterEach + void afterEach() { + Metrics.globalRegistry.clear(); + } + + @Test + void testVersionController() { + assertEquals(ResponseEntity.ok(buildProperties.getVersion()), versionController.getVersion()); } - @ParameterizedTest - @ValueSource(strings={"x.y.z", "x.y.z-rc", "x.y.z-NEXT-SNAPSHOT"}) - void testVersionController(String version) { - when(versionService.getVersion()).thenReturn(version); - assertEquals(ResponseEntity.ok(version), versionController.getVersion()); + @Test + void shouldReturnInfoGauge() { + final Gauge info = Metrics.globalRegistry.find(VersionController.METRIC_INFO_GAUGE_NAME).gauge(); + assertThat(info) + .isNotNull() + .extracting(Gauge::getId) + .isNotNull() + .extracting( + id -> id.getTag(VersionController.METRIC_INFO_LABEL_APP_NAME), + id -> id.getTag(VersionController.METRIC_INFO_LABEL_APP_VERSION) + ) + .containsExactly(buildProperties.getName(), buildProperties.getVersion()); } } diff --git a/src/test/java/com/iexec/sms/version/VersionServiceTests.java b/src/test/java/com/iexec/sms/version/VersionServiceTests.java deleted file mode 100644 index 1de3226a..00000000 --- a/src/test/java/com/iexec/sms/version/VersionServiceTests.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2020-2023 IEXEC BLOCKCHAIN TECH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.iexec.sms.version; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.springframework.boot.info.BuildProperties; - -class VersionServiceTests { - - @Mock - private BuildProperties buildProperties; - - @InjectMocks - private VersionService versionService; - - @BeforeEach - public void preflight() { - MockitoAnnotations.openMocks(this); - } - - @ParameterizedTest - @ValueSource(strings={"x.y.z", "x.y.z-rc", "x.y.z-NEXT-SNAPSHOT"}) - void testVersions(String version) { - Mockito.when(buildProperties.getVersion()).thenReturn(version); - Assertions.assertEquals(version, versionService.getVersion()); - } - -} From 407b7a9f0140d830ba4cbded3d0f398296396c7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Tue, 5 Dec 2023 09:52:05 +0100 Subject: [PATCH 55/64] Fix OpenAPI name --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 379e37ac..a5abef4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ All notable changes to this project will be documented in this file. - Upgrade to `eclipse-temurin:11.0.21_9-jre-focal`. (#219) - Upgrade to Spring Boot 2.7.17. (#218) - Upgrade to Spring Dependency Management Plugin 1.1.4. (#218) -- Upgrade to Spring Doc Openapi 1.7.0. (#220) +- Upgrade to Spring Doc OpenAPI 1.7.0. (#220) - Upgrade to `jenkins-library` 2.7.4. (#216) ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 From e292a0310f24e9b587ba32c6235862687b6d5075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 7 Dec 2023 10:12:31 +0100 Subject: [PATCH 56/64] Fix NaN value for gauge version and add tee framework --- CHANGELOG.md | 2 +- .../iexec/sms/version/VersionController.java | 14 ++++-- .../sms/version/VersionControllerTests.java | 45 +++++++++++++------ 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5abef4f..f566250e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ All notable changes to this project will be documented in this file. - Add the ability to trigger a delete via a dedicated endpoint. (#213) - Add the ability to trigger a backup replication via a dedicated endpoint. (#214) - Add the ability to trigger a backup copy via a dedicated endpoint. (#217) -- Expose version through prometheus endpoint and through VersionController. (#220) +- Expose version through prometheus endpoint and through VersionController. (#220 #221) ### Dependency Upgrades diff --git a/src/main/java/com/iexec/sms/version/VersionController.java b/src/main/java/com/iexec/sms/version/VersionController.java index 89286598..ff7408b4 100644 --- a/src/main/java/com/iexec/sms/version/VersionController.java +++ b/src/main/java/com/iexec/sms/version/VersionController.java @@ -16,6 +16,7 @@ package com.iexec.sms.version; +import com.iexec.sms.api.config.TeeServicesProperties; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.Metrics; import org.springframework.boot.info.BuildProperties; @@ -32,19 +33,24 @@ public class VersionController { public static final String METRIC_INFO_GAUGE_DESC = "A metric to expose version and application name."; public static final String METRIC_INFO_LABEL_APP_NAME = "iexecAppName"; public static final String METRIC_INFO_LABEL_APP_VERSION = "iexecAppVersion"; - + public static final String METRIC_INFO_LABEL_TEE_FRAMEWORK = "teeFramework"; + // Must be static final to avoid garbage collect and side effect on gauge + public static final int METRIC_VALUE = 1; private final BuildProperties buildProperties; + private final TeeServicesProperties teeServicesProperties; - public VersionController(BuildProperties buildProperties) { + public VersionController(BuildProperties buildProperties, TeeServicesProperties teeServicesProperties) { this.buildProperties = buildProperties; + this.teeServicesProperties = teeServicesProperties; } @PostConstruct void initializeGaugeVersion() { - Gauge.builder(METRIC_INFO_GAUGE_NAME, 1.0, n -> n) + Gauge.builder(METRIC_INFO_GAUGE_NAME, METRIC_VALUE, n -> METRIC_VALUE) .description(METRIC_INFO_GAUGE_DESC) .tags(METRIC_INFO_LABEL_APP_VERSION, buildProperties.getVersion(), - METRIC_INFO_LABEL_APP_NAME, buildProperties.getName()) + METRIC_INFO_LABEL_APP_NAME, buildProperties.getName(), + METRIC_INFO_LABEL_TEE_FRAMEWORK, teeServicesProperties.getTeeFramework().name()) .register(Metrics.globalRegistry); } diff --git a/src/test/java/com/iexec/sms/version/VersionControllerTests.java b/src/test/java/com/iexec/sms/version/VersionControllerTests.java index 10c04006..66af0fe1 100644 --- a/src/test/java/com/iexec/sms/version/VersionControllerTests.java +++ b/src/test/java/com/iexec/sms/version/VersionControllerTests.java @@ -16,14 +16,20 @@ package com.iexec.sms.version; +import com.iexec.commons.poco.tee.TeeFramework; +import com.iexec.sms.api.config.GramineServicesProperties; +import com.iexec.sms.api.config.SconeServicesProperties; +import com.iexec.sms.api.config.TeeAppProperties; +import com.iexec.sms.api.config.TeeServicesProperties; import io.micrometer.core.instrument.Gauge; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.simple.SimpleMeterRegistry; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.info.ProjectInfoAutoConfiguration; import org.springframework.boot.info.BuildProperties; @@ -37,9 +43,9 @@ @ExtendWith(SpringExtension.class) @Import(ProjectInfoAutoConfiguration.class) class VersionControllerTests { - + TeeAppProperties preComputeProperties; + TeeAppProperties postComputeProperties; private VersionController versionController; - @Autowired private BuildProperties buildProperties; @@ -48,12 +54,6 @@ static void initRegistry() { Metrics.globalRegistry.add(new SimpleMeterRegistry()); } - @BeforeEach - void init() { - versionController = new VersionController(buildProperties); - versionController.initializeGaugeVersion(); - } - @AfterEach void afterEach() { Metrics.globalRegistry.clear(); @@ -64,8 +64,25 @@ void testVersionController() { assertEquals(ResponseEntity.ok(buildProperties.getVersion()), versionController.getVersion()); } - @Test - void shouldReturnInfoGauge() { + @ParameterizedTest + @EnumSource(value = TeeFramework.class) + void shouldReturnInfoGauge(TeeFramework teeFramework) { + TeeServicesProperties properties; + if (teeFramework == TeeFramework.SCONE) { + properties = new SconeServicesProperties( + preComputeProperties, + postComputeProperties, + "lasImage" + ); + } else { + properties = new GramineServicesProperties( + preComputeProperties, + postComputeProperties + ); + } + versionController = new VersionController(buildProperties, properties); + versionController.initializeGaugeVersion(); + final Gauge info = Metrics.globalRegistry.find(VersionController.METRIC_INFO_GAUGE_NAME).gauge(); assertThat(info) .isNotNull() @@ -73,8 +90,10 @@ void shouldReturnInfoGauge() { .isNotNull() .extracting( id -> id.getTag(VersionController.METRIC_INFO_LABEL_APP_NAME), - id -> id.getTag(VersionController.METRIC_INFO_LABEL_APP_VERSION) + id -> id.getTag(VersionController.METRIC_INFO_LABEL_APP_VERSION), + id -> id.getTag(VersionController.METRIC_INFO_LABEL_TEE_FRAMEWORK) ) - .containsExactly(buildProperties.getName(), buildProperties.getVersion()); + .containsExactly(buildProperties.getName(), buildProperties.getVersion(), teeFramework.name()); + assertThat(info.value()).isEqualTo(VersionController.METRIC_VALUE); } } From b68b9a1fee3e2306cb5d0b58e8743ff75e18a174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 7 Dec 2023 14:01:58 +0100 Subject: [PATCH 57/64] Switch to more simple builder for gauge --- src/main/java/com/iexec/sms/version/VersionController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/iexec/sms/version/VersionController.java b/src/main/java/com/iexec/sms/version/VersionController.java index ff7408b4..c11a123a 100644 --- a/src/main/java/com/iexec/sms/version/VersionController.java +++ b/src/main/java/com/iexec/sms/version/VersionController.java @@ -46,7 +46,7 @@ public VersionController(BuildProperties buildProperties, TeeServicesProperties @PostConstruct void initializeGaugeVersion() { - Gauge.builder(METRIC_INFO_GAUGE_NAME, METRIC_VALUE, n -> METRIC_VALUE) + Gauge.builder(METRIC_INFO_GAUGE_NAME, () -> METRIC_VALUE) .description(METRIC_INFO_GAUGE_DESC) .tags(METRIC_INFO_LABEL_APP_VERSION, buildProperties.getVersion(), METRIC_INFO_LABEL_APP_NAME, buildProperties.getName(), From 5fef786bfea2fc2b37996af7b6eeb523cf576167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 7 Dec 2023 14:14:54 +0100 Subject: [PATCH 58/64] Fix tests --- .../java/com/iexec/sms/version/VersionControllerTests.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/com/iexec/sms/version/VersionControllerTests.java b/src/test/java/com/iexec/sms/version/VersionControllerTests.java index 66af0fe1..56fcf89f 100644 --- a/src/test/java/com/iexec/sms/version/VersionControllerTests.java +++ b/src/test/java/com/iexec/sms/version/VersionControllerTests.java @@ -61,6 +61,12 @@ void afterEach() { @Test void testVersionController() { + TeeServicesProperties properties = new SconeServicesProperties( + preComputeProperties, + postComputeProperties, + "lasImage" + ); + versionController = new VersionController(buildProperties, properties); assertEquals(ResponseEntity.ok(buildProperties.getVersion()), versionController.getVersion()); } From 3f2e310d41ba2cd803f7e42e1f014b9b2c23ff1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 7 Dec 2023 15:16:40 +0100 Subject: [PATCH 59/64] set scone as framework for TeeTaskComputeSecretIntegrationTests --- .../com/iexec/sms/MockTeeConfiguration.java | 20 ------------------- .../TeeTaskComputeSecretIntegrationTests.java | 3 +-- 2 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 src/itest/java/com/iexec/sms/MockTeeConfiguration.java diff --git a/src/itest/java/com/iexec/sms/MockTeeConfiguration.java b/src/itest/java/com/iexec/sms/MockTeeConfiguration.java deleted file mode 100644 index b1ecce4d..00000000 --- a/src/itest/java/com/iexec/sms/MockTeeConfiguration.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.iexec.sms; - -import com.iexec.sms.api.config.TeeServicesProperties; -import com.iexec.sms.tee.session.generic.TeeSessionHandler; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Profile; - -import static com.iexec.sms.MockTeeConfiguration.MOCK_TEE_PROFILE; - -@Configuration -@Profile(MOCK_TEE_PROFILE) -public class MockTeeConfiguration { - public static final String MOCK_TEE_PROFILE = "mock-tee"; - - @MockBean - private TeeSessionHandler teeSessionHandler; - @MockBean - private TeeServicesProperties teeServicesProperties; -} diff --git a/src/itest/java/com/iexec/sms/secret/compute/TeeTaskComputeSecretIntegrationTests.java b/src/itest/java/com/iexec/sms/secret/compute/TeeTaskComputeSecretIntegrationTests.java index eb27ae79..f8908c30 100644 --- a/src/itest/java/com/iexec/sms/secret/compute/TeeTaskComputeSecretIntegrationTests.java +++ b/src/itest/java/com/iexec/sms/secret/compute/TeeTaskComputeSecretIntegrationTests.java @@ -47,12 +47,11 @@ import static com.iexec.commons.poco.utils.SignatureUtils.signMessageHashAndGetSignature; import static com.iexec.sms.MockChainConfiguration.MOCK_CHAIN_PROFILE; -import static com.iexec.sms.MockTeeConfiguration.MOCK_TEE_PROFILE; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @Slf4j -@ActiveProfiles({MOCK_TEE_PROFILE, MOCK_CHAIN_PROFILE, "test"}) +@ActiveProfiles({"scone", MOCK_CHAIN_PROFILE, "test"}) public class TeeTaskComputeSecretIntegrationTests extends CommonTestSetup { private static final String APP_ADDRESS = "0xabcd1339ec7e762e639f4887e2bfe5ee8023e23e"; private static final String UPPER_CASE_APP_ADDRESS = "0xABCD1339EC7E762E639F4887E2BFE5EE8023E23E"; From 45f0ee3387c2d3cccbe6ca4c33c6fe960fde1190 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 7 Dec 2023 15:22:02 +0100 Subject: [PATCH 60/64] Fix IT by setting scone as profile instead of mock --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f566250e..f6d44637 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,10 @@ All notable changes to this project will be documented in this file. - Upgrade to Spring Doc OpenAPI 1.7.0. (#220) - Upgrade to `jenkins-library` 2.7.4. (#216) +### Bug Fixes + +- Remove MockTeeConfiguration and set scone instead in `TeeTaskComputeSecretIntegrationTests`. (#222) + ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 ### Bug Fixes From 354a5e2ffc98c8558004687da72ca99251281149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Cordier?= Date: Thu, 21 Dec 2023 09:29:16 +0100 Subject: [PATCH 61/64] Upgrade to `iexec-commons-poco` 3.2.0 and to `iexec-common` 8.3.1 --- CHANGELOG.md | 2 ++ gradle.properties | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6d44637..493a80a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,8 @@ All notable changes to this project will be documented in this file. - Upgrade to Spring Dependency Management Plugin 1.1.4. (#218) - Upgrade to Spring Doc OpenAPI 1.7.0. (#220) - Upgrade to `jenkins-library` 2.7.4. (#216) +- Upgrade to `iexec-commons-poco` 3.2.0. (#223) +- Upgrade to `iexec-common` 8.3.1. (#223) ### Bug Fixes diff --git a/gradle.properties b/gradle.properties index 4d7adc7e..8120bbb1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ version=8.3.0 -iexecCommonVersion=8.3.0 -iexecCommonsPocoVersion=3.1.0 +iexecCommonVersion=8.3.1 +iexecCommonsPocoVersion=3.2.0 nexusUser nexusPassword From 9a14da39e42b2234abe7de08010fe14d25ee4c5f Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Fri, 22 Dec 2023 10:52:58 +0100 Subject: [PATCH 62/64] Remove `/up` endpoint (#224) --- CHANGELOG.md | 9 +++-- .../java/com/iexec/sms/api/SmsClient.java | 4 +- .../java/com/iexec/sms/AppController.java | 37 ------------------- 3 files changed, 6 insertions(+), 44 deletions(-) delete mode 100644 src/main/java/com/iexec/sms/AppController.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 493a80a7..869b9d13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ All notable changes to this project will be documented in this file. - Add the ability to trigger a backup copy via a dedicated endpoint. (#217) - Expose version through prometheus endpoint and through VersionController. (#220 #221) +### Bug Fixes + +- Remove MockTeeConfiguration and set scone instead in `TeeTaskComputeSecretIntegrationTests`. (#222) +- Remove `/up` endpoint. (#224) + ### Dependency Upgrades - Upgrade to `eclipse-temurin:11.0.21_9-jre-focal`. (#219) @@ -26,10 +31,6 @@ All notable changes to this project will be documented in this file. - Upgrade to `iexec-commons-poco` 3.2.0. (#223) - Upgrade to `iexec-common` 8.3.1. (#223) -### Bug Fixes - -- Remove MockTeeConfiguration and set scone instead in `TeeTaskComputeSecretIntegrationTests`. (#222) - ## [[8.3.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.3.0) 2023-09-28 ### Bug Fixes diff --git a/iexec-sms-library/src/main/java/com/iexec/sms/api/SmsClient.java b/iexec-sms-library/src/main/java/com/iexec/sms/api/SmsClient.java index 1452dd3d..0af33a03 100644 --- a/iexec-sms-library/src/main/java/com/iexec/sms/api/SmsClient.java +++ b/iexec-sms-library/src/main/java/com/iexec/sms/api/SmsClient.java @@ -31,13 +31,11 @@ * Interface allowing to instantiate a Feign client targeting SMS REST endpoints. *

* To create the client, see the related builder. + * * @see SmsClientBuilder */ public interface SmsClient { - @RequestLine("GET /up") - String isUp(); - // region Secrets @RequestLine("POST /apps/{appAddress}/secrets/1") @Headers("Authorization: {authorization}") diff --git a/src/main/java/com/iexec/sms/AppController.java b/src/main/java/com/iexec/sms/AppController.java deleted file mode 100644 index a6735a53..00000000 --- a/src/main/java/com/iexec/sms/AppController.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2020 IEXEC BLOCKCHAIN TECH - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.iexec.sms; - - -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RestController; - -import java.util.Date; - -@RestController -public class AppController { - - //TODO: Remove this endpoint, use actuator endpoints instead. Update client too. - @GetMapping(value = "/up") - public ResponseEntity isUp() { - String message = String.format("Up! (%s)", new Date()); - return ResponseEntity.ok(message); - } - -} - From c8d4bc8688a8cd6ba7a4b4bf2da04b2aef953926 Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Tue, 9 Jan 2024 11:46:56 +0100 Subject: [PATCH 63/64] Fix `README.md` and remove some code smells (#225) --- CHANGELOG.md | 3 +- README.md | 2 +- .../base/SecretSessionBaseService.java | 59 +++++++++---------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 869b9d13..14c79cd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. -## [[NEXT]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/vNEXT) 2023 +## [[NEXT]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/vNEXT) 2024 ### New Features @@ -20,6 +20,7 @@ All notable changes to this project will be documented in this file. - Remove MockTeeConfiguration and set scone instead in `TeeTaskComputeSecretIntegrationTests`. (#222) - Remove `/up` endpoint. (#224) +- Fix `README.md` and remove some code smells. (#225) ### Dependency Upgrades diff --git a/README.md b/README.md index f5e7c0fe..d8362910 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ To support: | --- | --- | --- | --- | --- | | `IEXEC_SMS_TEE_RUNTIME_FRAMEWORK` | Define which TEE framework this _iExec SMS_ supports. | `scone` or `gramine` | | | | `IEXEC_SMS_PORT` | Server HTTP port. | Positive integer | `13300` | `13300` | -| `IEXEC_SMS_H2_URL` | JDBC URL of the database. | URL | `jdbc:h2:file:/tmp/h2/sms-h2` | `jdbc:h2:file:/tmp/h2/sms-h2` | +| `IEXEC_SMS_H2_URL` | JDBC URL of the database. | URL | `jdbc:h2:file:/data/sms-h2` | `jdbc:h2:file:/data/sms-h2` | | `IEXEC_SMS_H2_CONSOLE` | Whether to enable the H2 console. | Boolean | `false` | `false` | | `IEXEC_SMS_STORAGE_ENCRYPTION_AES_KEY_PATH` | Path to the key created and used to encrypt secrets. | String | `src/main/resources/iexec-sms-aes.key` | `src/main/resources/iexec-sms-aes.key` | | `IEXEC_SMS_ADMIN_API_KEY` | API key used to authorize calls to `/admin` endpoints. | String | | | diff --git a/src/main/java/com/iexec/sms/tee/session/base/SecretSessionBaseService.java b/src/main/java/com/iexec/sms/tee/session/base/SecretSessionBaseService.java index a6509661..19ae3765 100644 --- a/src/main/java/com/iexec/sms/tee/session/base/SecretSessionBaseService.java +++ b/src/main/java/com/iexec/sms/tee/session/base/SecretSessionBaseService.java @@ -119,8 +119,9 @@ public SecretSessionBase getSecretsTokens(TeeSessionRequest request) throws TeeS /** * Get tokens to be injected in the pre-compute enclave. * - * @return map of pre-compute tokens - * @throws TeeSessionGenerationException if dataset secret is not found. + * @param request Session request details + * @return {@link SecretEnclaveBase} instance + * @throws TeeSessionGenerationException if dataset secret is not found */ public SecretEnclaveBase getPreComputeTokens(TeeSessionRequest request) throws TeeSessionGenerationException { @@ -157,18 +158,22 @@ public SecretEnclaveBase getPreComputeTokens(TeeSessionRequest request) .entrySet() .stream() .filter(e -> - // extract trusted en vars to include - trustedEnv.contains(e.getKey()) - // extract - || e.getKey().startsWith(IexecEnvUtils.IEXEC_INPUT_FILE_URL_PREFIX)) + // extract trusted en vars to include + trustedEnv.contains(e.getKey()) + // extract + || e.getKey().startsWith(IexecEnvUtils.IEXEC_INPUT_FILE_URL_PREFIX)) .forEach(e -> tokens.put(e.getKey(), e.getValue())); return enclaveBase .environment(tokens) .build(); } - /* - * Compute (App) + /** + * Get tokens to be injected in the application enclave. + * + * @param request Session request details + * @return {@link SecretEnclaveBase} instance + * @throws TeeSessionGenerationException if {@code TaskDescription} is {@literal null} or does not contain a {@code TeeEnclaveConfiguration} */ public SecretEnclaveBase getAppTokens(TeeSessionRequest request) throws TeeSessionGenerationException { @@ -220,11 +225,11 @@ private Map getApplicationComputeSecrets(TaskDescription taskDes if (applicationAddress != null) { final String secretIndex = "1"; String appDeveloperSecret = teeTaskComputeSecretService.getSecret( - OnChainObjectType.APPLICATION, - applicationAddress.toLowerCase(), - SecretOwnerRole.APPLICATION_DEVELOPER, - "", - secretIndex) + OnChainObjectType.APPLICATION, + applicationAddress.toLowerCase(), + SecretOwnerRole.APPLICATION_DEVELOPER, + "", + secretIndex) .map(TeeTaskComputeSecret::getValue) .orElse(EMPTY_YML_VALUE); if (!StringUtils.isEmpty(appDeveloperSecret)) { @@ -252,11 +257,11 @@ private Map getApplicationComputeSecrets(TaskDescription taskDes continue; } String requesterSecret = teeTaskComputeSecretService.getSecret( - OnChainObjectType.APPLICATION, - "", - SecretOwnerRole.REQUESTER, - taskDescription.getRequester().toLowerCase(), - secretEntry.getValue()) + OnChainObjectType.APPLICATION, + "", + SecretOwnerRole.REQUESTER, + taskDescription.getRequester().toLowerCase(), + secretEntry.getValue()) .map(TeeTaskComputeSecret::getValue) .orElse(EMPTY_YML_VALUE); requesterSecrets.put(IexecEnvUtils.IEXEC_REQUESTER_SECRET_PREFIX + secretEntry.getKey(), requesterSecret); @@ -265,8 +270,12 @@ private Map getApplicationComputeSecrets(TaskDescription taskDes return tokens; } - /* - * Post-Compute (Result) + /** + * Get tokens to be injected in the post-compute enclave. + * + * @param request Session request details + * @return {@link SecretEnclaveBase} instance + * @throws TeeSessionGenerationException if {@code TaskDescription} is {@literal null} */ public SecretEnclaveBase getPostComputeTokens(TeeSessionRequest request) throws TeeSessionGenerationException { @@ -278,16 +287,6 @@ public SecretEnclaveBase getPostComputeTokens(TeeSessionRequest request) if (taskDescription == null) { throw new TeeSessionGenerationException(NO_TASK_DESCRIPTION, "Task description must not be null"); } - // ############################################################################### - // TODO: activate this when user specific post-compute is properly - // supported. See - // https://github.com/iExecBlockchainComputing/iexec-sms/issues/52. - // ############################################################################### - // // Use specific post-compute image if requested. - // if (taskDescription.containsPostCompute()) { - // teePostComputeFingerprint = taskDescription.getTeePostComputeFingerprint(); - // //add entrypoint too - // } // encryption Map encryptionTokens = getPostComputeEncryptionTokens(request); tokens.putAll(encryptionTokens); From 8adb1868be491724fb93bac9ea45f839d54090fd Mon Sep 17 00:00:00 2001 From: Jeremy Bernard Date: Wed, 10 Jan 2024 16:49:54 +0100 Subject: [PATCH 64/64] Release version 8.4.0 --- CHANGELOG.md | 2 +- gradle.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14c79cd1..726e0738 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. -## [[NEXT]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/vNEXT) 2024 +## [[8.4.0]](https://github.com/iExecBlockchainComputing/iexec-sms/releases/tag/v8.4.0) 2024-01-10 ### New Features diff --git a/gradle.properties b/gradle.properties index 8120bbb1..c23e23fb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -version=8.3.0 +version=8.4.0 iexecCommonVersion=8.3.1 iexecCommonsPocoVersion=3.2.0