Skip to content

Commit

Permalink
Merge pull request #58 from iExecBlockchainComputing/hotfix/8.1.1
Browse files Browse the repository at this point in the history 
Hotfix/8.1.1
  • Loading branch information
@jbern0rd
jbern0rd authored Jun 22, 2023
2 parents 0ffeab3 + a348ced commit 114fd4c
Show file tree
Hide file tree
Showing 4 changed files with 133 additions and 36 deletions.
7 changes: 7 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@

All notable changes to this project will be documented in this file.

## [[8.1.1]](https://github.com/iExecBlockchainComputing/tee-worker-pre-compute/releases/tag/v8.1.1) 2023-06-22

### Features
- Retry dataset download on several IPFS gateways. (#58)
### Dependency Upgrades
- Upgrade to `iexec-commons-poco` 3.0.3. (#58)

## [[8.1.0]](https://github.com/iExecBlockchainComputing/tee-worker-pre-compute/releases/tag/v8.1.0) 2023-06-06

### Dependency Upgrades
Expand Down
4 changes: 2 additions & 2 deletions gradle.properties
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
version=8.1.0
version=8.1.1
iexecCommonVersion=8.2.0
iexecCommonsPocoVersion=3.0.0
iexecCommonsPocoVersion=3.0.3

nexusUser
nexusPassword
21 changes: 16 additions & 5 deletions src/main/java/com/iexec/worker/tee/pre/PreComputeApp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.iexec.common.security.CipherUtils;
import com.iexec.common.utils.FileHelper;
import com.iexec.commons.poco.utils.HashUtils;
import com.iexec.commons.poco.utils.MultiAddressHelper;
import lombok.extern.slf4j.Slf4j;

import java.io.File;
Expand All @@ -39,7 +40,7 @@ public PreComputeApp(String chainTaskId) {
* Download, decrypt, and save the plain dataset file in "/iexec_in".
* If the decrypted file is an archive, it won't be extracted.
*
* @throws PreComputeException
* @throws PreComputeException if dataset or input files could not be made available for the application enclave
*/
void run() throws PreComputeException {
preComputeArgs = PreComputeArgs.readArgs(chainTaskId);
Expand Down Expand Up @@ -73,14 +74,24 @@ void checkOutputFolder() throws PreComputeException {
* Download encrypted dataset file and check its checksum.
*
* @return downloaded file bytes
* @throws PreComputeException if download fails or bad file
* checksum
* @throws PreComputeException if download fails or bad file checksum
*/
byte[] downloadEncryptedDataset() throws PreComputeException {
String encryptedDatasetUrl = getPreComputeArgs().getEncryptedDatasetUrl();
log.info("Downloading encrypted dataset file [chainTaskId:{}, url:{}]",
chainTaskId, encryptedDatasetUrl);
byte[] encryptedContent = FileHelper.readFileBytesFromUrl(encryptedDatasetUrl);
byte[] encryptedContent = null;
if (MultiAddressHelper.isMultiAddress(encryptedDatasetUrl)) {
for (String ipfsGateway : MultiAddressHelper.IPFS_GATEWAYS) {
log.debug("Try to download dataset from {}", ipfsGateway);
encryptedContent = FileHelper.readFileBytesFromUrl(encryptedDatasetUrl);
if (encryptedContent != null) {
break;
}
}
} else {
encryptedContent = FileHelper.readFileBytesFromUrl(encryptedDatasetUrl);
}
if (encryptedContent == null) {
log.error("Failed to download encrypted dataset file [chainTaskId:{}, url:{}]",
chainTaskId, encryptedDatasetUrl);
Expand Down Expand Up @@ -156,7 +167,7 @@ void downloadInputFiles() throws PreComputeException {

/**
* Added for testing purpose.
* @return
* @return A {@link PreComputeArgs} instance
*/
PreComputeArgs getPreComputeArgs() {
return preComputeArgs;
Expand Down
137 changes: 108 additions & 29 deletions src/test/java/com/iexec/worker/tee/pre/PreComputeAppTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,13 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.io.TempDir;
import org.mockito.MockedStatic;
import org.mockito.MockitoAnnotations;
import uk.org.webcompere.systemstubs.environment.EnvironmentVariables;
import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension;

import java.io.File;
import java.net.URL;
import java.util.List;

import static com.iexec.common.precompute.PreComputeUtils.*;
Expand All @@ -48,7 +50,9 @@ class PreComputeAppTests {

private static final String CHAIN_TASK_ID = "0xabc";
private static final String DATASET_FILENAME = "my-dataset";
private static final String DATASET_URL = REPO_URL + "encrypted-data.bin";
private static final String DATASET_RESOURCE_NAME = "encrypted-data.bin";
private static final String HTTP_DATASET_URL = REPO_URL + DATASET_RESOURCE_NAME;
private static final String IPFS_DATASET_URL = "/ipfs/QmUbh7ugQ9WVprTVYjzrCS4d9cCy73zUz4MMchsrqzzu1w";
private static final String DATASET_CHECKSUM =
"0x02a12ef127dcfbdb294a090c8f0b69a0ca30b7940fc36cabf971f488efd374d7";
private static final String RESOURCES = "src/test/resources/";
Expand Down Expand Up @@ -77,7 +81,7 @@ void shouldRunSuccessfullyWithDataset(EnvironmentVariables environment) throws P
IEXEC_TASK_ID, CHAIN_TASK_ID,
IEXEC_PRE_COMPUTE_OUT, outputDir.getAbsolutePath(),
IS_DATASET_REQUIRED, true,
IEXEC_DATASET_URL, DATASET_URL,
IEXEC_DATASET_URL, HTTP_DATASET_URL,
IEXEC_DATASET_KEY, FileHelper.readFile(KEY_FILE),
IEXEC_DATASET_CHECKSUM, DATASET_CHECKSUM,
IEXEC_DATASET_FILENAME, DATASET_FILENAME,
Expand Down Expand Up @@ -114,22 +118,24 @@ void shouldRunSuccessfullyWithoutDataset(EnvironmentVariables environment) throw

assertDoesNotThrow(() -> preComputeApp.run());

verify(preComputeApp, times(0)).downloadEncryptedDataset();
verify(preComputeApp, times(0)).decryptDataset(any());
verify(preComputeApp, times(0)).savePlainDatasetFile(any());
verify(preComputeApp, never()).downloadEncryptedDataset();
verify(preComputeApp, never()).decryptDataset(any());
verify(preComputeApp, never()).savePlainDatasetFile(any());
}
//endregion

@Test
void shouldFindOutputFolder() {
doReturn(goodPreComputeArgs()).when(preComputeApp).getPreComputeArgs();
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(HTTP_DATASET_URL).build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
assertDoesNotThrow(() -> preComputeApp.checkOutputFolder());
}

@Test
void shouldThrowSinceOutputFolderNotFound() {
PreComputeArgs preComputeArgs = goodPreComputeArgs();
preComputeArgs.setOutputDir("bad-output-dir-path");
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(HTTP_DATASET_URL)
.outputDir("bad-output-dir-path")
.build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
PreComputeException e = assertThrows(
PreComputeException.class,
Expand All @@ -139,27 +145,95 @@ void shouldThrowSinceOutputFolderNotFound() {

@Test
void shouldDownloadEncryptedDataset() throws Exception {
doReturn(goodPreComputeArgs()).when(preComputeApp).getPreComputeArgs();
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(HTTP_DATASET_URL).build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
byte[] actualContent = preComputeApp.downloadEncryptedDataset();
byte[] expectedContent = FileHelper.readFileBytesFromUrl(DATASET_URL);
byte[] expectedContent = FileHelper.readFileBytesFromUrl(HTTP_DATASET_URL);
assertThat(actualContent).isEqualTo(expectedContent);
}

@Test
void shouldThrowSinceDownloadFailed() {
PreComputeArgs preComputeArgs = goodPreComputeArgs();
preComputeArgs.setEncryptedDatasetUrl("http://bad-url");
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(HTTP_DATASET_URL)
.encryptedDatasetUrl("http://bad-url")
.build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
PreComputeException e = assertThrows(
PreComputeException.class,
() -> preComputeApp.downloadEncryptedDataset());
assertThat(e.getExitCause()).isEqualTo(ReplicateStatusCause.PRE_COMPUTE_DATASET_DOWNLOAD_FAILED);
}

@Test
void shouldDownloadEncryptedDatasetFromIexecGateway() throws PreComputeException {
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(IPFS_DATASET_URL).build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
final URL resourceFile = this.getClass().getClassLoader().getResource(DATASET_RESOURCE_NAME);
assertThat(resourceFile).isNotNull();
final byte[] expectedBytes = FileHelper.readAllBytes(resourceFile.getFile());
try (MockedStatic<FileHelper> fileHelperMock = mockStatic(FileHelper.class)) {
fileHelperMock.when(() -> FileHelper.readFileBytesFromUrl(anyString()))
.thenReturn(expectedBytes);
assertThat(preComputeApp.downloadEncryptedDataset()).isNotNull();
fileHelperMock.verify(() -> FileHelper.readFileBytesFromUrl(anyString()), times(1));
}
}

@Test
void shouldDownloadEncryptedDatasetFromIpfsGateway() throws PreComputeException {
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(IPFS_DATASET_URL).build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
final URL resourceFile = this.getClass().getClassLoader().getResource(DATASET_RESOURCE_NAME);
assertThat(resourceFile).isNotNull();
final byte[] expectedBytes = FileHelper.readAllBytes(resourceFile.getFile());
try (MockedStatic<FileHelper> fileHelperMock = mockStatic(FileHelper.class)) {
fileHelperMock.when(() -> FileHelper.readFileBytesFromUrl(anyString()))
.thenReturn(null)
.thenReturn(expectedBytes);
assertThat(preComputeApp.downloadEncryptedDataset()).isNotNull();
fileHelperMock.verify(() -> FileHelper.readFileBytesFromUrl(anyString()), times(2));
}
}

@Test
void shouldDownloadEncryptedDatasetFromPinataGateway() throws PreComputeException {
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(IPFS_DATASET_URL).build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
final URL resourceFile = this.getClass().getClassLoader().getResource(DATASET_RESOURCE_NAME);
assertThat(resourceFile).isNotNull();
final byte[] expectedBytes = FileHelper.readAllBytes(resourceFile.getFile());
try (MockedStatic<FileHelper> fileHelperMock = mockStatic(FileHelper.class)) {
fileHelperMock.when(() -> FileHelper.readFileBytesFromUrl(anyString()))
.thenReturn(null)
.thenReturn(null)
.thenReturn(expectedBytes);
assertThat(preComputeApp.downloadEncryptedDataset()).isNotNull();
fileHelperMock.verify(() -> FileHelper.readFileBytesFromUrl(anyString()), times(3));
}
}

@Test
void shouldNotDownloadEncryptedDatasetWhenFailureOnAllGateways() {
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(IPFS_DATASET_URL).build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
final URL resourceFile = this.getClass().getClassLoader().getResource(DATASET_RESOURCE_NAME);
assertThat(resourceFile).isNotNull();
try (MockedStatic<FileHelper> fileHelperMock = mockStatic(FileHelper.class)) {
fileHelperMock.when(() -> FileHelper.readFileBytesFromUrl(anyString()))
.thenReturn(null);
assertThrows(
PreComputeException.class,
() -> preComputeApp.downloadEncryptedDataset()
);
fileHelperMock.verify(() -> FileHelper.readFileBytesFromUrl(anyString()), times(3));
}
}

@Test
void shouldThrowSinceDatasetChecksumNotValid() {
PreComputeArgs preComputeArgs = goodPreComputeArgs();
preComputeArgs.setEncryptedDatasetChecksum("badChecksum");
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(HTTP_DATASET_URL)
.encryptedDatasetChecksum("badChecksum")
.build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
PreComputeException e = assertThrows(
PreComputeException.class,
Expand All @@ -169,19 +243,21 @@ void shouldThrowSinceDatasetChecksumNotValid() {

@Test
void shouldDecryptDataset() throws Exception {
doReturn(goodPreComputeArgs()).when(preComputeApp).getPreComputeArgs();
byte[] encryptedData = FileHelper.readFileBytesFromUrl(DATASET_URL);
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(HTTP_DATASET_URL).build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
byte[] encryptedData = FileHelper.readFileBytesFromUrl(HTTP_DATASET_URL);
byte[] expectedPlainData = FileHelper.readAllBytes(PLAIN_DATA_FILE);
byte[] actualPlainData = preComputeApp.decryptDataset(encryptedData);
assertThat(actualPlainData).isEqualTo(expectedPlainData);
}

@Test
void shouldThrowSinceDecryptionFailed() {
PreComputeArgs preComputeArgs = goodPreComputeArgs();
String badKey = FileHelper.readFile(KEY_FILE).replace("A", "B");
preComputeArgs.setEncryptedDatasetBase64Key(badKey);
byte[] encryptedData = FileHelper.readFileBytesFromUrl(DATASET_URL);
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(HTTP_DATASET_URL)
.encryptedDatasetBase64Key(badKey)
.build();
byte[] encryptedData = FileHelper.readFileBytesFromUrl(HTTP_DATASET_URL);
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
PreComputeException e = assertThrows(
PreComputeException.class,
Expand All @@ -191,16 +267,18 @@ void shouldThrowSinceDecryptionFailed() {

@Test
void shouldSavePlainDatasetFile() throws Exception {
doReturn(goodPreComputeArgs()).when(preComputeApp).getPreComputeArgs();
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(HTTP_DATASET_URL).build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
byte[] plainContent = FileHelper.readAllBytes(PLAIN_DATA_FILE);
preComputeApp.savePlainDatasetFile(plainContent);
assertThat(new File(outputDir, DATASET_FILENAME)).exists();
}

@Test
void shouldThrowSinceFailedToSavePlainDataset() {
PreComputeArgs preComputeArgs = goodPreComputeArgs();
preComputeArgs.setOutputDir("/some-folder-123/not-found");
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(HTTP_DATASET_URL)
.outputDir("/some-folder-123/not-found")
.build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
PreComputeException e = assertThrows(PreComputeException.class,
() -> preComputeApp.savePlainDatasetFile("data".getBytes()));
Expand All @@ -209,33 +287,34 @@ void shouldThrowSinceFailedToSavePlainDataset() {

@Test
void shouldDownloadInputFiles() throws Exception {
doReturn(goodPreComputeArgs()).when(preComputeApp).getPreComputeArgs();
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(HTTP_DATASET_URL).build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
preComputeApp.downloadInputFiles();
assertThat(new File(outputDir, "plain-data.txt")).exists();
assertThat(new File(outputDir, "key.txt")).exists();
}

@Test
void shouldThrowSinceFailedToDownloadInputFiles() {
PreComputeArgs preComputeArgs = goodPreComputeArgs();
preComputeArgs.setInputFiles(List.of("http://bad-input-file-url"));
final PreComputeArgs preComputeArgs = getPreComputeArgsBuilder(HTTP_DATASET_URL)
.inputFiles(List.of("http://bad-input-file-url"))
.build();
doReturn(preComputeArgs).when(preComputeApp).getPreComputeArgs();
PreComputeException e = assertThrows(PreComputeException.class,
() -> preComputeApp.downloadInputFiles());
assertThat(e.getExitCause()).isEqualTo(ReplicateStatusCause.PRE_COMPUTE_INPUT_FILE_DOWNLOAD_FAILED);
}

private PreComputeArgs goodPreComputeArgs() {
private PreComputeArgs.PreComputeArgsBuilder getPreComputeArgsBuilder(String datasetUrl) {
return PreComputeArgs.builder()
.chainTaskId(CHAIN_TASK_ID)
.outputDir(outputDir.getAbsolutePath())
.isDatasetRequired(true)
.encryptedDatasetUrl(DATASET_URL)
.encryptedDatasetUrl(datasetUrl)
.encryptedDatasetBase64Key(FileHelper.readFile(KEY_FILE))
.encryptedDatasetChecksum(DATASET_CHECKSUM)
.plainDatasetFilename(DATASET_FILENAME)
.inputFiles(List.of(INPUT_FILE_1_URL, INPUT_FILE_2_URL))
.build();
.inputFiles(List.of(INPUT_FILE_1_URL, INPUT_FILE_2_URL));
}

}

0 comments on commit 114fd4c

Please sign in to comment.