Skip to content

Commit

Permalink
Merge pull request #465 from iExecBlockchainComputing/release/8.6.0
Browse files Browse the repository at this point in the history
Release/8.6.0
  • Loading branch information
jbern0rd authored Dec 19, 2024
2 parents b56e85f + de60f74 commit ffbe625
Show file tree
Hide file tree
Showing 11 changed files with 208 additions and 130 deletions.
16 changes: 16 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,22 @@

All notable changes to this project will be documented in this file.

## [[8.6.0]](https://github.com/iExecBlockchainComputing/iexec-common/releases/tag/v8.6.0) 2024-12-19

### New Features

- Add `createFileNameFromUri` to `FileHashUtils`, use it in `IexecEnvUtils`. (#463)

### Quality

- Update `HTTPS_FILE_DIGEST` in `FileHelperTests`. (#460)
- Refactor `IexecEnvUtils` to use `dealParams` instead of deprecated `TaskDescription` fields. (#462)

### Dependency Upgrades

- Upgrade to Gradle 8.10.2. (#461)
- Upgrade to `iexec-commons-poco` 4.2.0. (#464)

## [[8.5.0]](https://github.com/iExecBlockchainComputing/iexec-common/releases/tag/v8.5.0) 2024-06-17

### New Features
Expand Down
4 changes: 2 additions & 2 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
plugins {
id 'java-library'
id 'io.freefair.lombok' version '8.6'
id 'io.freefair.lombok' version '8.10.2'
id 'jacoco'
id 'org.sonarqube' version '5.0.0.4638'
id 'org.sonarqube' version '5.1.0.4882'
id 'maven-publish'
}

Expand Down
4 changes: 2 additions & 2 deletions gradle.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
version=8.5.0
iexecCommonsPocoVersion=4.1.0
version=8.6.0
iexecCommonsPocoVersion=4.2.0

nexusUser
nexusPassword
Binary file modified gradle/wrapper/gradle-wrapper.jar
Binary file not shown.
2 changes: 1 addition & 1 deletion gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip
networkTimeout=10000
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
Expand Down
7 changes: 5 additions & 2 deletions gradlew
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0
#

##############################################################################
#
Expand Down Expand Up @@ -55,7 +57,7 @@
# Darwin, MinGW, and NonStop.
#
# (3) This script is generated from the Groovy template
# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
# within the Gradle project.
#
# You can find Gradle at https://github.com/gradle/gradle/.
Expand Down Expand Up @@ -84,7 +86,8 @@ done
# shellcheck disable=SC2034
APP_BASE_NAME=${0##*/}
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
' "$PWD" ) || exit

# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum
Expand Down
2 changes: 2 additions & 0 deletions gradlew.bat
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem
@rem SPDX-License-Identifier: Apache-2.0
@rem

@if "%DEBUG%"=="" @echo off
@rem ##########################################################################
Expand Down
42 changes: 29 additions & 13 deletions src/main/java/com/iexec/common/utils/FileHashUtils.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020-2023 IEXEC BLOCKCHAIN TECH
* Copyright 2020-2024 IEXEC BLOCKCHAIN TECH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -17,27 +17,37 @@
package com.iexec.common.utils;

import com.iexec.commons.poco.utils.HashUtils;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;

import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;

@Slf4j
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class FileHashUtils {

private FileHashUtils() {
throw new UnsupportedOperationException();
/**
* Creates a file name from a URI SHA-256 hash.
*
* @param uri File name to hash
* @return A file name generated from a SHA-256 hash
*/
public static String createFileNameFromUri(final String uri) {
return HashUtils.sha256(uri);
}

/**
* Generates SHA-256 digest for the given file.
* Generates SHA-256 digest for the given file content.
*
* @param file
* @param file The file whose content will be hashed
* @return SHA-256 digest in hex string format,
* e.g: 0x66daf4e6810d83d4859846a5df1afabf88c9fda135bc732ea977f25348d98ede
*/
public static String sha256(File file) {
public static String sha256(final File file) {
Objects.requireNonNull(file, "File must not be null");
String filepath = file.getAbsolutePath();
if (!file.exists()) {
Expand All @@ -52,8 +62,14 @@ public static String sha256(File file) {
return HashUtils.sha256(fileBytes);
}

public static String getFileTreeSha256(String fileTreePath) {
File fileTree = new File(fileTreePath);
/**
* Generate a hash of a file tree.
*
* @param fileTreePath File tree to hash
* @return A 256-bits hash of the file tree
*/
public static String getFileTreeSha256(final String fileTreePath) {
final File fileTree = new File(fileTreePath);
if (!fileTree.exists()) {
return "";
}
Expand All @@ -62,11 +78,11 @@ public static String getFileTreeSha256(String fileTreePath) {
return sha256(fileTree);
}
//fileTree is a tree, with multiple files
File[] files = fileTree.listFiles();
final File[] files = fileTree.listFiles();
if (files != null) {
List<String> hashes = new ArrayList<>();
java.util.Arrays.sort(files); // /!\ files MUST be sorted to ensure final concatenateAndHash(..) is always the same (order matters)
for (File file : files) {
final List<String> hashes = new ArrayList<>();
Arrays.sort(files); // /!\ files MUST be sorted to ensure final concatenateAndHash(..) is always the same (order matters)
for (final File file : files) {
hashes.add(sha256(file));
}
return HashUtils.concatenateAndHash(hashes.toArray(new String[0]));
Expand Down
102 changes: 54 additions & 48 deletions src/main/java/com/iexec/common/utils/IexecEnvUtils.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020-2023 IEXEC BLOCKCHAIN TECH
* Copyright 2020-2024 IEXEC BLOCKCHAIN TECH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -17,107 +17,113 @@
package com.iexec.common.utils;

import com.iexec.commons.poco.task.TaskDescription;
import org.apache.commons.io.FilenameUtils;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* Provides environment variables for worker {@code AppComputeService} and SMS {@code SecretSessionBaseService}
*
* @see <a href="https://protocol.docs.iex.ec/for-developers/technical-references/application-io#runtime-variables">
* Runtime variables in protocol documentation</a>
*/
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class IexecEnvUtils {

/*
* Env variables that will be injected in the container of
* a task computation.
*
* /!\ If you change those values please don't forget to update
* the palaemon config file.
*/
public static final String IEXEC_TASK_ID = "IEXEC_TASK_ID";
public static final String IEXEC_IN = "IEXEC_IN";
public static final String IEXEC_OUT = "IEXEC_OUT";
// BoT
public static final String IEXEC_BOT_TASK_INDEX = "IEXEC_BOT_TASK_INDEX";
public static final String IEXEC_BOT_SIZE = "IEXEC_BOT_SIZE";
public static final String IEXEC_BOT_FIRST_INDEX = "IEXEC_BOT_FIRST_INDEX";
// dataset
public static final String IEXEC_DATASET_ADDRESS = "IEXEC_DATASET_ADDRESS";
public static final String IEXEC_DATASET_URL = "IEXEC_DATASET_URL";
public static final String IEXEC_DATASET_FILENAME = "IEXEC_DATASET_FILENAME";
public static final String IEXEC_DATASET_CHECKSUM = "IEXEC_DATASET_CHECKSUM";
// BoT
public static final String IEXEC_BOT_TASK_INDEX = "IEXEC_BOT_TASK_INDEX";
public static final String IEXEC_BOT_SIZE = "IEXEC_BOT_SIZE";
public static final String IEXEC_BOT_FIRST_INDEX = "IEXEC_BOT_FIRST_INDEX";
// input files
public static final String IEXEC_INPUT_FILES_NUMBER = "IEXEC_INPUT_FILES_NUMBER";
public static final String IEXEC_INPUT_FILES_FOLDER = "IEXEC_INPUT_FILES_FOLDER";
public static final String IEXEC_INPUT_FILE_NAME_PREFIX = "IEXEC_INPUT_FILE_NAME_";
public static final String IEXEC_INPUT_FILE_URL_PREFIX = "IEXEC_INPUT_FILE_URL_";
// secrets
public static final String IEXEC_APP_DEVELOPER_SECRET_PREFIX = "IEXEC_APP_DEVELOPER_SECRET_";
public static final String IEXEC_REQUESTER_SECRET_PREFIX = "IEXEC_REQUESTER_SECRET_";

private IexecEnvUtils() {
throw new UnsupportedOperationException();
}

/**
* Get compute stage environment variables plus other additional ones
* used by the pre-compute stage (e.g. IEXEC_DATASET_URL,
* IEXEC_DATASET_CHECKSUM, IEXEC_INPUT_FILE_URL_1, ...etc).
*
* @param taskDescription
* @return
* Get compute stage environment variables plus additional ones used by the pre-compute stage
* <p>
* Pre-compute stage specific variables:
* IEXEC_DATASET_URL, IEXEC_DATASET_CHECKSUM, IEXEC_INPUT_FILE_URL_1, ...
*
* @param taskDescription The description of the task
* @return A key-value map containing each environment variable and its associated value
* @see #getComputeStageEnvMap(TaskDescription)
*/
public static Map<String, String> getAllIexecEnv(TaskDescription taskDescription) {
Map<String, String> envMap = new HashMap<>();
public static Map<String, String> getAllIexecEnv(final TaskDescription taskDescription) {
final Map<String, String> envMap = new HashMap<>();
envMap.putAll(getComputeStageEnvMap(taskDescription));
envMap.put(IEXEC_DATASET_URL, taskDescription.getDatasetUri());
envMap.put(IEXEC_DATASET_CHECKSUM, taskDescription.getDatasetChecksum());
if (taskDescription.getInputFiles() == null) {
if (!taskDescription.containsInputFiles()) {
return envMap;
}
int index = 1;
for (String inputFileUrl : taskDescription.getInputFiles()) {
for (final String inputFileUrl : taskDescription.getDealParams().getIexecInputFiles()) {
envMap.put(IEXEC_INPUT_FILE_URL_PREFIX + index, inputFileUrl);
index++;
}
return envMap;
}

/**
* Get environment variables available for the compute stage
* (e.g. IEXEC_TASK_ID, IEXEC_IN, IEXEC_OUT,
* IEXEC_DATASET_FILENAME, IEXEC_INPUT_FILE_NAME_1, ...etc).
*
* @param taskDescription
* @return
* Get environment variables available for the compute stage.
* <p>
* Compute stage specific variables :
* IEXEC_TASK_ID, IEXEC_IN, IEXEC_OUT, IEXEC_DATASET_FILENAME, IEXEC_INPUT_FILE_NAME_1, ...
*
* @param taskDescription The description of the task
* @return A key-value map containing each environment variable and its associated value
*/
public static Map<String, String> getComputeStageEnvMap(TaskDescription taskDescription) {
Map<String, String> map = new HashMap<>();
public static Map<String, String> getComputeStageEnvMap(final TaskDescription taskDescription) {
final Map<String, String> map = new HashMap<>();
map.put(IEXEC_TASK_ID, taskDescription.getChainTaskId());
map.put(IEXEC_IN, IexecFileHelper.SLASH_IEXEC_IN);
map.put(IEXEC_OUT, IexecFileHelper.SLASH_IEXEC_OUT);
// dataset
map.put(IEXEC_DATASET_ADDRESS, taskDescription.getDatasetAddress());
map.put(IEXEC_DATASET_FILENAME, taskDescription.getDatasetAddress());
// BoT
map.put(IEXEC_BOT_SIZE, String.valueOf(taskDescription.getBotSize()));
map.put(IEXEC_BOT_FIRST_INDEX, String.valueOf(taskDescription.getBotFirstIndex()));
map.put(IEXEC_BOT_TASK_INDEX, String.valueOf(taskDescription.getBotIndex()));
// dataset
map.put(IEXEC_DATASET_ADDRESS, taskDescription.getDatasetAddress());
map.put(IEXEC_DATASET_FILENAME, taskDescription.getDatasetAddress());
// input files
int nbFiles = taskDescription.getInputFiles() == null ? 0 : taskDescription.getInputFiles().size();
map.put(IEXEC_INPUT_FILES_NUMBER, String.valueOf(nbFiles));
map.put(IEXEC_INPUT_FILES_FOLDER, IexecFileHelper.SLASH_IEXEC_IN);
if (nbFiles == 0) {
if (!taskDescription.containsInputFiles()) {
map.put(IEXEC_INPUT_FILES_NUMBER, "0");
return map;
}
// We are sure input files are present
final List<String> inputFiles = taskDescription.getDealParams().getIexecInputFiles();
map.put(IEXEC_INPUT_FILES_FOLDER, IexecFileHelper.SLASH_IEXEC_IN);
map.put(IEXEC_INPUT_FILES_NUMBER, String.valueOf(inputFiles.size()));
int index = 1;
for (String inputFileUrl : taskDescription.getInputFiles()) {
map.put(IEXEC_INPUT_FILE_NAME_PREFIX + index, FilenameUtils.getName(inputFileUrl));
for (final String inputFileUrl : inputFiles) {
map.put(IEXEC_INPUT_FILE_NAME_PREFIX + index, FileHashUtils.createFileNameFromUri(inputFileUrl));
index++;
}
return map;
}

public static List<String> getComputeStageEnvList(TaskDescription taskDescription) {
/**
* Get compute stage environment variable for docker-java API.
*
* @param taskDescription The description of the task
* @return A list of KEY=VALUE strings corresponding to environment variables to run a standard app
* @see #getComputeStageEnvMap(TaskDescription)
*/
public static List<String> getComputeStageEnvList(final TaskDescription taskDescription) {
List<String> list = new ArrayList<>();
getComputeStageEnvMap(taskDescription).forEach((key, value) -> list.add(key + "=" + value));
return list;
Expand Down
4 changes: 2 additions & 2 deletions src/test/java/com/iexec/common/utils/FileHelperTests.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020-2023 IEXEC BLOCKCHAIN TECH
* Copyright 2020-2024 IEXEC BLOCKCHAIN TECH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -43,7 +43,7 @@ class FileHelperTests {
"/wikipedia/commons/thumb/6/65/600px_Black_bordered_HEX-0082D6.svg/600px-600px_Black_bordered_HEX-0082D6.svg.png";
// private static final String HTTPS_FILENAME = "token.svg";
private static final String HTTPS_FILE_DIGEST =
"0xa7e1a706b2d60dad840c56594662deb37061815fd3ae2a04c1ca6328b0683fe6";
"0x45a4b7a3666f3885f3e5ac0e8c865c6f841890d2beb8d279c2bab9578403e05f";
// redirection
// private static final String REDIRECTION_URL = "https://goo.gl/t8JxoX";
// private static final String REDIRECTION_FILE_DIGEST = "TODO";
Expand Down
Loading

0 comments on commit ffbe625

Please sign in to comment.