diff --git a/Java/PoolAndResourceFile/src/main/java/PoolAndResourceFile.java b/Java/PoolAndResourceFile/src/main/java/PoolAndResourceFile.java index 53483da1..7e6f890e 100644 --- a/Java/PoolAndResourceFile/src/main/java/PoolAndResourceFile.java +++ b/Java/PoolAndResourceFile/src/main/java/PoolAndResourceFile.java @@ -124,13 +124,14 @@ private static CloudPool createPoolIfNotExists(BatchClient client, String poolId * storage account name * @param storageAccountKey * storage account key + * @param containerName + * container name * @return CloudBlobContainer instance * @throws URISyntaxException * @throws StorageException */ - private static CloudBlobContainer createBlobContainer(String storageAccountName, String storageAccountKey) + private static CloudBlobContainer createBlobContainer(String storageAccountName, String storageAccountKey, String containerName) throws URISyntaxException, StorageException { - String CONTAINER_NAME = "poolsandresourcefiles"; // Create storage credential from name and key StorageCredentials credentials = new StorageCredentialsAccountAndKey(storageAccountName, storageAccountKey); @@ -144,7 +145,28 @@ private static CloudBlobContainer createBlobContainer(String storageAccountName, // Get a reference to a container. // The container name must be lower case - return blobClient.getContainerReference(CONTAINER_NAME); + return blobClient.getContainerReference(containerName); + } + + /** + * Get shared access blob policy with access permissions and expiry to be used to create sas token + * + * @param sharedAccessBlobPermissions + * @param expiry + * expiry in number of days + * @return + * shared access blob policy with permission and expiry + */ + private static SharedAccessBlobPolicy getSharedAccessBlobPolicy(SharedAccessBlobPermissions sharedAccessBlobPermissions, int expiry) { + SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy(); + EnumSet perEnumSet = EnumSet.of(sharedAccessBlobPermissions); + policy.setPermissions(perEnumSet); + + Calendar c = Calendar.getInstance(); + c.setTime(new Date()); + c.add(Calendar.DATE, expiry); + policy.setSharedAccessExpiryTime(c.getTime()); + return policy; } /** @@ -173,27 +195,54 @@ private static String uploadFileToCloud(CloudBlobContainer container, String fil blob.upload(new FileInputStream(source), source.length()); // Create policy with 1 day read permission - SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy(); - EnumSet perEnumSet = EnumSet.of(SharedAccessBlobPermissions.READ); - policy.setPermissions(perEnumSet); - - Calendar c = Calendar.getInstance(); - c.setTime(new Date()); - c.add(Calendar.DATE, 1); - policy.setSharedAccessExpiryTime(c.getTime()); + SharedAccessBlobPolicy policy = getSharedAccessBlobPolicy(SharedAccessBlobPermissions.READ, 1); // Create SAS key String sas = blob.generateSharedAccessSignature(policy, null); return blob.getUri() + "?" + sas; } + /** + * Creates a list of output files matching a file pattern, these files will be uploaded to azure container after task completion + * + * @param container + * container to upload the file + * @param filePattern + * all the files matching the given pattern will be upload to container + * @return list of output files + * @throws StorageException + * @throws InvalidKeyException + */ + private static List createOutputFiles(CloudBlobContainer container, String filePattern) throws StorageException, InvalidKeyException { + // Create policy with 1 day write permission + SharedAccessBlobPolicy policy = getSharedAccessBlobPolicy(SharedAccessBlobPermissions.WRITE, 1); + String sasToken = container.generateSharedAccessSignature(policy, null); + String containerSasUrl = String.format("%s?%s", container.getUri().toString(), sasToken); + OutputFileBlobContainerDestination outputFileBlobContainerDestination = + new OutputFileBlobContainerDestination() + .withContainerUrl(containerSasUrl); + + List outputFiles = new ArrayList<>(); + OutputFile outputFile = new OutputFile() + .withFilePattern(filePattern) + .withDestination(new OutputFileDestination() + .withContainer(outputFileBlobContainerDestination)) + .withUploadOptions(new OutputFileUploadOptions() + .withUploadCondition(OutputFileUploadCondition.TASK_COMPLETION)); + + outputFiles.add(outputFile); + return outputFiles; + } + /** * Create a job with a single task * * @param client * batch client instance - * @param container + * @param resourceFileUploadContainer * blob container to upload the resource file + * @param outputFileUploadContainer + * blob container to upload the output files generated during task execution * @param poolId * pool id * @param jobId @@ -204,7 +253,7 @@ private static String uploadFileToCloud(CloudBlobContainer container, String fil * @throws InvalidKeyException * @throws URISyntaxException */ - private static void submitJobAndAddTask(BatchClient client, CloudBlobContainer container, String poolId, + private static void submitJobAndAddTask(BatchClient client, CloudBlobContainer resourceFileUploadContainer, CloudBlobContainer outputFileUploadContainer, String poolId, String jobId) throws BatchErrorException, IOException, StorageException, InvalidKeyException, URISyntaxException { String BLOB_FILE_NAME = "test.txt"; @@ -217,9 +266,11 @@ private static void submitJobAndAddTask(BatchClient client, CloudBlobContainer c // Create task TaskAddParameter taskToAdd = new TaskAddParameter(); - taskToAdd.withId("mytask").withCommandLine(String.format("cat %s", BLOB_FILE_NAME)); + // Create files named test2.txt and test3.csv. later we can transfer these files from vm to storage container using a file pattern + String command = String.format("cat %s && cp %s test2.txt && cp %s test3.csv", BLOB_FILE_NAME, BLOB_FILE_NAME, BLOB_FILE_NAME); + taskToAdd.withId("mytask").withCommandLine(String.format("/bin/bash -c \"%s\"", command)); - String sas = uploadFileToCloud(container, BLOB_FILE_NAME, LOCAL_FILE_PATH); + String sas = uploadFileToCloud(resourceFileUploadContainer, BLOB_FILE_NAME, LOCAL_FILE_PATH); // Associate resource file with task ResourceFile file = new ResourceFile(); @@ -228,6 +279,9 @@ private static void submitJobAndAddTask(BatchClient client, CloudBlobContainer c files.add(file); taskToAdd.withResourceFiles(files); + // + taskToAdd.withOutputFiles(getOutputFiles(outputFileUploadContainer, "*.txt")); + // Add task to job client.taskOperations().createTask(jobId, taskToAdd); } @@ -319,7 +373,9 @@ public static void main(String argv[]) throws Exception { BatchClient client = BatchClient.open(cred); // Create storage container - CloudBlobContainer container = createBlobContainer(storageAccountName, storageAccountKey); + CloudBlobContainer resourceFileUploadContainer = createBlobContainer(storageAccountName, storageAccountKey, "poolsandresourcefiles"); + CloudBlobContainer taskOutputFileUploadContainer = createBlobContainer(storageAccountName, storageAccountKey, "taskoutputfiles"); + taskOutputFileUploadContainer.createIfNotExists(); String userName = System.getProperty("user.name"); String poolId = userName + "-pooltest"; @@ -328,8 +384,12 @@ public static void main(String argv[]) throws Exception { try { CloudPool sharedPool = createPoolIfNotExists(client, poolId); - submitJobAndAddTask(client, container, sharedPool.id(), jobId); + submitJobAndAddTask(client, resourceFileUploadContainer, taskOutputFileUploadContainer, sharedPool.id(), jobId); if (waitForTasksToComplete(client, jobId, TASK_COMPLETE_TIMEOUT)) { + // Check of the task output files uploaded to container + for (ListBlobItem listBlobItem : taskOutputFileUploadContainer.listBlobs()) { + System.out.println("Uri of uploaded files after task completion: " + listBlobItem.getUri().toString()); + } // Get the task command output file CloudTask task = client.taskOperations().getTask(jobId, "mytask"); @@ -363,7 +423,8 @@ public static void main(String argv[]) throws Exception { } if (shouldDeleteContainer) { - container.deleteIfExists(); + resourceFileUploadContainer.deleteIfExists(); + taskOutputFileUploadContainer.deleteIfExists(); } } }