aws: use paginator API, fix timestamp serialization (#156)

Author: ibodrov

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Change log
 
+## [2.3.1] - Unreleased
+
+### Changed
+
+- aws: use paginator API, fix timestamp serialization
+([#155](https://github.com/walmartlabs/concord-plugins/pull/155)).
+
+
+
 ## [2.3.0] - 2024-06-20
 
 - aws: add a basic ecr describe-images wrapper

tasks/aws/examples/ecr/concord.yml

@@ -1,7 +1,7 @@
 configuration:
   runtime: concord-v2
   dependencies:
-    - mvn://com.walmartlabs.concord.plugins:aws-tasks:2.3.0
+    - mvn://com.walmartlabs.concord.plugins:aws-tasks:2.3.1
 
 flows:
   default:
@@ -10,5 +10,6 @@ flows:
         action: describe-images
         region: us-east-1
         repositoryName: foo
+        maxResults: 1
       out: result
     - log: "Image Details: ${resource.prettyPrintJson(result.imageDetails)}"

tasks/aws/pom.xml

@@ -33,6 +33,10 @@
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>ecr</artifactId>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>sdk-core</artifactId>
+        </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>aws-core</artifactId>

tasks/aws/src/main/java/com/walmartlabs/concord/plugins/aws/EcrTask.java

@@ -23,13 +23,16 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import com.walmartlabs.concord.runtime.v2.sdk.Context;
 import com.walmartlabs.concord.runtime.v2.sdk.Task;
 import com.walmartlabs.concord.runtime.v2.sdk.TaskResult;
 import com.walmartlabs.concord.runtime.v2.sdk.Variables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.ecr.EcrClient;
+import software.amazon.awssdk.services.ecr.model.DescribeImagesRequest;
+import software.amazon.awssdk.services.ecr.model.ImageDetail;
 
 import javax.inject.Inject;
 import javax.inject.Named;
@@ -42,13 +45,16 @@ public class EcrTask implements Task {
 
     private static final Logger log = LoggerFactory.getLogger(EcrTask.class);
 
+    private final Context context;
     private final ObjectMapper objectMapper;
 
     @Inject
-    public EcrTask(ObjectMapper objectMapper) {
+    public EcrTask(Context context, ObjectMapper objectMapper) {
+        this.context = context;
         this.objectMapper = requireNonNull(objectMapper).copy()
                 .registerModule(new JavaTimeModule())
-                .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+                .configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false)
+                .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
     }
 
     @Override
@@ -63,29 +69,38 @@ public TaskResult execute(Variables input) {
     private TaskResult describeImages(Variables input) {
         var region = assertRegion(input, "region");
         var repositoryName = input.assertString("repositoryName");
-        var verbose = input.getBoolean("verbose", false);
+        var maxResults = input.getInt("maxResults", 100);
+        var debug = input.getBoolean("debug", context.processConfiguration().debug());
 
-        // create the client
-        if (verbose) {
-            log.info("Using region: {}", region);
+        if (debug) {
+            log.info("Using region={}, maxResults={}", region, maxResults);
         }
-        var client = EcrClient.builder()
+
+        try (var client = EcrClient.builder()
                 .region(region)
-                .build();
+                .build()) {
 
-        // describe-images
-        if (verbose) {
-            log.info("Describing images in repository '{}'", repositoryName);
-        }
-        var result = client.describeImages(r -> r.repositoryName(repositoryName));
-        if (verbose) {
-            log.info("Done: {}", result.imageDetails().size());
-        }
+            if (debug) {
+                log.info("Describing images in repository '{}'", repositoryName);
+            }
+
+            var request = DescribeImagesRequest.builder()
+                    .repositoryName(repositoryName)
+                    .maxResults(maxResults)
+                    .build();
 
-        // serialize result into POJOs
-        var data = objectMapper.convertValue(result.toBuilder(), Map.class);
-        //noinspection unchecked
-        return TaskResult.success().values(data);
+            var data = client.describeImagesPaginator(request).stream()
+                    .flatMap(response -> response.imageDetails().stream())
+                    .map(ImageDetail::toBuilder)
+                    .map(b -> (Map<?, ?>) objectMapper.convertValue(b, Map.class))
+                    .toList();
+
+            if (debug) {
+                log.info("Done: {}", data.size());
+            }
+
+            return TaskResult.success().values(Map.of("imageDetails", data));
+        }
     }
 
     private static Region assertRegion(Variables input, String key) {

tasks/aws/src/test/java/com/walmartlabs/concord/plugins/aws/EcrTaskTest.java

@@ -36,7 +36,7 @@ public class EcrTaskTest {
 
     @Test
     public void testDescribeImages() {
-        var task = new EcrTask(new ObjectMapper());
+        var task = new EcrTask(new MockContext(), new ObjectMapper());
         var input = new MapBackedVariables(Map.of(
                 "action", "describe-images",
                 "region", "us-east-1",

tasks/aws/src/test/java/com/walmartlabs/concord/plugins/aws/MockContext.java

@@ -0,0 +1,121 @@
+package com.walmartlabs.concord.plugins.aws;
+
+/*-
+ * *****
+ * Concord
+ * -----
+ * Copyright (C) 2017 - 2021 Walmart Inc., Concord Authors
+ * -----
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * =====
+ */
+
+import com.walmartlabs.concord.runtime.v2.sdk.Compiler;
+import com.walmartlabs.concord.runtime.v2.sdk.*;
+
+import java.io.Serializable;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.UUID;
+
+public class MockContext implements Context {
+    private final Variables variables;
+    private final Variables defaultVariables;
+
+    public MockContext() {
+        this(Map.of(), Map.of());
+    }
+
+    public MockContext(Map<String, Object> vars, Map<String, Object> defs) {
+        this.variables = new MapBackedVariables(vars);
+        this.defaultVariables = new MapBackedVariables(defs);
+    }
+
+    @Override
+    public Path workingDirectory() {
+        return null;
+    }
+
+    @Override
+    public UUID processInstanceId() {
+        return null;
+    }
+
+    @Override
+    public Variables variables() {
+        return variables;
+    }
+
+    @Override
+    public Variables defaultVariables() {
+        return defaultVariables;
+    }
+
+    @Override
+    public FileService fileService() {
+        return null;
+    }
+
+    @Override
+    public DockerService dockerService() {
+        return null;
+    }
+
+    @Override
+    public SecretService secretService() {
+        return null;
+    }
+
+    @Override
+    public LockService lockService() {
+        return null;
+    }
+
+    @Override
+    public ApiConfiguration apiConfiguration() {
+        return null;
+    }
+
+    @Override
+    public ProcessConfiguration processConfiguration() {
+        return ProcessConfiguration.builder().build();
+    }
+
+    @Override
+    public Execution execution() {
+        return null;
+    }
+
+    @Override
+    public Compiler compiler() {
+        return null;
+    }
+
+    @Override
+    public <T> T eval(Object o, Class<T> aClass) {
+        return null;
+    }
+
+    @Override
+    public <T> T eval(Object o, Map<String, Object> map, Class<T> aClass) {
+        return null;
+    }
+
+    @Override
+    public void suspend(String s) {
+    }
+
+    @Override
+    public void reentrantSuspend(String s, Map<String, Serializable> map) {
+    }
+}