From 94289963bf597f950cc85ece9311ffc1e9949115 Mon Sep 17 00:00:00 2001 From: Kiran Godishala <53332225+kirangodishala@users.noreply.github.com> Date: Sat, 28 Sep 2024 01:41:34 +0530 Subject: [PATCH] feat(clouddriver): make fetching properties file more resilient for k8s jobs (#4783) * tests(orca/clouddriver): refactor tests to remove some test code duplications and make them more descriptive * feat(orca/clouddriver): make waitOnJobCompletion retries configurable * feat(clouddriver): make fetching properties file more resilient for k8s jobs If a k8s run job is marked as succeeded, and property file is defined in the stage context, then it can so happen that multiple pods are created for that job. See https://kubernetes.io/docs/concepts/workloads/controllers/job/#handling-pod-and-container-failures In extreme edge cases, the first pod may be around before the second one succeeds. That leads to kubectl logs job/ command failing as seen below: kubectl -n test logs job/test-run-job-5j2vl -c parser Found 2 pods, using pod/test-run-job-5j2vl-fj8hd Error from server (BadRequest): container "parser" in pod "test-run-job-5j2vl-fj8hd" is terminated or Found 2 pods, using pod/test-run-job-5j2vl-fj8hd Error from server (BadRequest): container "parser" in pod "test-run-job-5j2vl-fj8hd" is waiting to start: PodInitializing where that commands defaults to using one of the two pods. To fix this issue, if we encounter an error from the kubectl logs job/ command, we find a successful pod in the job and directly query it for logs. --------- Co-authored-by: Apoorv Mahajan Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../tasks/job/WaitOnJobCompletion.groovy | 227 ++++- .../DelegatingKatoRestService.java | 7 + .../orca/clouddriver/KatoRestService.java | 8 + .../config/TaskConfigurationProperties.java | 16 + .../tasks/job/WaitOnJobCompletionTest.java | 430 +++++----- .../failed-runjob-status.json | 0 ...njob-stage-context-with-property-file.json | 10 + ...-stage-context-without-property-file.json} | 2 +- ...sful-runjob-status-with-multiple-pods.json | 786 ++++++++++++++++++ .../successful-runjob-status.json | 0 .../tasks/job/runjob-context-success.json | 266 ------ .../tasks/job/runjob-stage-context.json | 8 - .../tasks/job/successful-run-job-state.json | 564 ------------- 13 files changed, 1279 insertions(+), 1045 deletions(-) rename orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/{ => kubernetes}/failed-runjob-status.json (100%) create mode 100644 orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json rename orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/{runjob-stage-context-with-property-file.json => kubernetes/runjob-stage-context-without-property-file.json} (77%) create mode 100644 orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json rename orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/{ => kubernetes}/successful-runjob-status.json (100%) delete mode 100644 orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/runjob-context-success.json delete mode 100644 orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/runjob-stage-context.json delete mode 100644 orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/successful-run-job-state.json diff --git a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy index 605bc07d71..4ef895c188 100644 --- a/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy +++ b/orca-clouddriver/src/main/groovy/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletion.groovy @@ -147,7 +147,11 @@ public class WaitOnJobCompletion implements CloudProviderAware, OverridableTimeo InputStream jobStream retrySupport.retry({ jobStream = katoRestService.collectJob(appName, account, location, name).body.in() - }, 6, 5000, false) // retry for 30 seconds + }, + configProperties.getJobStatusRetry().maxAttempts, + Duration.ofMillis(configProperties.getJobStatusRetry().getBackOffInMs()), + configProperties.getJobStatusRetry().exponentialBackoffEnabled + ) Map job = objectMapper.readValue(jobStream, new TypeReference() {}) outputs.jobStatus = job @@ -165,24 +169,16 @@ public class WaitOnJobCompletion implements CloudProviderAware, OverridableTimeo if ((status == ExecutionStatus.SUCCEEDED) || (status == ExecutionStatus.TERMINAL)) { if (stage.context.propertyFile) { - Map properties = [:] - try { - retrySupport.retry({ - properties = katoRestService.getFileContents(appName, account, location, name, stage.context.propertyFile) - }, 6, 5000, false) // retry for 30 seconds - } catch (Exception e) { - if (status == ExecutionStatus.SUCCEEDED) { - throw new ConfigurationException("Property File: ${stage.context.propertyFile} contents could not be retrieved. Error: " + e) - } - log.warn("failed to get file contents for ${appName}, account: ${account}, namespace: ${location}, " + - "manifest: ${name} from propertyFile: ${stage.context.propertyFile}. Error: ", e) - } - - if (properties.size() == 0) { - if (status == ExecutionStatus.SUCCEEDED) { - throw new ConfigurationException("Expected properties file ${stage.context.propertyFile} but it was either missing, empty or contained invalid syntax") - } - } else if (properties.size() > 0) { + Map properties = getPropertyFileContents( + job, + appName, + status, + account, + location, + name, + stage.context.propertyFile as String) + + if (properties.size() > 0) { outputs << properties outputs.propertyFileContents = properties } @@ -251,4 +247,197 @@ public class WaitOnJobCompletion implements CloudProviderAware, OverridableTimeo } throw new JobFailedException(errorMessage) } + + /** + *

this method attempts to get property file from clouddriver and then parses its contents. Depending + * on the job itself, it could be handled by any job provider in clouddriver. This method should only be + * called for jobs with ExecutionStatus as either SUCCEEDED or TERMINAL. + * + *

If property file contents could not be retrieved from clouddriver, then the error handling depends + * on the job's ExecutionStatus. If it is SUCCEEDED, then an exception is thrown. Otherwise, no exception + * is thrown since we don't want to mask the real reason behind the job failure. + * + *

If ExecutionStatus == SUCCEEDED, and especially for kubernetes run jobs, it can so happen that a user + * has configured the job spec to run 1 pod, have completions and parallelism == 1, and + * restartPolicy == Never. Despite that, kubernetes may end up running another pod as stated here: + * https://kubernetes.io/docs/concepts/workloads/controllers/job/#handling-pod-and-container-failures + * In such a scenario, it may so happen that two pods are created for that job. The first pod may still be + * around, such as in a PodInitializing state and the second pod could complete before the first one is + * terminated. This leads to the getFileContents() call failing, since under the covers, kubernetes job + * provider runs kubectl logs job/ command, which picks one out of the two pods to obtain the + * logs as seen here: + * + *

kubectl -n test logs job/test-run-job-5j2vl -c parser + * Found 2 pods, using pod/test-run-job-5j2vl-fj8hd + * Error from server (BadRequest): container "parser" in pod "test-run-job-5j2vl-fj8hd" is PodInitializing + * + *

That means, even if kubernetes and clouddriver marked the job as successful, since number of + * succeeded pods >= number of completions, the kubectl command shown above could still end using + * the failed pod for obtaining the logs. + * + *

To handle this case, if we get an error while making the getFileContents() call or if we don't receive + * any properties, then for kubernetes jobs, we figure out if the job status has any pod with phase + * SUCCEEDED. If we find such a pod, then we directly get the logs from this succeeded pod. Otherwise, + * we throw an exception as before. + * + *

we aren't handling the above case for ExecutionStatus == TERMINAL, because at that point, we wouldn't + * know which pod to query for properties file contents. It could so happen that all the pods in such a job + * have failed, then we would have to loop over each pod and see what it generated. Then if say, two pods + * generated different property values for the same key, which one do we choose? Bearing this complexity + * in mind, and knowing that for succeeded jobs, this solution actually helps prevent a pipeline failure, + * we are limiting this logic to succeeded jobs only for now. + * + * @param job - job status returned by clouddriver + * @param appName - application name where the job is run + * @param status - Execution status of the job. Should either be SUCCEEDED or TERMINAL + * @param account - account under which this job is run + * @param location - where this job is run + * @param name - name of the job + * @param propertyFile - file name to query from the job + * @return map of property file contents + */ + private Map getPropertyFileContents( + Map job, + String appName, + ExecutionStatus status, + String account, + String location, + String name, + String propertyFile + ) { + Map properties = [:] + try { + retrySupport.retry({ + properties = katoRestService.getFileContents(appName, account, location, name, propertyFile) + }, + configProperties.getFileContentRetry().maxAttempts, + Duration.ofMillis(configProperties.getFileContentRetry().getBackOffInMs()), + configProperties.getFileContentRetry().exponentialBackoffEnabled + ) + } catch (Exception e) { + log.warn("Error occurred while retrieving property file contents from job: ${name}" + + " in application: ${appName}, in account: ${account}, location: ${location}," + + " using propertyFile: ${propertyFile}. Error: ", e + ) + + // For succeeded kubernetes jobs, let's try one more time to get property file contents. + if (status == ExecutionStatus.SUCCEEDED) { + properties = getPropertyFileContentsForSucceededKubernetesJob( + job, + appName, + account, + location, + propertyFile + ) + if (properties.size() == 0) { + // since we didn't get any properties, we fail with this exception + throw new ConfigurationException("Expected properties file: ${propertyFile} in " + + "job: ${name}, application: ${appName}, location: ${location}, account: ${account} " + + "but it was either missing, empty or contained invalid syntax. Error: ${e}") + } + } + } + + if (properties.size() == 0) { + log.warn("Could not parse propertyFile: ${propertyFile} in job: ${name}" + + " in application: ${appName}, in account: ${account}, location: ${location}." + + " It is either missing, empty or contains invalid syntax" + ) + + // For succeeded kubernetes jobs, let's try one more time to get property file contents. + if (status == ExecutionStatus.SUCCEEDED) { + // let's try one more time to get properties from a kubernetes pod + properties = getPropertyFileContentsForSucceededKubernetesJob( + job, + appName, + account, + location, + propertyFile + ) + if (properties.size() == 0) { + // since we didn't get any properties, we fail with this exception + throw new ConfigurationException("Expected properties file: ${propertyFile} in " + + "job: ${name}, application: ${appName}, location: ${location}, account: ${account} " + + "but it was either missing, empty or contained invalid syntax") + } + } + } + return properties + } + + /** + * This method is supposed to be called from getPropertyFileContents(). This is only applicable for + * Kubernetes jobs. It finds a successful pod in the job and directly queries it for property file + * contents. + * + *

It is meant to handle the following case: + * + *

if ExecutionStatus == SUCCEEDED, and especially for kubernetes run jobs, it can so happen that a + * user has configured the job spec to run 1 pod, have completions and parallelism == 1, and + * restartPolicy == Never. Despite that, kubernetes may end up running another pod as stated here: + * https://kubernetes.io/docs/concepts/workloads/controllers/job/#handling-pod-and-container-failures + * In such a scenario, it may so happen that two pods are created for that job. The first pod may still be + * around, such as in a PodInitializing state and the second pod could complete before the first one is + * terminated. This leads to the getFileContents() call failing, since under the covers, kubernetes job + * provider runs kubectl logs job/ command, which picks one out of the two pods to obtain the + * logs as seen here: + * + *

kubectl -n test logs job/test-run-job-5j2vl -c parser + * Found 2 pods, using pod/test-run-job-5j2vl-fj8hd + * Error from server (BadRequest): container "parser" in pod "test-run-job-5j2vl-fj8hd" is PodInitializing + * + *

That means, even if kubernetes and clouddriver marked the job as successful, since number of + * succeeded pods >= number of completions, the kubectl command shown above could still end using + * the failed pod for obtaining the logs. + * + *

To handle this case, if we get an error while making the getFileContents() call or if we don't receive + * any properties, then for kubernetes jobs, we figure out if the job status has any pod with phase + * SUCCEEDED. If we find such a pod, then we directly get the logs from this succeeded pod. Otherwise, + * we throw an exception as before. + * + *

To keep it simple, and not worry about how to deal with property file + * contents obtained from various successful pods in a job, if that may happen, we simply query the first + * successful pod in that job. + * + * @param job - job status returned by clouddriver + * @param appName - application in which this job is run + * @param account - account under which this job is run + * @param namespace - where this job is run + * @param propertyFile - file name to query from the job + * @return map of property file contents + */ + private Map getPropertyFileContentsForSucceededKubernetesJob( + Map job, + String appName, + String account, + String namespace, + String propertyFile + ) { + Map properties = [:] + if (job.get("provider", "unknown") == "kubernetes") { + Optional succeededPod = job.get("pods", []) + .stream() + .filter({ Map pod -> pod.get("status", [:]).get("phase", "Running") == "Succeeded" + }) + .findFirst() + + if (succeededPod.isPresent()) { + String podName = (succeededPod.get() as Map).get("name") + retrySupport.retry({ + properties = katoRestService.getFileContentsFromKubernetesPod( + appName, + account, + namespace, + podName, + propertyFile + ) + }, + configProperties.getFileContentRetry().maxAttempts, + Duration.ofMillis(configProperties.getFileContentRetry().getBackOffInMs()), + configProperties.getFileContentRetry().exponentialBackoffEnabled + ) + } + } + return properties + } } diff --git a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java index 9b3e823542..f480a72b67 100644 --- a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java +++ b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/DelegatingKatoRestService.java @@ -71,6 +71,13 @@ public Map getFileContents( return getService().getFileContents(app, account, region, id, fileName); } + @Override + public Map getFileContentsFromKubernetesPod( + String app, String account, String namespace, String podName, String fileName) { + return getService() + .getFileContentsFromKubernetesPod(app, account, namespace, podName, fileName); + } + @Override public Task lookupTask(String id) { return getService().lookupTask(id); diff --git a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java index eac1ebce17..ebbb51af4d 100644 --- a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java +++ b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/KatoRestService.java @@ -73,6 +73,14 @@ Map getFileContents( @Path("id") String id, @Path("fileName") String fileName); + @GET("/applications/{app}/kubernetes/pods/{account}/{namespace}/{podName}/{fileName}") + Map getFileContentsFromKubernetesPod( + @Path("app") String app, + @Path("account") String account, + @Path("namespace") String namespace, + @Path("podName") String podName, + @Path("fileName") String fileName); + /** * This should _only_ be called if there is a problem retrieving the Task from * CloudDriverTaskStatusService (ie. a clouddriver replica). diff --git a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/config/TaskConfigurationProperties.java b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/config/TaskConfigurationProperties.java index 95d074334b..afaa4b4048 100644 --- a/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/config/TaskConfigurationProperties.java +++ b/orca-clouddriver/src/main/java/com/netflix/spinnaker/orca/clouddriver/config/TaskConfigurationProperties.java @@ -47,6 +47,22 @@ public static class WaitOnJobCompletionTaskConfig { * Default or empty set means that no keys will be excluded. */ private Set excludeKeysFromOutputs = Set.of(); + + private Retries jobStatusRetry = new Retries(); + + private Retries fileContentRetry = new Retries(); + + @Data + public static class Retries { + // total number of attempts + int maxAttempts = 6; + + // time in ms to wait before subsequent retry attempts + long backOffInMs = 5000; + + // flag to enable exponential backoff + boolean exponentialBackoffEnabled = false; + } } @Data diff --git a/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java b/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java index e9c58ed4a9..a03fad6349 100644 --- a/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java +++ b/orca-clouddriver/src/test/java/com/netflix/spinnaker/orca/clouddriver/tasks/job/WaitOnJobCompletionTest.java @@ -51,11 +51,6 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import okhttp3.MediaType; -import okhttp3.Protocol; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.ResponseBody; import org.apache.commons.io.IOUtils; import org.assertj.core.api.AssertionsForClassTypes; import org.junit.jupiter.api.BeforeEach; @@ -63,14 +58,12 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import retrofit.client.Response; import retrofit.mime.TypedByteArray; -import retrofit.mime.TypedInput; public final class WaitOnJobCompletionTest { private ObjectMapper objectMapper; - private RetrySupport retrySupport; private KatoRestService mockKatoRestService; - private JobUtils mockJobUtils; private ExecutionRepository mockExecutionRepository; private TaskConfigurationProperties configProperties; private Front50Service mockFront50Service; @@ -79,15 +72,22 @@ public final class WaitOnJobCompletionTest { @BeforeEach public void setup() { objectMapper = new ObjectMapper(); - retrySupport = new RetrySupport(); + RetrySupport retrySupport = new RetrySupport(); + mockKatoRestService = mock(KatoRestService.class); + JobUtils mockJobUtils = mock(JobUtils.class); + mockExecutionRepository = mock(ExecutionRepository.class); + mockFront50Service = mock(Front50Service.class); + configProperties = new TaskConfigurationProperties(); + TaskConfigurationProperties.WaitOnJobCompletionTaskConfig.Retries retries = + new TaskConfigurationProperties.WaitOnJobCompletionTaskConfig.Retries(); + retries.setMaxAttempts(3); + retries.setBackOffInMs(1); + configProperties.getWaitOnJobCompletionTask().setFileContentRetry(retries); + configProperties.getWaitOnJobCompletionTask().setJobStatusRetry(retries); configProperties .getWaitOnJobCompletionTask() .setExcludeKeysFromOutputs(Set.of("completionDetails")); - mockKatoRestService = mock(KatoRestService.class); - mockJobUtils = mock(JobUtils.class); - mockExecutionRepository = mock(ExecutionRepository.class); - mockFront50Service = mock(Front50Service.class); task = new WaitOnJobCompletion( @@ -116,8 +116,8 @@ void jobTimeoutSpecifiedByRunJobTask() { @Test void taskSearchJobByApplicationUsingContextApplication() { - retrofit.client.Response mockResponse = - new retrofit.client.Response( + Response mockResponse = + new Response( "test-url", 200, "test-reason", @@ -142,8 +142,8 @@ void taskSearchJobByApplicationUsingContextApplication() { @Test void taskSearchJobByApplicationUsingContextMoniker() { - retrofit.client.Response mockResponse = - new retrofit.client.Response( + Response mockResponse = + new Response( "test-url", 200, "test-reason", @@ -167,8 +167,8 @@ void taskSearchJobByApplicationUsingContextMoniker() { @Test void taskSearchJobByApplicationUsingParsedName() { - retrofit.client.Response mockResponse = - new retrofit.client.Response( + Response mockResponse = + new Response( "test-url", 200, "test-reason", @@ -191,8 +191,8 @@ void taskSearchJobByApplicationUsingParsedName() { @Test void taskSearchJobByApplicationUsingExecutionApp() { - retrofit.client.Response mockResponse = - new retrofit.client.Response( + Response mockResponse = + new Response( "test-url", 200, "test-reason", @@ -213,24 +213,16 @@ void taskSearchJobByApplicationUsingExecutionApp() { } @DisplayName( - "parameterized test for checking how property file contents are set in the stage context for a successful runjob") + "parameterized test for checking how property file contents are set in the stage context for a successful k8s runjob") @ParameterizedTest(name = "{index} ==> isPropertyFileContentsEmpty = {0}") @ValueSource(booleans = {true, false}) - void testPropertyFileContentsHandlingForASuccessfulRunJob(boolean isPropertyFileContentsEmpty) + void testPropertyFileContentsHandlingForASuccessfulK8sRunJob(boolean isPropertyFileContentsEmpty) throws IOException { // setup - InputStream jobStatusInputStream = - getResourceAsStream("clouddriver/tasks/job/successful-runjob-status.json"); - - retrofit.client.Response mockResponse = - new retrofit.client.Response( - "test-url", - 200, - "test-reason", - Collections.emptyList(), - new TypedByteArray("application/json", IOUtils.toByteArray(jobStatusInputStream))); - - when(mockKatoRestService.collectJob(any(), any(), any(), any())).thenReturn(mockResponse); + when(mockKatoRestService.collectJob(any(), any(), any(), any())) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status.json")); Map propertyFileContents = new HashMap<>(); if (!isPropertyFileContentsEmpty) { @@ -240,12 +232,9 @@ void testPropertyFileContentsHandlingForASuccessfulRunJob(boolean isPropertyFile eq("test-app"), eq("test-account"), eq("test"), eq("job testrep"), eq("testrep"))) .thenReturn(propertyFileContents); - Map stageContext = - getResource( - objectMapper, - "clouddriver/tasks/job/runjob-stage-context-with-property-file.json", - Map.class); - StageExecutionImpl myStage = createStageWithContext(stageContext); + StageExecution myStage = + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json"); // when ConfigurationException thrown = null; @@ -275,7 +264,9 @@ void testPropertyFileContentsHandlingForASuccessfulRunJob(boolean isPropertyFile thrown .getMessage() .matches( - "Expected properties file testrep but it was either missing, empty or contained invalid syntax")); + "Expected properties file: testrep in job: job testrep, application: test-app," + + " location: test, account: test-account but it was either missing, empty or" + + " contained invalid syntax")); } else { assertNotNull(result); assertThat(result.getContext().containsKey("propertyFileContents")).isTrue(); @@ -284,42 +275,36 @@ void testPropertyFileContentsHandlingForASuccessfulRunJob(boolean isPropertyFile } @Test - void testPropertyFileContentsErrorHandlingForASuccessfulRunJob() throws IOException { - // setup - InputStream jobStatusInputStream = - getResourceAsStream("clouddriver/tasks/job/successful-runjob-status.json"); - - retrofit.client.Response mockResponse = - new retrofit.client.Response( - "test-url", - 200, - "test-reason", - Collections.emptyList(), - new TypedByteArray("application/json", IOUtils.toByteArray(jobStatusInputStream))); - - when(mockKatoRestService.collectJob(any(), any(), any(), any())).thenReturn(mockResponse); + void testPropertyFileContentsErrorHandlingForASuccessfulK8sRunJob() throws IOException { + when(mockKatoRestService.collectJob(any(), any(), any(), any())) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status.json")); when(mockKatoRestService.getFileContents( eq("test-app"), eq("test-account"), eq("test"), eq("job testrep"), eq("testrep"))) .thenThrow(new RuntimeException("some exception")); - Map stageContext = - getResource( - objectMapper, - "clouddriver/tasks/job/runjob-stage-context-with-property-file.json", - Map.class); - StageExecutionImpl myStage = createStageWithContext(stageContext); - // when ConfigurationException thrown = - assertThrows(ConfigurationException.class, () -> task.execute(myStage)); + assertThrows( + ConfigurationException.class, + () -> + task.execute( + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json"))); // then verify(mockKatoRestService, times(1)) .collectJob(eq("test-app"), eq("test-account"), eq("test"), eq("job testrep")); - // since there are 6 tries made for this call if it fails - verify(mockKatoRestService, times(6)) + verify( + mockKatoRestService, + times( + configProperties + .getWaitOnJobCompletionTask() + .getFileContentRetry() + .getMaxAttempts())) .getFileContents( eq("test-app"), eq("test-account"), eq("test"), eq("job testrep"), eq("testrep")); @@ -332,42 +317,158 @@ void testPropertyFileContentsErrorHandlingForASuccessfulRunJob() throws IOExcept thrown .getMessage() .matches( - "Property File: testrep contents could not be retrieved. " - + "Error: java.lang.RuntimeException: some exception")); + "Expected properties file: testrep in job: job testrep, application: test-app," + + " location: test, account: test-account but it was either missing, empty or contained" + + " invalid syntax. Error: java.lang.RuntimeException: some exception")); } @DisplayName( - "parameterized test for checking if an exception is thrown when a run job fails, with or without a propertyFile") - @ParameterizedTest(name = "{index} ==> includePropertyFile = {0}") - @ValueSource(booleans = {true, false}) - void testRunJobFailuresErrorHandling(boolean includePropertyFile) throws IOException { + "test to parse properties file for a successful k8s job having 2 pods - first failed, and second succeeded. The" + + " properties file should be obtained from the getFileContents() call, if that is successful") + @Test + void + testParsePropertiesFileContentsForSuccessfulK8sJobWith2PodsWithSuccessfulGetFileContentsCall() + throws IOException { // setup - InputStream jobStatusInputStream = - getResourceAsStream("clouddriver/tasks/job/failed-runjob-status.json"); - retrofit.client.Response mockResponse = - new retrofit.client.Response( - "test-url", - 200, - "test-reason", - Collections.emptyList(), - new TypedByteArray("application/json", IOUtils.toByteArray(jobStatusInputStream))); + // mocked JobStatus response from clouddriver + when(mockKatoRestService.collectJob("test-app", "test-account", "test", "job testrep")) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json")); - when(mockKatoRestService.collectJob(any(), any(), any(), any())).thenReturn(mockResponse); + // when + when(mockKatoRestService.getFileContents( + "test-app", "test-account", "test", "job testrep", "testrep")) + .thenReturn(Map.of("some-key", "some-value")); + + TaskResult result = + task.execute( + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json")); + + // then + assertThat(result.getOutputs()).isNotEmpty(); + assertThat(result.getContext()).isNotEmpty(); + + assertThat(result.getOutputs().containsKey("some-key")); + assertThat(result.getOutputs().containsValue("some-value")); + verify(mockKatoRestService) + .getFileContents("test-app", "test-account", "test", "job testrep", "testrep"); + // no need to get file contents from a specific pod if the getFileContents call was successful + verify(mockKatoRestService, never()) + .getFileContentsFromKubernetesPod( + anyString(), anyString(), anyString(), anyString(), anyString()); + } + + @DisplayName( + "test to parse properties file for a successful k8s job having 2 pods - first failed, and second succeeded. The" + + " the properties file should be read directly from the succeeded pod if the getFileContents() call fails") + @Test + void testParsePropertiesFileContentsForSuccessfulK8sJobWith2PodsWithFailedGetFileContentsCall() + throws IOException { + // setup + + // mocked JobStatus response from clouddriver + when(mockKatoRestService.collectJob("test-app", "test-account", "test", "job testrep")) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json")); - Map stageContext = new HashMap<>(); + // when + when(mockKatoRestService.getFileContents( + "test-app", "test-account", "test", "job testrep", "testrep")) + .thenReturn(Map.of()); + + when(mockKatoRestService.getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep")) + .thenReturn(Map.of("some-key", "some-value")); + + TaskResult result = + task.execute( + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json")); + + // then + assertThat(result.getOutputs()).isNotEmpty(); + assertThat(result.getContext()).isNotEmpty(); + + assertThat(result.getOutputs().containsKey("some-key")); + assertThat(result.getOutputs().containsValue("some-value")); + verify(mockKatoRestService) + .getFileContents("test-app", "test-account", "test", "job testrep", "testrep"); + verify(mockKatoRestService) + .getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep"); + } + + @DisplayName( + "test to parse properties file for a successful k8s job having 2 pods - first failed, and second succeeded. The" + + " the properties file should be read from the getFileContents() call first. If that fails, a call to " + + " get the properties from the getFileContentsFromPod() should be made. If that fails," + + " an exception should be thrown") + @Test + void testParsePropertiesFileContentsErrorHandlingForSuccessfulK8sJobWith2Pods() + throws IOException { + // setup + + // mocked JobStatus response from clouddriver + when(mockKatoRestService.collectJob("test-app", "test-account", "test", "job testrep")) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json")); + + // when + when(mockKatoRestService.getFileContents( + "test-app", "test-account", "test", "job testrep", "testrep")) + .thenReturn(Map.of()); + + when(mockKatoRestService.getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep")) + .thenReturn(Map.of()); + + // then + ConfigurationException thrown = + assertThrows( + ConfigurationException.class, + () -> + task.execute( + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json"))); + + verify(mockKatoRestService) + .getFileContents("test-app", "test-account", "test", "job testrep", "testrep"); + verify(mockKatoRestService) + .getFileContentsFromKubernetesPod( + "test-app", "test-account", "test", "testrep-rn5qt", "testrep"); + assertTrue( + thrown + .getMessage() + .matches( + "Expected properties file: testrep in job: job testrep, application: test-app," + + " location: test, account: test-account but it was either missing, empty or contained" + + " invalid syntax")); + } + + @DisplayName( + "parameterized test for checking if an exception is thrown when a k8s run job fails, with or without a propertyFile") + @ParameterizedTest(name = "{index} ==> includePropertyFile = {0}") + @ValueSource(booleans = {true, false}) + void testK8sRunJobFailuresErrorHandling(boolean includePropertyFile) throws IOException { + // setup + when(mockKatoRestService.collectJob(any(), any(), any(), any())) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/failed-runjob-status.json")); + + String stageContextResource = + "clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json"; if (includePropertyFile) { - stageContext = - getResource( - objectMapper, - "clouddriver/tasks/job/runjob-stage-context-with-property-file.json", - Map.class); - } else { - stageContext = - getResource(objectMapper, "clouddriver/tasks/job/runjob-stage-context.json", Map.class); + stageContextResource = + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json"; } - StageExecutionImpl myStage = createStageWithContext(stageContext); + StageExecution myStage = createStageFromResource(stageContextResource); // when JobFailedException thrown = assertThrows(JobFailedException.class, () -> task.execute(myStage)); @@ -407,21 +508,13 @@ void testRunJobFailuresErrorHandling(boolean includePropertyFile) throws IOExcep "parameterized test for checking how property file contents are set in the stage context on a runjob failure") @ParameterizedTest(name = "{index} ==> isPropertyFileContentsEmpty = {0}") @ValueSource(booleans = {true, false}) - void testPropertyFileContentsHandlingForRunJobFailures(boolean isPropertyFileContentsEmpty) + void testPropertyFileContentsHandlingForK8sRunJobFailures(boolean isPropertyFileContentsEmpty) throws IOException { // setup - InputStream jobStatusInputStream = - getResourceAsStream("clouddriver/tasks/job/failed-runjob-status.json"); - - retrofit.client.Response mockResponse = - new retrofit.client.Response( - "test-url", - 200, - "test-reason", - Collections.emptyList(), - new TypedByteArray("application/json", IOUtils.toByteArray(jobStatusInputStream))); - - when(mockKatoRestService.collectJob(any(), any(), any(), any())).thenReturn(mockResponse); + when(mockKatoRestService.collectJob(any(), any(), any(), any())) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/failed-runjob-status.json")); Map propertyFileContents = new HashMap<>(); if (!isPropertyFileContentsEmpty) { @@ -432,12 +525,9 @@ void testPropertyFileContentsHandlingForRunJobFailures(boolean isPropertyFileCon eq("test-app"), eq("test-account"), eq("test"), eq("job testrep"), eq("testrep"))) .thenReturn(propertyFileContents); - Map stageContext = - getResource( - objectMapper, - "clouddriver/tasks/job/runjob-stage-context-with-property-file.json", - Map.class); - StageExecutionImpl myStage = createStageWithContext(stageContext); + StageExecution myStage = + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json"); // when JobFailedException thrown = assertThrows(JobFailedException.class, () -> task.execute(myStage)); @@ -457,8 +547,7 @@ void testPropertyFileContentsHandlingForRunJobFailures(boolean isPropertyFileCon verify(mockExecutionRepository, times(1)).storeStage(myStage); // validate that depending on the response obtained from the getFileContents() call, we either - // set - // propertyFileContents in the stage context or not + // set propertyFileContents in the stage context or not if (isPropertyFileContentsEmpty) { assertThat(myStage.getContext().containsKey("propertyFileContents")).isFalse(); } else { @@ -480,31 +569,20 @@ void testPropertyFileContentsHandlingForRunJobFailures(boolean isPropertyFileCon } @Test - void testPropertyFileContentsErrorHandlingForRunJobFailures() throws IOException { + void testPropertyFileContentsErrorHandlingForK8sRunJobFailures() throws IOException { // setup - InputStream jobStatusInputStream = - getResourceAsStream("clouddriver/tasks/job/failed-runjob-status.json"); - - retrofit.client.Response mockResponse = - new retrofit.client.Response( - "test-url", - 200, - "test-reason", - Collections.emptyList(), - new TypedByteArray("application/json", IOUtils.toByteArray(jobStatusInputStream))); - - when(mockKatoRestService.collectJob(any(), any(), any(), any())).thenReturn(mockResponse); + when(mockKatoRestService.collectJob(any(), any(), any(), any())) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/failed-runjob-status.json")); when(mockKatoRestService.getFileContents( eq("test-app"), eq("test-account"), eq("test"), eq("job testrep"), eq("testrep"))) .thenThrow(new RuntimeException("some exception")); - Map stageContext = - getResource( - objectMapper, - "clouddriver/tasks/job/runjob-stage-context-with-property-file.json", - Map.class); - StageExecutionImpl myStage = createStageWithContext(stageContext); + StageExecution myStage = + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json"); // when JobFailedException thrown = assertThrows(JobFailedException.class, () -> task.execute(myStage)); @@ -513,10 +591,13 @@ void testPropertyFileContentsErrorHandlingForRunJobFailures() throws IOException verify(mockKatoRestService, times(1)) .collectJob(eq("test-app"), eq("test-account"), eq("test"), eq("job testrep")); - // since there are 6 tries made for this call if it fails - this is a slow call since retry - // config options are - // hard-coded - verify(mockKatoRestService, times(6)) + verify( + mockKatoRestService, + times( + configProperties + .getWaitOnJobCompletionTask() + .getFileContentRetry() + .getMaxAttempts())) .getFileContents( eq("test-app"), eq("test-account"), eq("test"), eq("job testrep"), eq("testrep")); @@ -555,6 +636,22 @@ private StageExecutionImpl createStageWithContextWithoutExecutionApplication( new PipelineExecutionImpl(ExecutionType.PIPELINE, null), "test", new HashMap<>(context)); } + private StageExecution createStageFromResource(String resourceName) { + Map context = getResource(objectMapper, resourceName, Map.class); + return createStageWithContext(context); + } + + private Response createJobStatusFromResource(String resourceName) throws IOException { + InputStream jobStatusInputStream = getResourceAsStream(resourceName); + + return new Response( + "test-url", + 200, + "test-reason", + Collections.emptyList(), + new TypedByteArray("application/json", IOUtils.toByteArray(jobStatusInputStream))); + } + @DisplayName( "parameterized test to see how keys in the outputs object are filtered based on the inputs") @ParameterizedTest(name = "{index} ==> keys to be excluded from outputs = {0}") @@ -570,61 +667,20 @@ void testOutputFilter(String keysToFilter) throws IOException { .getWaitOnJobCompletionTask() .setExcludeKeysFromOutputs(expectedKeysToBeExcludedFromOutput); - InputStream inputStream = - getResourceAsStream("clouddriver/tasks/job/successful-run-job-state.json"); - - // mocked response from clouddriver - Response response = - new Response.Builder() - .code(200) - .message("some message") - .request(new Request.Builder().url("http://url").build()) - .protocol(Protocol.HTTP_1_0) - .body( - ResponseBody.create( - MediaType.parse("application/json"), IOUtils.toByteArray(inputStream))) - .addHeader("content-type", "application/json") - .build(); - - retrofit.client.Response retrofitResponse = - new retrofit.client.Response( - "http://url", - 200, - "", - Collections.emptyList(), - new TypedInput() { - @Override - public String mimeType() { - okhttp3.MediaType mediaType = response.body().contentType(); - return mediaType == null ? null : mediaType.toString(); - } - - @Override - public long length() { - return response.body().contentLength(); - } - - @Override - public InputStream in() { - return response.body().byteStream(); - } - }); - - when(mockKatoRestService.collectJob("testrep", "test-account", "test", "job testrep")) - .thenReturn(retrofitResponse); + // when + when(mockKatoRestService.collectJob("test-app", "test-account", "test", "job testrep")) + .thenReturn( + createJobStatusFromResource( + "clouddriver/tasks/job/kubernetes/successful-runjob-status.json")); when(mockKatoRestService.getFileContents( - "testrep", "test-account", "test", "job testrep", "testrep")) + "test-app", "test-account", "test", "job testrep", "testrep")) .thenReturn(Map.of("some-key", "some-value")); - Map context = - getResource(objectMapper, "clouddriver/tasks/job/runjob-context-success.json", Map.class); - StageExecution stageExecution = - new StageExecutionImpl( - new PipelineExecutionImpl(ExecutionType.PIPELINE, "testrep"), "test", context); - - // when - TaskResult result = task.execute(stageExecution); + TaskResult result = + task.execute( + createStageFromResource( + "clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json")); // then assertThat(result.getOutputs()).isNotEmpty(); diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/failed-runjob-status.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/failed-runjob-status.json similarity index 100% rename from orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/failed-runjob-status.json rename to orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/failed-runjob-status.json diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json new file mode 100644 index 0000000000..479e9e92aa --- /dev/null +++ b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-with-property-file.json @@ -0,0 +1,10 @@ +{ + "deploy.jobs": { + "test": [ + "job testrep" + ] + }, + "account": "test-account", + "propertyFile": "testrep", + "application": "test-app" +} diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/runjob-stage-context-with-property-file.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json similarity index 77% rename from orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/runjob-stage-context-with-property-file.json rename to orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json index 1a375774aa..31177b1b5e 100644 --- a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/runjob-stage-context-with-property-file.json +++ b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/runjob-stage-context-without-property-file.json @@ -5,5 +5,5 @@ ] }, "account": "test-account", - "propertyFile": "testrep" + "application": "test-app" } diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json new file mode 100644 index 0000000000..092fe2cfc5 --- /dev/null +++ b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status-with-multiple-pods.json @@ -0,0 +1,786 @@ +{ + "account": "test-account", + "completionDetails": { + "exitCode": "", + "message": "", + "reason": "", + "signal": "" + }, + "createdTime": 1633792127000, + "jobState": "Succeeded", + "location": "test", + "name": "testrep", + "pods": [ + { + "name": "testrep-bf2g7", + "status": { + "conditions": [ + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "message": "containers with incomplete status: [sidecar1 sidecar2]", + "reason": "ContainersNotInitialized", + "status": "False", + "type": "Initialized" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "message": "containers with unready status: [testrep]", + "reason": "ContainersNotReady", + "status": "False", + "type": "Ready" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "message": "containers with unready status: [testrep]", + "reason": "ContainersNotReady", + "status": "False", + "type": "ContainersReady" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "image": "some-main-app-image:1", + "imageID": "", + "lastState": {}, + "name": "testrep", + "ready": false, + "restartCount": 0, + "state": { + "waiting": { + "reason": "PodInitializing" + } + } + } + ], + "hostIP": "1.1.1.1", + "initContainerStatuses": [ + { + "containerID": "docker://2ed", + "image": "some-init-container-image:1", + "imageID": "docker-pullable://some-init-container-repo/some-init-container-image:1", + "lastState": {}, + "name": "sidecar1", + "ready": false, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://2ed", + "exitCode": 137, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792167000, + "millisOfDay": 54567000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54567, + "secondOfMinute": 27, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "OOMKilled", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792166000, + "millisOfDay": 54566000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54566, + "secondOfMinute": 26, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + }, + { + "image": "some-init-container-other-image:1", + "imageID": "", + "lastState": {}, + "name": "sidecar2", + "ready": false, + "restartCount": 0, + "state": { + "waiting": { + "reason": "PodInitializing" + } + } + } + ], + "phase": "Failed", + "podIP": "1.1.1.1", + "qosClass": "Burstable", + "startTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792128000, + "millisOfDay": 54528000, + "millisOfSecond": 0, + "minuteOfDay": 908, + "minuteOfHour": 8, + "monthOfYear": 10, + "secondOfDay": 54528, + "secondOfMinute": 48, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + }, + { + "name": "testrep-rn5qt", + "status": { + "conditions": [ + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792204000, + "millisOfDay": 54604000, + "millisOfSecond": 0, + "minuteOfDay": 910, + "minuteOfHour": 10, + "monthOfYear": 10, + "secondOfDay": 54604, + "secondOfMinute": 4, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "Initialized" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792321000, + "millisOfDay": 54721000, + "millisOfSecond": 0, + "minuteOfDay": 912, + "minuteOfHour": 12, + "monthOfYear": 10, + "secondOfDay": 54721, + "secondOfMinute": 1, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "Ready" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792321000, + "millisOfDay": 54721000, + "millisOfSecond": 0, + "minuteOfDay": 912, + "minuteOfHour": 12, + "monthOfYear": 10, + "secondOfDay": 54721, + "secondOfMinute": 1, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "ContainersReady" + }, + { + "lastTransitionTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792170000, + "millisOfDay": 54570000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54570, + "secondOfMinute": 30, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "status": "True", + "type": "PodScheduled" + } + ], + "containerStatuses": [ + { + "containerID": "docker://451", + "image": "some-main-app-image:1", + "imageID": "docker-pullable://some-main-app-repo/some-main-app-image:1", + "lastState": {}, + "name": "testrep", + "ready": true, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://451", + "exitCode": 0, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633791818000, + "millisOfDay": 54218000, + "millisOfSecond": 0, + "minuteOfDay": 903, + "minuteOfHour": 3, + "monthOfYear": 10, + "secondOfDay": 54218, + "secondOfMinute": 38, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "Completed", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633791758000, + "millisOfDay": 54158000, + "millisOfSecond": 0, + "minuteOfDay": 902, + "minuteOfHour": 2, + "monthOfYear": 10, + "secondOfDay": 54158, + "secondOfMinute": 38, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + } + ], + "hostIP": "1.1.1.1", + "initContainerStatuses": [ + { + "containerID": "docker://51b", + "image": "some-init-container-image:1", + "imageID": "docker-pullable://some-init-container-repo/some-init-container-image:1", + "lastState": {}, + "name": "casam-sidecar", + "ready": true, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://51b", + "exitCode": 0, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792194000, + "millisOfDay": 54594000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54594, + "secondOfMinute": 54, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "Completed", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792194000, + "millisOfDay": 54594000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54594, + "secondOfMinute": 54, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + }, + { + "containerID": "docker://b5b", + "image": "some-other-init-container-image:1", + "imageID": "docker-pullable://some-other-init-container-repo/some-other-init-container-image:1", + "lastState": {}, + "name": "sidecar2", + "ready": true, + "restartCount": 0, + "state": { + "terminated": { + "containerID": "docker://b5b", + "exitCode": 0, + "finishedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792195000, + "millisOfDay": 54595000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54595, + "secondOfMinute": 55, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "reason": "Completed", + "startedAt": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792195000, + "millisOfDay": 54595000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54595, + "secondOfMinute": 55, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + } + ], + "phase": "Succeeded", + "podIP": "1.1.1.1", + "qosClass": "Burstable", + "startTime": { + "afterNow": false, + "beforeNow": true, + "centuryOfEra": 20, + "chronology": { + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + }, + "dayOfMonth": 9, + "dayOfWeek": 6, + "dayOfYear": 282, + "equalNow": false, + "era": 1, + "hourOfDay": 15, + "millis": 1633792170000, + "millisOfDay": 54570000, + "millisOfSecond": 0, + "minuteOfDay": 909, + "minuteOfHour": 9, + "monthOfYear": 10, + "secondOfDay": 54570, + "secondOfMinute": 30, + "weekOfWeekyear": 40, + "weekyear": 2021, + "year": 2021, + "yearOfCentury": 21, + "yearOfEra": 2021, + "zone": { + "fixed": true, + "id": "Etc/GMT" + } + } + } + } + ], + "provider": "kubernetes" +} diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/successful-runjob-status.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status.json similarity index 100% rename from orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/successful-runjob-status.json rename to orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/kubernetes/successful-runjob-status.json diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/runjob-context-success.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/runjob-context-success.json deleted file mode 100644 index 625ee0f898..0000000000 --- a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/runjob-context-success.json +++ /dev/null @@ -1,266 +0,0 @@ -{ - "notification.type": "runjob", - "deploy.account.name": "test-account", - "outputs.createdArtifacts": [ - { - "customKind": false, - "reference": "testrep", - "metadata": { - "account": "test-account" - }, - "name": "testrep", - "location": "test", - "type": "kubernetes/job", - "version": "" - } - ], - "propertyFile": "testrep", - "consumeArtifactSource": "propertyFile", - "stageEnabled": {}, - "source": "text", - "dev": true, - "cloudProvider": "kubernetes", - "kato.result.expected": false, - "alias": "runJob", - "deploy.server.groups": {}, - "kato.last.task.id": { - "id": "123" - }, - "artifacts": [ - { - "customKind": false, - "reference": "testrep", - "metadata": { - "account": "test-account" - }, - "name": "testrep", - "location": "test", - "type": "kubernetes/job", - "version": "" - } - ], - "manifest": { - "metadata": { - "name": " testrep", - "namespace": "test", - "labels": { - "p_environment_type": "dev" - } - }, - "apiVersion": "batch/v1", - "kind": "Job", - "spec": { - "template": { - "spec": { - "dnsPolicy": "ClusterFirst", - "terminationGracePeriodSeconds": 30, - "automountServiceAccountToken": true, - "serviceAccountName": "test", - "volumes": [ - { - "configMap": { - "name": "configmap" - }, - "name": "configmap-volume" - }, - { - "configMap": { - "name": "configmap-1" - }, - "name": "configs-volume-1" - }, - { - "name": "creds", - "secret": { - "secretName": "secret" - } - }, - { - "emptyDir": {}, - "name": "secrets-volume" - } - ], - "containers": [ - { - "image": "main-app-image:1", - "terminationMessagePolicy": "FallbackToLogsOnError", - "name": "testrepl1", - "env": [ - { - "name": "LEVEL", - "value": "service" - } - ], - "volumeMounts": [ - { - "mountPath": "/usr/test/configs", - "name": "configmap-volume" - } - ] - } - ], - "securityContext": { - "fsGroup": 100 - }, - "restartPolicy": "Never", - "initContainers": [] - } - }, - "backoffLimit": 0, - "activeDeadlineSeconds": 900, - "ttlSecondsAfterFinished": 600 - } - }, - "kato.task.terminalRetryCount": 0, - "isNew": true, - "kato.task.firstNotFoundRetry": -1, - "failOnFailedExpressions": true, - "outputs.manifestNamesByNamespace": { - "test": [ - "job testrep" - ] - }, - "application": "testrep", - "outputs.boundArtifacts": [], - "credentails": "test-account", - "kato.tasks": [ - { - "outputs": [], - "resultObjects": [ - { - "createdArtifacts": [ - { - "customKind": false, - "reference": "testrep", - "metadata": { - "account": "test-account" - }, - "name": "testrep", - "location": "test", - "type": "kubernetes/job", - "version": "" - } - ], - "boundArtifacts": [] - } - ], - "id": "123", - "history": [ - { - "phase": "ORCHESTRATION", - "status": "Initializing Orchestration Task" - }, - { - "phase": "ORCHESTRATION", - "status": "Processing op: KubernetesRunJobOperation" - }, - { - "phase": "RUN_KUBERNETES_JOB", - "status": "Running Kubernetes job..." - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Beginning deployment of manifests in account test-account ..." - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Swapping out artifacts in job testrep from context..." - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Finding deployer for job..." - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Checking if all requested artifacts were bound..." - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Sorting manifests by priority..." - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Deploy order is: job testrep" - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Finding deployer for job..." - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Annotating manifest job testrep with artifact, relationships & moniker..." - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Swapping out artifacts in job testrep from other deployments..." - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Submitting manifest job testrep to kubernetes master..." - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Deploy manifest task completed successfully for manifest job testrep in account test-account" - }, - { - "phase": "DEPLOY_KUBERNETES_MANIFEST", - "status": "Deploy manifest task completed successfully for all manifests in account test-account" - }, - { - "phase": "ORCHESTRATION", - "status": "Orchestration completed." - }, - { - "phase": "ORCHESTRATION", - "status": "Orchestration completed." - } - ], - "status": { - "retryable": false, - "completed": true, - "failed": false - } - } - ], - "deploy.jobs": { - "test": [ - "job testrep" - ] - }, - "outputs.manifests": [ - { - "metadata": { - "uid": "eaf5237d-774e-49f6-b3a1-ac90a7e47756", - "resourceVersion": "172459430", - "creationTimestamp": "2021-08-11T01:40:00Z", - "name": "testrep", - "namespace": "test" - }, - "apiVersion": "batch/v1", - "kind": "Job", - "spec": { - "backoffLimit": 0, - "parallelism": 1, - "completions": 1, - "selector": { - "matchLabels": { - "controller-uid": "eaf" - } - }, - "activeDeadlineSeconds": 900 - }, - "status": {} - } - ], - "kato.task.notFoundRetryCount": 0, - "account": "test-account", - "kato.task.lastStatus": "SUCCEEDED", - "propertyFileContents": { - "dev": true, - "some_other_output": {} - }, - "output_from_runjob": { - "target_level": "service" - } - } -} diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/runjob-stage-context.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/runjob-stage-context.json deleted file mode 100644 index 15f1120a56..0000000000 --- a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/runjob-stage-context.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "deploy.jobs": { - "test": [ - "job testrep" - ] - }, - "account": "test-account" -} diff --git a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/successful-run-job-state.json b/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/successful-run-job-state.json deleted file mode 100644 index 338a9ea57a..0000000000 --- a/orca-clouddriver/src/test/resources/com/netflix/spinnaker/orca/clouddriver/tasks/job/successful-run-job-state.json +++ /dev/null @@ -1,564 +0,0 @@ -{ -"provider": "kubernetes", -"completionDetails": { -"summary": "", -"reason": "", -"exitCode": "", -"message": "", -"signal": "" -}, -"jobState": "Succeeded", -"name": "testrep", -"createdTime": 1628646000000, -"location": "test", -"pods": [ -{ -"containerExecutionDetails": [], -"name": "testrep-lnk4x", -"status": { -"phase": "Succeeded", -"podIP": "1.1.1.1", -"containerStatuses": [ -{ -"image": "sha256:123", -"imageID": "docker-pullable://some-repo:1", -"restartCount": 0, -"ready": false, -"name": "testrepv2-l1", -"started": false, -"state": { -"terminated": { -"reason": "Completed", -"exitCode": 0, -"startedAt": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 24, -"millisOfDay": 6024000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6024, -"millis": 1628646024000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -}, -"containerID": "docker://456", -"finishedAt": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 27, -"millisOfDay": 6027000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6027, -"millis": 1628646027000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -} -} -}, -"containerID": "docker://456", -"lastState": {} -} -], -"hostIP": "1.1.1.1", -"startTime": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 1, -"millisOfDay": 6001000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6001, -"millis": 1628646001000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -}, -"qosClass": "Burstable", -"conditions": [ -{ -"reason": "PodCompleted", -"lastTransitionTime": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 23, -"millisOfDay": 6023000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6023, -"millis": 1628646023000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -}, -"type": "Initialized", -"status": "True" -}, -{ -"reason": "PodCompleted", -"lastTransitionTime": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 27, -"millisOfDay": 6027000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6027, -"millis": 1628646027000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -}, -"type": "Ready", -"status": "False" -}, -{ -"reason": "PodCompleted", -"lastTransitionTime": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 27, -"millisOfDay": 6027000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6027, -"millis": 1628646027000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -}, -"type": "ContainersReady", -"status": "False" -}, -{ -"lastTransitionTime": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 1, -"millisOfDay": 6001000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6001, -"millis": 1628646001000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -}, -"type": "PodScheduled", -"status": "True" -} -], -"initContainerStatuses": [ -{ -"image": "some-init-container-image:1", -"imageID": "docker-pullable://some-init-container-repo/some-init-container-image:1", -"restartCount": 0, -"ready": true, -"name": "init", -"state": { -"terminated": { -"reason": "Completed", -"exitCode": 0, -"startedAt": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 2, -"millisOfDay": 6002000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6002, -"millis": 1628646002000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -}, -"containerID": "docker://789", -"finishedAt": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 19, -"millisOfDay": 6019000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6019, -"millis": 1628646019000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -} -} -}, -"containerID": "docker://789", -"lastState": {} -}, -{ -"image": "init-container-2:1", -"imageID": "docker-pullable://some-repo/init-container-2:1", -"restartCount": 0, -"ready": true, -"name": "init2", -"state": { -"terminated": { -"reason": "Completed", -"exitCode": 0, -"startedAt": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 20, -"millisOfDay": 6020000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6020, -"millis": 1628646020000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -}, -"containerID": "docker://012", -"finishedAt": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 21, -"millisOfDay": 6021000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6021, -"millis": 1628646021000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -} -} -}, -"containerID": "docker://012", -"lastState": {} -}, -{ -"image": "init-container-3:1", -"imageID": "docker-pullable://some-repo/init-container-3:1", -"restartCount": 0, -"ready": true, -"name": "init-3", -"state": { -"terminated": { -"reason": "Completed", -"exitCode": 0, -"startedAt": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 21, -"millisOfDay": 6021000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6021, -"millis": 1628646021000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -}, -"containerID": "docker://345", -"finishedAt": { -"dayOfYear": 223, -"equalNow": false, -"year": 2021, -"weekyear": 2021, -"chronology": { -"zone": { -"fixed": true, -"id": "Etc/GMT" -} -}, -"weekOfWeekyear": 32, -"secondOfMinute": 23, -"millisOfDay": 6023000, -"monthOfYear": 8, -"beforeNow": true, -"dayOfWeek": 3, -"minuteOfDay": 100, -"dayOfMonth": 11, -"era": 1, -"zone": { -"fixed": true, -"id": "Etc/GMT" -}, -"yearOfCentury": 21, -"centuryOfEra": 20, -"hourOfDay": 1, -"secondOfDay": 6023, -"millis": 1628646023000, -"yearOfEra": 2021, -"minuteOfHour": 40, -"afterNow": false, -"millisOfSecond": 0 -} -} -}, -"containerID": "docker://345", -"lastState": {} -} -], -"podIPs": [ -{ -"ip": "10.33.167.185" -} -] -} -} -], -"account": "test-account" -}