Refactor deploy from URL to avoid stuck jobs

cloudfoundry · Dec 15, 2023 · 0573010 · 0573010
1 parent 84f38cc
commit 0573010
Show file tree

Hide file tree

Showing 18 changed files with 354 additions and 138 deletions.
diff --git a/...-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/AsyncUploadResult.java b/...-api/src/main/java/org/cloudfoundry/multiapps/controller/api/model/AsyncUploadResult.java
@@ -8,6 +8,8 @@
 import org.cloudfoundry.multiapps.common.Nullable;
 import org.immutables.value.Value;
 
+import java.util.List;
+
 @Value.Immutable
 @JsonSerialize(as = ImmutableAsyncUploadResult.class)
 @JsonDeserialize(as = ImmutableAsyncUploadResult.class)
@@ -17,6 +19,10 @@ enum JobStatus {
         RUNNING, FINISHED, ERROR
     }
 
+    enum ClientAction {
+        RETRY_UPLOAD
+    }
+
     @ApiModelProperty
     @JsonProperty("status")
     JobStatus getStatus();
@@ -40,4 +46,9 @@ enum JobStatus {
     @ApiModelProperty
     @JsonProperty("mta_id")
     String getMtaId();
+
+    @Nullable
+    @ApiModelProperty
+    @JsonProperty("client_actions")
+    List<ClientAction> getClientActions();
 }
diff --git a/...s-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/FilesApi.java b/...s-controller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/FilesApi.java
@@ -4,14 +4,14 @@
 
 import javax.inject.Inject;
 
-import org.cloudfoundry.multiapps.controller.api.FilesApiService;
 import org.cloudfoundry.multiapps.controller.api.Constants.Endpoints;
 import org.cloudfoundry.multiapps.controller.api.Constants.PathVariables;
 import org.cloudfoundry.multiapps.controller.api.Constants.RequestVariables;
 import org.cloudfoundry.multiapps.controller.api.Constants.Resources;
+import org.cloudfoundry.multiapps.controller.api.FilesApiService;
+import org.cloudfoundry.multiapps.controller.api.model.AsyncUploadResult;
 import org.cloudfoundry.multiapps.controller.api.model.FileMetadata;
 import org.cloudfoundry.multiapps.controller.api.model.FileUrl;
-import org.cloudfoundry.multiapps.controller.api.model.AsyncUploadResult;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.GetMapping;
@@ -64,10 +64,10 @@ public class FilesApi {
     }
 
     @PostMapping(path = Endpoints.ASYNC_UPLOAD, consumes = MediaType.APPLICATION_JSON_VALUE)
-    @ApiOperation(value = "", nickname = "startUploadFromUrl", notes = "Uploads a Multi Target Application archive or an Extension Descriptor from a remote endpoint",  authorizations = {
-            @Authorization(value = "oauth2", scopes = {
+    @ApiOperation(value = "", nickname = "startUploadFromUrl", notes = "Uploads a Multi Target Application archive or an Extension Descriptor from a remote endpoint", authorizations = {
+        @Authorization(value = "oauth2", scopes = {
 
-            }) }, tags = {})
+        }) }, tags = {})
     @ApiResponses(value = { @ApiResponse(code = 202, message = "Accepted") })
     public ResponseEntity<Void>
            startUploadFromUrl(@ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid,
@@ -78,13 +78,15 @@ public class FilesApi {
 
     @GetMapping(path = Endpoints.ASYNC_UPLOAD_JOB, produces = MediaType.APPLICATION_JSON_VALUE)
     @ApiOperation(value = "", nickname = "getAsyncUploadJob", notes = "Gets the status of an async upload job", response = AsyncUploadResult.class, authorizations = {
-            @Authorization(value = "oauth2", scopes = {
+        @Authorization(value = "oauth2", scopes = {
 
-            }) }, tags = {})
-    @ApiResponses(value = { @ApiResponse(code = 200, message = "OK"), @ApiResponse(code = 201, message = "Created", response = AsyncUploadResult.class) })
-    public ResponseEntity<AsyncUploadResult> getUploadFromUrlJob(@ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid,
-                                                                 @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace,
-                                                                 @ApiParam(value = "ID of the upload job") @PathVariable(PathVariables.JOB_ID) String jobId) {
+        }) }, tags = {})
+    @ApiResponses(value = { @ApiResponse(code = 200, message = "OK"),
+        @ApiResponse(code = 201, message = "Created", response = AsyncUploadResult.class) })
+    public ResponseEntity<AsyncUploadResult>
+           getUploadFromUrlJob(@ApiParam(value = "GUID of space you wish to deploy in") @PathVariable(PathVariables.SPACE_GUID) String spaceGuid,
+                               @ApiParam(value = "file namespace") @RequestParam(name = RequestVariables.NAMESPACE, required = false) String namespace,
+                               @ApiParam(value = "ID of the upload job") @PathVariable(PathVariables.JOB_ID) String jobId) {
         return delegate.getUploadFromUrlJob(spaceGuid, namespace, jobId);
     }
 

diff --git a/...troller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/OperationsApi.java b/...troller-api/src/main/java/org/cloudfoundry/multiapps/controller/api/v1/OperationsApi.java
@@ -5,12 +5,12 @@
 import javax.inject.Inject;
 import javax.servlet.http.HttpServletRequest;
 
-import org.cloudfoundry.multiapps.controller.api.OperationsApiService;
 import org.cloudfoundry.multiapps.controller.api.Constants.Endpoints;
 import org.cloudfoundry.multiapps.controller.api.Constants.PathVariables;
 import org.cloudfoundry.multiapps.controller.api.Constants.QueryVariables;
 import org.cloudfoundry.multiapps.controller.api.Constants.RequestVariables;
 import org.cloudfoundry.multiapps.controller.api.Constants.Resources;
+import org.cloudfoundry.multiapps.controller.api.OperationsApiService;
 import org.cloudfoundry.multiapps.controller.api.model.Log;
 import org.cloudfoundry.multiapps.controller.api.model.Operation;
 import org.springframework.http.MediaType;

diff --git a/multiapps-controller-api/src/main/resources/mtarest.yaml b/multiapps-controller-api/src/main/resources/mtarest.yaml
@@ -426,6 +426,13 @@ definitions:
       mta_id:
         type: "string"
         readOnly: true
+      client_actions:
+        type: "array"
+        readOnly: true
+        items:
+          type: "string"
+          enum:
+          - "RETRY_UPLOAD"
   FileMetadata:
     type: "object"
     properties:

diff --git a/...rc/main/java/org/cloudfoundry/multiapps/controller/persistence/dto/AsyncUploadJobDto.java b/...rc/main/java/org/cloudfoundry/multiapps/controller/persistence/dto/AsyncUploadJobDto.java
@@ -1,12 +1,13 @@
 package org.cloudfoundry.multiapps.controller.persistence.dto;
 
-import org.cloudfoundry.multiapps.controller.persistence.model.PersistenceMetadata;
+import java.time.LocalDateTime;
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
 import javax.persistence.Id;
 import javax.persistence.Table;
-import java.time.LocalDateTime;
+
+import org.cloudfoundry.multiapps.controller.persistence.model.PersistenceMetadata;
 
 @Entity
 @Table(name = PersistenceMetadata.TableNames.ASYNC_UPLOAD_JOB_TABLE)
@@ -21,6 +22,7 @@ private AttributeNames() {
         public static final String USER = "mtaUser";
         public static final String STATE = "state";
         public static final String URL = "url";
+        public static final String ADDED_AT = "addedAt";
         public static final String STARTED_AT = "startedAt";
         public static final String FINISHED_AT = "finishedAt";
         public static final String NAMESPACE = "namespace";
@@ -44,6 +46,9 @@ private AttributeNames() {
     @Column(name = PersistenceMetadata.TableColumnNames.ASYNC_UPLOAD_JOB_URL, nullable = false)
     private String url;
 
+    @Column(name = PersistenceMetadata.TableColumnNames.ASYNC_UPLOAD_JOB_ADDED_AT)
+    private LocalDateTime addedAt;
+
     @Column(name = PersistenceMetadata.TableColumnNames.ASYNC_UPLOAD_JOB_STARTED_AT)
     private LocalDateTime startedAt;
 
@@ -72,13 +77,14 @@ public AsyncUploadJobDto() {
         // Required by JPA
     }
 
-    public AsyncUploadJobDto(String id, String mtaUser, String state, String url, LocalDateTime startedAt,
-                             LocalDateTime finishedAt, String namespace, String spaceGuid, String mtaId, String fileId,
-                             String error, Integer instanceIndex) {
+    public AsyncUploadJobDto(String id, String mtaUser, String state, String url, LocalDateTime addedAt, LocalDateTime startedAt,
+                             LocalDateTime finishedAt, String namespace, String spaceGuid, String mtaId, String fileId, String error,
+                             Integer instanceIndex) {
         this.id = id;
         this.mtaUser = mtaUser;
         this.state = state;
         this.url = url;
+        this.addedAt = addedAt;
         this.startedAt = startedAt;
         this.finishedAt = finishedAt;
         this.namespace = namespace;
@@ -123,6 +129,14 @@ public void setUrl(String url) {
         this.url = url;
     }
 
+    public LocalDateTime getAddedAt() {
+        return addedAt;
+    }
+
+    public void setAddedAt(LocalDateTime addedAt) {
+        this.addedAt = addedAt;
+    }
+
     public LocalDateTime getStartedAt() {
         return startedAt;
     }
@@ -194,6 +208,7 @@ public String toString() {
                 ", mtaUser='" + mtaUser + '\'' +
                 ", state='" + state + '\'' +
                 ", url='" + url + '\'' +
+                ", addedAt=" + addedAt +
                 ", startedAt=" + startedAt +
                 ", finishedAt=" + finishedAt +
                 ", namespace='" + namespace + '\'' +

diff --git a/...ain/java/org/cloudfoundry/multiapps/controller/persistence/model/AsyncUploadJobEntry.java b/...ain/java/org/cloudfoundry/multiapps/controller/persistence/model/AsyncUploadJobEntry.java
@@ -1,10 +1,10 @@
 package org.cloudfoundry.multiapps.controller.persistence.model;
 
+import java.time.LocalDateTime;
+
 import org.cloudfoundry.multiapps.common.Nullable;
 import org.immutables.value.Value;
 
-import java.time.LocalDateTime;
-
 @Value.Immutable
 public interface AsyncUploadJobEntry {
 
@@ -20,6 +20,9 @@ enum State {
 
     String getUrl();
 
+    @Nullable
+    LocalDateTime getAddedAt();
+
     @Nullable
     LocalDateTime getStartedAt();
 

diff --git a/...ain/java/org/cloudfoundry/multiapps/controller/persistence/model/PersistenceMetadata.java b/...ain/java/org/cloudfoundry/multiapps/controller/persistence/model/PersistenceMetadata.java
@@ -90,6 +90,7 @@ private TableColumnNames() {
         public static final String ASYNC_UPLOAD_JOB_STATE = "state";
         public static final String ASYNC_UPLOAD_JOB_URL = "url";
         public static final String ASYNC_UPLOAD_JOB_USER = "mta_user";
+        public static final String ASYNC_UPLOAD_JOB_ADDED_AT = "added_at";
         public static final String ASYNC_UPLOAD_JOB_STARTED_AT = "started_at";
         public static final String ASYNC_UPLOAD_JOB_FINISHED_AT = "finished_at";
         public static final String ASYNC_UPLOAD_JOB_NAMESPACE = "namespace";

diff --git a/...in/java/org/cloudfoundry/multiapps/controller/persistence/query/AsyncUploadJobsQuery.java b/...in/java/org/cloudfoundry/multiapps/controller/persistence/query/AsyncUploadJobsQuery.java
@@ -1,9 +1,9 @@
 package org.cloudfoundry.multiapps.controller.persistence.query;
 
-import org.cloudfoundry.multiapps.controller.persistence.model.AsyncUploadJobEntry;
-
 import java.time.LocalDateTime;
 
+import org.cloudfoundry.multiapps.controller.persistence.model.AsyncUploadJobEntry;
+
 public interface AsyncUploadJobsQuery extends Query<AsyncUploadJobEntry, AsyncUploadJobsQuery> {
 
     AsyncUploadJobsQuery id(String id);
@@ -23,4 +23,12 @@ public interface AsyncUploadJobsQuery extends Query<AsyncUploadJobEntry, AsyncUp
     AsyncUploadJobsQuery withStateAnyOf(AsyncUploadJobEntry.State... states);
 
     AsyncUploadJobsQuery startedBefore(LocalDateTime startedBefore);
+
+    AsyncUploadJobsQuery addedBefore(LocalDateTime addedBefore);
+
+    AsyncUploadJobsQuery withoutStartedAt();
+
+    AsyncUploadJobsQuery withoutAddedAt();
+
+    AsyncUploadJobsQuery instanceIndex(int instanceIndex);
 }
diff --git a/...rg/cloudfoundry/multiapps/controller/persistence/query/impl/AsyncUploadJobsQueryImpl.java b/...rg/cloudfoundry/multiapps/controller/persistence/query/impl/AsyncUploadJobsQueryImpl.java
@@ -1,5 +1,15 @@
 package org.cloudfoundry.multiapps.controller.persistence.query.impl;
 
+import java.time.LocalDateTime;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import javax.persistence.EntityManager;
+import javax.persistence.NoResultException;
+import javax.persistence.NonUniqueResultException;
+import javax.persistence.criteria.Expression;
+
 import org.cloudfoundry.multiapps.controller.persistence.dto.AsyncUploadJobDto;
 import org.cloudfoundry.multiapps.controller.persistence.dto.AsyncUploadJobDto.AttributeNames;
 import org.cloudfoundry.multiapps.controller.persistence.model.AsyncUploadJobEntry;
@@ -9,15 +19,6 @@
 import org.cloudfoundry.multiapps.controller.persistence.query.criteria.QueryCriteria;
 import org.cloudfoundry.multiapps.controller.persistence.services.AsyncUploadJobService.AsyncUploadJobMapper;
 
-import javax.persistence.EntityManager;
-import javax.persistence.NoResultException;
-import javax.persistence.NonUniqueResultException;
-import javax.persistence.criteria.Expression;
-import java.time.LocalDateTime;
-import java.util.Arrays;
-import java.util.List;
-import java.util.stream.Collectors;
-
 public class AsyncUploadJobsQueryImpl extends AbstractQueryImpl<AsyncUploadJobEntry, AsyncUploadJobsQuery> implements AsyncUploadJobsQuery {
 
     private final QueryCriteria queryCriteria = new QueryCriteria();
@@ -99,6 +100,7 @@ public AsyncUploadJobsQuery withoutFinishedAt() {
         return this;
     }
 
+    @Override
     public AsyncUploadJobsQuery withStateAnyOf(State... states) {
         queryCriteria.addRestriction(ImmutableQueryAttributeRestriction.<List<State>> builder()
                                                                        .attribute(AttributeNames.STATE)
@@ -108,6 +110,17 @@ public AsyncUploadJobsQuery withStateAnyOf(State... states) {
         return this;
     }
 
+    @Override
+    public AsyncUploadJobsQuery addedBefore(LocalDateTime addedBefore) {
+        queryCriteria.addRestriction(ImmutableQueryAttributeRestriction.<LocalDateTime> builder()
+                                                                       .attribute(AttributeNames.ADDED_AT)
+                                                                       .condition(getCriteriaBuilder()::lessThan)
+                                                                       .value(addedBefore)
+                                                                       .build());
+        return this;
+    }
+
+    @Override
     public AsyncUploadJobsQuery startedBefore(LocalDateTime startedBefore) {
         queryCriteria.addRestriction(ImmutableQueryAttributeRestriction.<LocalDateTime> builder()
                                                                        .attribute(AttributeNames.STARTED_AT)
@@ -117,6 +130,35 @@ public AsyncUploadJobsQuery startedBefore(LocalDateTime startedBefore) {
         return this;
     }
 
+    @Override
+    public AsyncUploadJobsQuery withoutStartedAt() {
+        queryCriteria.addRestriction(ImmutableQueryAttributeRestriction.builder()
+                                                                       .attribute(AttributeNames.STARTED_AT)
+                                                                       .condition((attribute, value) -> attribute.isNull())
+                                                                       .build());
+        return this;
+    }
+
+    @Override
+    public AsyncUploadJobsQuery withoutAddedAt() {
+        queryCriteria.addRestriction(ImmutableQueryAttributeRestriction.builder()
+                                                                       .attribute(AttributeNames.ADDED_AT)
+                                                                       .condition((attribute, value) -> attribute.isNull())
+                                                                       .build());
+        return this;
+    }
+
+    @Override
+    public AsyncUploadJobsQuery instanceIndex(int instanceIndex) {
+        queryCriteria.addRestriction(ImmutableQueryAttributeRestriction.builder()
+                                                                       .attribute(AttributeNames.INSTANCE_INDEX)
+                                                                       .condition(getCriteriaBuilder()::equal)
+                                                                       .value(instanceIndex)
+                                                                       .build());
+        return this;
+
+    }
+
     @Override
     public AsyncUploadJobEntry singleResult() throws NoResultException, NonUniqueResultException {
         AsyncUploadJobDto dto = executeInTransaction(manager -> createQuery(manager, queryCriteria,

diff --git a/...ava/org/cloudfoundry/multiapps/controller/persistence/services/AsyncUploadJobService.java b/...ava/org/cloudfoundry/multiapps/controller/persistence/services/AsyncUploadJobService.java
@@ -1,5 +1,9 @@
 package org.cloudfoundry.multiapps.controller.persistence.services;
 
+import javax.inject.Inject;
+import javax.inject.Named;
+import javax.persistence.EntityManagerFactory;
+
 import org.cloudfoundry.multiapps.common.ConflictException;
 import org.cloudfoundry.multiapps.common.NotFoundException;
 import org.cloudfoundry.multiapps.controller.persistence.Messages;
@@ -9,10 +13,6 @@
 import org.cloudfoundry.multiapps.controller.persistence.query.AsyncUploadJobsQuery;
 import org.cloudfoundry.multiapps.controller.persistence.query.impl.AsyncUploadJobsQueryImpl;
 
-import javax.inject.Inject;
-import javax.inject.Named;
-import javax.persistence.EntityManagerFactory;
-
 @Named
 public class AsyncUploadJobService extends PersistenceService<AsyncUploadJobEntry, AsyncUploadJobDto, String> {
 
@@ -48,6 +48,7 @@ public AsyncUploadJobEntry fromDto(AsyncUploadJobDto dto) {
                                                .id(dto.getPrimaryKey())
                                                .user(dto.getMtaUser())
                                                .state(AsyncUploadJobEntry.State.valueOf(dto.getState()))
+                                               .addedAt(dto.getAddedAt())
                                                .startedAt(dto.getStartedAt())
                                                .finishedAt(dto.getFinishedAt())
                                                .namespace(dto.getNamespace())
@@ -67,6 +68,7 @@ public AsyncUploadJobDto toDto(AsyncUploadJobEntry entry) {
                                          entry.getState()
                                               .toString(),
                                          entry.getUrl(),
+                                         entry.getAddedAt(),
                                          entry.getStartedAt(),
                                          entry.getFinishedAt(),
                                          entry.getNamespace(),

diff --git a/...oundry/multiapps/controller/persistence/db/changelog/db-changelog-1.164.0-persistence.xml b/...oundry/multiapps/controller/persistence/db/changelog/db-changelog-1.164.0-persistence.xml
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+         http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.6.xsd">
+
+    <changeSet author="sap.com" id="add_column_added_at_async_upload_job">
+        <preConditions onFail="MARK_RAN">
+            <tableExists tableName="async_upload_job"/>
+        </preConditions>
+        <addColumn tableName="async_upload_job">
+            <column name="added_at" type="TIMESTAMP"/>
+        </addColumn>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/...resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog.xml b/...resources/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog.xml
@@ -28,4 +28,6 @@
 
     <include file="/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-1.160.0-persistence.xml" />
 
+    <include file="/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-1.164.0-persistence.xml" />
+
 </databaseChangeLog>
Original file line number	Diff line number	Diff line change
Expand Up		@@ -28,4 +28,6 @@

		<include file="/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-1.160.0-persistence.xml" />

		<include file="/org/cloudfoundry/multiapps/controller/persistence/db/changelog/db-changelog-1.164.0-persistence.xml" />

		</databaseChangeLog>