Skip to content

Commit

Permalink
Merge pull request #13 from samply/feature/exporter-job
Browse files Browse the repository at this point in the history
Feature/exporter job
  • Loading branch information
djuarezgf authored Apr 18, 2024
2 parents 9405b1c + f8cc582 commit 37ce9c2
Show file tree
Hide file tree
Showing 20 changed files with 372 additions and 148 deletions.
5 changes: 4 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).

## [0.0.1 - 2024-04-15]
## [0.0.1 - 2024-04-18]
### Added
- First version of the project
- Spring Application
Expand Down Expand Up @@ -130,3 +130,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
- Create notification after sending an email
- Send message while requesting changes in script or project
- Send message while rejecting a project
- Exporter Job
- Check Queries in Exporter Job
- Max time to wait focus task in minutes
2 changes: 2 additions & 0 deletions src/main/java/de/samply/annotations/StateConstraints.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import de.samply.project.state.ProjectBridgeheadState;
import de.samply.project.state.ProjectState;
import de.samply.query.QueryState;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
Expand All @@ -14,5 +15,6 @@
public @interface StateConstraints {
ProjectState[] projectStates() default {};
ProjectBridgeheadState[] projectBridgeheadStates() default {};
QueryState[] queryStates() default {};

}
25 changes: 19 additions & 6 deletions src/main/java/de/samply/aop/ConstraintsService.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import de.samply.project.ProjectType;
import de.samply.project.state.ProjectBridgeheadState;
import de.samply.project.state.ProjectState;
import de.samply.query.QueryState;
import de.samply.security.SessionUser;
import de.samply.user.roles.OrganisationRole;
import de.samply.user.roles.OrganisationRoleToProjectRoleMapper;
Expand Down Expand Up @@ -110,16 +111,28 @@ public Optional<ResponseEntity> checkStateConstraints(Optional<StateConstraints>
return Optional.of(ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).build());
}
}
if (stateConstraints.get().projectBridgeheadStates().length > 0) {
if (stateConstraints.get().projectBridgeheadStates().length > 0 || stateConstraints.get().queryStates().length > 0) {
Optional<ProjectBridgehead> projectBridgehead = fetchProjectBridgehead(project.get(), bridgehead);
if (projectBridgehead.isEmpty()) {
return Optional.of(ResponseEntity.status(HttpStatus.METHOD_NOT_ALLOWED).build());
}
boolean hasAnyProjectBridgeheadStateConstraint = false;
for (ProjectBridgeheadState projectBridgeheadState : stateConstraints.get().projectBridgeheadStates()) {
if (projectBridgehead.get().getState() == projectBridgeheadState) {
hasAnyProjectBridgeheadStateConstraint = true;
break;
boolean hasAnyProjectBridgeheadStateConstraint = true;
if (stateConstraints.get().projectBridgeheadStates().length > 0) {
hasAnyProjectBridgeheadStateConstraint = false;
for (ProjectBridgeheadState projectBridgeheadState : stateConstraints.get().projectBridgeheadStates()) {
if (projectBridgehead.get().getState() == projectBridgeheadState) {
hasAnyProjectBridgeheadStateConstraint = true;
break;
}
}
}
if (hasAnyProjectBridgeheadStateConstraint && stateConstraints.get().queryStates().length > 0) {
hasAnyProjectBridgeheadStateConstraint = false;
for (QueryState queryState : stateConstraints.get().queryStates()) {
if (projectBridgehead.get().getQueryState() == queryState) {
hasAnyProjectBridgeheadStateConstraint = true;
break;
}
}
}
if (!hasAnyProjectBridgeheadStateConstraint) {
Expand Down
21 changes: 18 additions & 3 deletions src/main/java/de/samply/app/ProjectManagerConst.java
Original file line number Diff line number Diff line change
Expand Up @@ -238,12 +238,18 @@ public class ProjectManagerConst {
public static final String EXPORTER_PARAM_QUERY_CONTEXT = "query-context";
public static final String EXPORTER_PARAM_DEFAULT_OUTPUT_FORMAT = "query-default-output-format";
public static final String EXPORTER_PARAM_DEFAULT_TEMPLATE_ID = "query-default-template-id";
public final static String EXPORTER_PARAM_QUERY_EXECUTION_ID = "query-execution-id";
public static final String EXPORTER_QUERY_CONTEXT_PROJECT_ID = "PROJECT-ID";
public final static String EXPORTER_QUERY_CONTEXT_SEPARATOR = ";";
public final static String API_KEY = "ApiKey";


// Focus
public final static String FOCUS_METADATA_PROJECT = "exporter";
public final static String FOCUS_TASK = "/v1/tasks";
public final static String FOCUS_TASK_PATH = "/v1/tasks";
public final static String FOCUS_TASK_RESULTS_PATH = "/results";
public final static String FOCUS_TASK_WAIT_TIME_PARAM = "wait_time";
public final static String FOCUS_TASK_WAIT_COUNT_PARAM = "wait_count";

// Token Manager Variables
public final static String TOKEN_MANAGER_ROOT = "/api";
Expand Down Expand Up @@ -297,6 +303,7 @@ public class ProjectManagerConst {
public final static String TOKEN_MANAGER_URL = "TOKEN_MANAGER_URL";
public final static String ENABLE_EMAILS = "ENABLE_EMAILS";
public final static String MANAGE_TOKENS_CRON_EXPRESSION = "MANAGE_TOKENS_CRON_EXPRESSION";
public final static String EXPORTER_CRON_EXPRESSION = "EXPORTER_CRON_EXPRESSION";
public final static String CHECK_EXPIRED_ACTIVE_PROJECTS_CRON_EXPRESSION = "CHECK_EXPIRED_ACTIVE_PROJECTS_CRON_EXPRESSION";
public final static String EXPLORER_URL = "EXPLORER_URL";
public final static String ENABLE_TOKEN_MANAGER = "ENABLE_TOKEN_MANAGER";
Expand All @@ -309,6 +316,8 @@ public class ProjectManagerConst {
public final static String ENABLE_RSTUDIO_GROUP_MANAGER = "ENABLE_RSTUDIO_GROUP_MANAGER";
public final static String OIDC_URL = "OIDC_URL";
public final static String OIDC_REALM = "OIDC_REALM";
public final static String ENABLE_EXPORTER = "ENABLE_EXPORTER";
public final static String MAX_TIME_TO_WAIT_FOCUS_TASK_IN_MINUTES = "MAX_TIME_TO_WAIT_FOCUS_TASK_IN_MINUTES";

// Spring Values (SV)
public final static String HEAD_SV = "${";
Expand Down Expand Up @@ -337,6 +346,8 @@ public class ProjectManagerConst {
HEAD_SV + WEBCLIENT_TCP_KEEP_INTERVAL_IN_SECONDS + ":60" + BOTTOM_SV;
public final static String WEBCLIENT_TCP_KEEP_CONNECTION_NUMBER_OF_TRIES_SV =
HEAD_SV + WEBCLIENT_TCP_KEEP_CONNECTION_NUMBER_OF_TRIES + ":10" + BOTTOM_SV;
public final static String MAX_TIME_TO_WAIT_FOCUS_TASK_IN_MINUTES_SV =
HEAD_SV + MAX_TIME_TO_WAIT_FOCUS_TASK_IN_MINUTES + ":5" + BOTTOM_SV;
public final static String WEBCLIENT_MAX_NUMBER_OF_RETRIES_SV =
HEAD_SV + WEBCLIENT_MAX_NUMBER_OF_RETRIES + ":3" + BOTTOM_SV;
public final static String WEBCLIENT_TIME_IN_SECONDS_AFTER_RETRY_WITH_FAILURE_SV =
Expand All @@ -359,10 +370,13 @@ public class ProjectManagerConst {
public final static String FOCUS_API_KEY_SV = HEAD_SV + FOCUS_API_KEY + BOTTOM_SV;
public final static String ENABLE_EMAILS_SV = HEAD_SV + ENABLE_EMAILS + ":true" + BOTTOM_SV;
public final static String ENABLE_TOKEN_MANAGER_SV = HEAD_SV + ENABLE_TOKEN_MANAGER + ":true" + BOTTOM_SV;
public final static String ENABLE_EXPORTER_SV = HEAD_SV + ENABLE_EXPORTER + ":true" + BOTTOM_SV;
public final static String MANAGE_TOKENS_CRON_EXPRESSION_SV =
HEAD_SV + MANAGE_TOKENS_CRON_EXPRESSION + ":#{'0 0 * * * *'}" + BOTTOM_SV;
HEAD_SV + MANAGE_TOKENS_CRON_EXPRESSION + ":#{'0 * * * * *'}" + BOTTOM_SV;
public final static String EXPORTER_CRON_EXPRESSION_SV =
HEAD_SV + EXPORTER_CRON_EXPRESSION + ":#{'45 * * * * *'}" + BOTTOM_SV;
public final static String CHECK_EXPIRED_ACTIVE_PROJECTS_CRON_EXPRESSION_SV =
HEAD_SV + CHECK_EXPIRED_ACTIVE_PROJECTS_CRON_EXPRESSION + ":#{'0 0 1,13 * * *'}" + BOTTOM_SV;
HEAD_SV + CHECK_EXPIRED_ACTIVE_PROJECTS_CRON_EXPRESSION + ":#{'30 * * * * *'}" + BOTTOM_SV;
public final static String EXPLORER_URL_SV = HEAD_SV + EXPLORER_URL + BOTTOM_SV;
public final static String EXPLORER_REDIRECT_URI_PARAMETER_SV = HEAD_SV + EXPLORER_REDIRECT_URI_PARAMETER + BOTTOM_SV;
public final static String FRONTEND_PROJECT_CONFIG_SV = HEAD_SV + FRONTEND_PROJECT_CONFIG + BOTTOM_SV;
Expand All @@ -387,4 +401,5 @@ public class ProjectManagerConst {
public final static String CUSTOM_PROJECT_CONFIGURATION = "CUSTOM";
public final static String EMAIL_SERVICE = "EMAIL_SERVICE";


}
17 changes: 10 additions & 7 deletions src/main/java/de/samply/app/ProjectManagerController.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import de.samply.query.OutputFormat;
import de.samply.query.QueryFormat;
import de.samply.query.QueryService;
import de.samply.query.QueryState;
import de.samply.token.DataShieldTokenManagerService;
import de.samply.user.UserService;
import de.samply.user.roles.OrganisationRole;
Expand Down Expand Up @@ -307,7 +308,7 @@ public ResponseEntity<String> editProject(
queryService.editQuery(projectCode, (query != null && query.trim().length() > 0 && !query.equals("{}")) ? query : null, queryFormat, label, description, outputFormat, templateId, humanReadable, explorerUrl, queryContext);
return convertToResponseEntity(() -> this.frontendService.fetchExplorerRedirectUri(
ProjectManagerConst.PROJECT_VIEW_SITE,
Map.of(ProjectManagerConst.QUERY_CODE, projectCode)
Map.of(ProjectManagerConst.PROJECT_CODE, projectCode)
));
}

Expand Down Expand Up @@ -491,7 +492,7 @@ public ResponseEntity<String> rejectProject(
}

@RoleConstraints(projectRoles = {ProjectRole.BRIDGEHEAD_ADMIN})
@StateConstraints(projectStates = {ProjectState.DEVELOP, ProjectState.PILOT, ProjectState.FINAL})
@StateConstraints(projectStates = {ProjectState.DEVELOP, ProjectState.PILOT, ProjectState.FINAL}, queryStates = {QueryState.FINISHED, QueryState.ERROR})
@EmailSender(templateType = EmailTemplateType.PROJECT_BRIDGEHEAD_ACCEPTED, recipients = {EmailRecipientType.PROJECT_MANAGER_ADMIN})
@FrontendSiteModule(site = ProjectManagerConst.PROJECT_VIEW_SITE, module = ProjectManagerConst.PROJECT_STATE_MODULE)
@FrontendAction(action = ProjectManagerConst.ACCEPT_BRIDGEHEAD_PROJECT_ACTION)
Expand All @@ -504,7 +505,7 @@ public ResponseEntity<String> acceptBridgeheadProject(
}

@RoleConstraints(projectRoles = {ProjectRole.BRIDGEHEAD_ADMIN})
@StateConstraints(projectStates = {ProjectState.DEVELOP, ProjectState.PILOT, ProjectState.FINAL})
@StateConstraints(projectStates = {ProjectState.DEVELOP, ProjectState.PILOT, ProjectState.FINAL}, queryStates = {QueryState.FINISHED, QueryState.ERROR})
@EmailSender(templateType = EmailTemplateType.PROJECT_BRIDGEHEAD_REJECTED, recipients = {EmailRecipientType.PROJECT_MANAGER_ADMIN})
@FrontendSiteModule(site = ProjectManagerConst.PROJECT_VIEW_SITE, module = ProjectManagerConst.PROJECT_STATE_MODULE)
@FrontendAction(action = ProjectManagerConst.REJECT_BRIDGEHEAD_PROJECT_ACTION)
Expand Down Expand Up @@ -976,27 +977,29 @@ private ByteArrayResource fetchResource(Path filePath) throws DocumentServiceExc
}

@RoleConstraints(projectRoles = {ProjectRole.BRIDGEHEAD_ADMIN})
@StateConstraints(projectStates = {ProjectState.DEVELOP, ProjectState.PILOT, ProjectState.FINAL})
@StateConstraints(projectStates = {ProjectState.DEVELOP, ProjectState.PILOT, ProjectState.FINAL},
queryStates = {QueryState.CREATED, QueryState.ERROR, QueryState.FINISHED})
@FrontendSiteModule(site = ProjectManagerConst.PROJECT_VIEW_SITE, module = ProjectManagerConst.EXPORT_MODULE)
@FrontendAction(action = ProjectManagerConst.SAVE_QUERY_IN_BRIDGEHEAD_ACTION)
@PostMapping(value = ProjectManagerConst.SAVE_QUERY_IN_BRIDGEHEAD)
public ResponseEntity<String> saveQueryInBridgehead(
@NotEmpty @ProjectCode @RequestParam(name = ProjectManagerConst.PROJECT_CODE) String projectCode,
@NotEmpty @Bridgehead @RequestParam(name = ProjectManagerConst.BRIDGEHEAD) String bridgehead
) {
return convertToResponseEntity(() -> this.exporterService.sendQueryToBridgehead(projectCode, bridgehead));
return convertToResponseEntity(() -> this.projectBridgeheadService.scheduleSendQueryToBridgehead(projectCode, bridgehead));
}

@RoleConstraints(projectRoles = {ProjectRole.BRIDGEHEAD_ADMIN})
@StateConstraints(projectStates = {ProjectState.DEVELOP, ProjectState.PILOT, ProjectState.FINAL})
@StateConstraints(projectStates = {ProjectState.DEVELOP, ProjectState.PILOT, ProjectState.FINAL},
queryStates = {QueryState.CREATED, QueryState.ERROR, QueryState.FINISHED})
@FrontendSiteModule(site = ProjectManagerConst.PROJECT_VIEW_SITE, module = ProjectManagerConst.EXPORT_MODULE)
@FrontendAction(action = ProjectManagerConst.SAVE_AND_EXECUTE_QUERY_IN_BRIDGEHEAD_ACTION)
@PostMapping(value = ProjectManagerConst.SAVE_AND_EXECUTE_QUERY_IN_BRIDGEHEAD)
public ResponseEntity<String> saveAndExecuteQueryInBridgehead(
@ProjectCode @RequestParam(name = ProjectManagerConst.PROJECT_CODE) String projectCode,
@Bridgehead @RequestParam(name = ProjectManagerConst.BRIDGEHEAD) String bridgehead
) {
return convertToResponseEntity(() -> this.exporterService.sendQueryToBridgeheadAndExecute(projectCode, bridgehead));
return convertToResponseEntity(() -> this.projectBridgeheadService.scheduleSendQueryToBridgeheadAndExecute(projectCode, bridgehead));
}

@RoleConstraints(projectRoles = {ProjectRole.DEVELOPER, ProjectRole.PILOT, ProjectRole.FINAL})
Expand Down
15 changes: 15 additions & 0 deletions src/main/java/de/samply/db/model/ProjectBridgehead.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@


import de.samply.project.state.ProjectBridgeheadState;
import de.samply.query.QueryState;
import jakarta.persistence.*;
import lombok.AllArgsConstructor;
import lombok.Data;
Expand Down Expand Up @@ -35,4 +36,18 @@ public class ProjectBridgehead {
@Column(name = "modified_at", nullable = false)
private Instant modifiedAt = Instant.now();

@Column(name = "query_state", nullable = false)
@Enumerated(EnumType.STRING)
private QueryState queryState = QueryState.CREATED;

@Column(name = "exporter_response")
private String exporterResponse;

@Column(name = "exporter_user")
private String exporterUser;

@Column(name = "exporter_execution_id")
private String exporterExecutionId;


}
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@

import de.samply.db.model.Project;
import de.samply.db.model.ProjectBridgehead;
import de.samply.db.model.ProjectBridgeheadUser;
import de.samply.project.ProjectType;
import de.samply.project.state.ProjectBridgeheadState;
import de.samply.project.state.ProjectState;
import de.samply.query.QueryState;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.jpa.repository.JpaRepository;
Expand All @@ -32,4 +32,7 @@ public interface ProjectBridgeheadRepository extends JpaRepository<ProjectBridge
@Query("SELECT DISTINCT pb FROM ProjectBridgehead pb WHERE pb.project.type = :projectType AND pb.project.state NOT IN :projectStates")
List<ProjectBridgehead> getByProjectTypeAndNotProjectState(ProjectType projectType, Set<ProjectState> projectStates);

@Query("SELECT DISTINCT pb FROM ProjectBridgehead pb WHERE pb.queryState = :queryState AND pb.project.state IN :projectStates")
Set<ProjectBridgehead> getByQueryStateAndProjectState(QueryState queryState, Set<ProjectState> projectStates);

}
88 changes: 88 additions & 0 deletions src/main/java/de/samply/exporter/ExporterJob.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
package de.samply.exporter;

import de.samply.app.ProjectManagerConst;
import de.samply.db.model.ProjectBridgehead;
import de.samply.db.repository.ProjectBridgeheadRepository;
import de.samply.project.state.ProjectState;
import de.samply.query.QueryState;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;

import java.time.Instant;
import java.util.Optional;
import java.util.Set;
import java.util.function.Consumer;
import java.util.function.Function;

@Component
public class ExporterJob {

private final boolean enabled;
private final ExporterService exporterService;
private final ProjectBridgeheadRepository projectBridgeheadRepository;
private final Set<ProjectState> activeStates = Set.of(ProjectState.DEVELOP, ProjectState.PILOT, ProjectState.FINAL);

public ExporterJob(
@Value(ProjectManagerConst.ENABLE_EXPORTER_SV) boolean enabled,
ExporterService exporterService,
ProjectBridgeheadRepository projectBridgeheadRepository) {
this.enabled = enabled;
this.exporterService = exporterService;
this.projectBridgeheadRepository = projectBridgeheadRepository;
}

@Scheduled(cron = ProjectManagerConst.EXPORTER_CRON_EXPRESSION_SV)
public void checkExports() {
if (enabled) {
Mono.when(
checkQueriesToSend(),
checkQueriesToSendAndExecute(),
checkQueriesAlreadySent(),
checkQueriesAlreadySentToBeExecuted()).block();

}
}

private Mono<Void> checkQueriesToSend() {
return checkQueries(QueryState.TO_BE_SENT, QueryState.SENDING, exporterService::sendQueryToBridgehead);
}

private Mono<Void> checkQueriesToSendAndExecute() {
return checkQueries(QueryState.TO_BE_SENT_AND_EXECUTED, QueryState.SENDING_AND_EXECUTING, exporterService::sendQueryToBridgeheadAndExecute);
}

private Mono<Void> checkQueriesAlreadySent() {
return checkQueries(QueryState.SENDING, QueryState.FINISHED, exporterService::checkIfQueryIsAlreadySent);
}

private Mono<Void> checkQueriesAlreadySentToBeExecuted() {
return checkQueries(QueryState.SENDING_AND_EXECUTING, QueryState.FINISHED, exporterService::checkIfQueryIsAlreadySent, Optional.of(
exporterServiceResult ->
exporterService.fetchExporterExecutionIdFromExporterResponse(exporterServiceResult.result()).ifPresent(exportExecutionId ->
exporterServiceResult.projectBridgehead().setExporterExecutionId(exportExecutionId))));
}

private Mono<Void> checkQueries(QueryState initialQueryState, QueryState finalQueryState,
Function<ProjectBridgehead, Mono<ExporterServiceResult>> exporterServiceFunction) {
return checkQueries(initialQueryState, finalQueryState, exporterServiceFunction, Optional.empty());
}

private Mono<Void> checkQueries(QueryState initialQueryState, QueryState finalQueryState,
Function<ProjectBridgehead, Mono<ExporterServiceResult>> exporterServiceFunction, Optional<Consumer<ExporterServiceResult>> exporterServiceResultConsumer) {
return Flux.fromIterable(projectBridgeheadRepository.getByQueryStateAndProjectState(initialQueryState, activeStates))
.flatMap(exporterServiceFunction)
.flatMap(exporterServiceResult -> {
exporterServiceResultConsumer.ifPresent(consumer -> consumer.accept(exporterServiceResult));
ProjectBridgehead projectBridgehead = exporterServiceResult.projectBridgehead();
projectBridgehead.setQueryState(finalQueryState);
projectBridgehead.setExporterResponse(exporterServiceResult.result());
projectBridgehead.setModifiedAt(Instant.now());
projectBridgeheadRepository.save(projectBridgehead);
return Mono.empty();
}).then();
}

}
Loading

0 comments on commit 37ce9c2

Please sign in to comment.