diff --git a/gradle.properties b/gradle.properties index d3888371d..8ca24ce53 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,7 +6,7 @@ gormMongoVersion=7.3.0 grailsViewsVersion=2.3.2 assetPipelineVersion=3.3.4 elasticsearchVersion=7.17.9 -alaSecurityLibsVersion=6.1.0 +alaSecurityLibsVersion=6.2.0 mongoDBVersion=7.3.2 #22.x+ causes issues with mongo / GORM javax.validation.spi, might need grails 5 geoToolsVersion=21.5 diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 99586e492..ab87b1c4a 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -526,9 +526,6 @@ if (!security.cas.adminRole) { if (!ecodata.use.uuids) { ecodata.use.uuids = false } -if (!userDetails.url) { - userDetails.url = "https://auth-test.ala.org.au/userdetails/" -} if (!authGetKeyUrl) { authGetKeyUrl = "https://m.ala.org.au/mobileauth/mobileKey/generateKey" @@ -542,6 +539,10 @@ ecodata.documentation.exampleProjectUrl = 'http://ecodata-test.ala.org.au/ws/act // Used by ParatooService to sync available protocols paratoo.core.baseUrl = 'https://merit-test.core-api.paratoo.tern.org.au/api' +auth.baseUrl = 'https://auth-test.ala.org.au' +userDetails.web.url = "${auth.baseUrl}/userdetails/" +userDetails.api.url = "${auth.baseUrl}/userdetails/userDetails/" + if (!grails.cache.ehcache) { grails { cache { diff --git a/grails-app/controllers/au/org/ala/ecodata/ManagementUnitController.groovy b/grails-app/controllers/au/org/ala/ecodata/ManagementUnitController.groovy index 1e36ec05d..ae366f220 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ManagementUnitController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ManagementUnitController.groovy @@ -1,13 +1,5 @@ package au.org.ala.ecodata -import au.org.ala.ecodata.reporting.ManagementUnitXlsExporter -import au.org.ala.ecodata.reporting.XlsExporter -import org.apache.http.HttpStatus - -import java.text.ParseException -import java.time.Instant - - @RequireApiKey class ManagementUnitController { @@ -98,24 +90,6 @@ class ManagementUnitController { respond managementUnitService.managementUnitSiteMap(ids) } - - /** - * startDate and endDate need to be ISO 8601 - * - * Get reports of all management units in a given period - */ - def generateReportsInPeriod(){ - try{ - Map message = managementUnitService.generateReportsInPeriods(params.startDate, params.endDate, params.reportDownloadBaseUrl, params.senderEmail, params.systemEmail,params.email,params.getBoolean("summaryFlag", false)) - respond(message, status:200) - }catch ( ParseException e){ - def message = [message: 'Error: You need to provide startDate and endDate in the format of ISO 8601'] - respond(message, status:HttpStatus.SC_NOT_ACCEPTABLE) - }catch(Exception e){ - def message = [message: 'Fatal: ' + e.message] - respond(message, status:HttpStatus.SC_NOT_ACCEPTABLE) - } - } /** * Get financial years of managment unit reports cover * @return diff --git a/grails-app/controllers/au/org/ala/ecodata/OrganisationController.groovy b/grails-app/controllers/au/org/ala/ecodata/OrganisationController.groovy index 5142156cf..8fece4f7d 100644 --- a/grails-app/controllers/au/org/ala/ecodata/OrganisationController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/OrganisationController.groovy @@ -3,12 +3,11 @@ package au.org.ala.ecodata import grails.converters.JSON import static au.org.ala.ecodata.ElasticIndex.DEFAULT_INDEX - /** * Exposes web services to perform CRUD operations on an organisation. */ class OrganisationController { - + static responseFormats = ['json', 'xml'] OrganisationService organisationService ElasticSearchService elasticSearchService @@ -96,7 +95,6 @@ class OrganisationController { } } - private Map buildParams(Map params) { Map values = [:] diff --git a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy index 559da56c3..3ac0e0cc8 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy @@ -422,8 +422,26 @@ class ParatooController { } @PUT - @Path("/projects") - def updateProjectSites(String id) { + @Path("/projects/{id}") + @SecurityRequirements([@SecurityRequirement(name = "jwt"), @SecurityRequirement(name = "openIdConnect"), @SecurityRequirement(name = "oauth")]) + @Operation( + method = "PUT", + responses = [ + @ApiResponse(responseCode = "200", description = "Updated project", content = @Content(mediaType = "application/json", schema = @Schema(implementation = Map.class))), + @ApiResponse(responseCode = "403", description = "Forbidden"), + @ApiResponse(responseCode = "500", description = "Internal server error") + ], + requestBody = @RequestBody( + description = "Project sites to update", + required = true, + content = @Content( + mediaType = 'application/json', + schema = @Schema(implementation = Map.class) + ) + ), + tags = "Org Interface" + ) + def updateProjectSites(@Parameter(name = "id", description = "Project id", required = true, in = ParameterIn.PATH, schema = @Schema(type = "string"))String id) { String userId = userService.currentUserDetails.userId List projects = paratooService.userProjects(userId) ParatooProject project = projects?.find { it.id == id } diff --git a/grails-app/controllers/au/org/ala/ecodata/ReportController.groovy b/grails-app/controllers/au/org/ala/ecodata/ReportController.groovy index 98c3a5136..a7de54e9b 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ReportController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ReportController.groovy @@ -1,13 +1,15 @@ package au.org.ala.ecodata import grails.converters.JSON -import org.grails.web.json.JSONObject +import org.apache.http.HttpStatus +import java.text.ParseException class ReportController { static responseFormats = ['json', 'xml'] def reportingService + ReportService reportService def get(String id) { respond reportingService.get(id, false) @@ -89,4 +91,23 @@ class ReportController { render reportingService.aggregateReports(params.searchCriteria, params.reportConfig) as JSON } + + /** + * startDate and endDate need to be ISO 8601 + * + * Get reports of all management units in a given period + */ + @RequireApiKey + def generateReportsInPeriod(){ + try{ + Map message = reportService.generateReportsInPeriods(params.startDate, params.endDate, params.reportDownloadBaseUrl, params.senderEmail, params.systemEmail,params.email,params.getBoolean("summaryFlag", false), params.entity, params.hubId) + respond(message, status:200) + }catch ( ParseException e){ + def message = [message: 'Error: You need to provide startDate and endDate in the format of ISO 8601'] + respond(message, status: HttpStatus.SC_NOT_ACCEPTABLE) + }catch(Exception e){ + def message = [message: 'Fatal: ' + e.message] + respond(message, status:HttpStatus.SC_NOT_ACCEPTABLE) + } + } } diff --git a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy index 32ec96a46..f40570607 100644 --- a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy @@ -183,6 +183,7 @@ class UrlMappings { "/ws/report/runReport"(controller:"report", action:"runReport") + "/ws/report/generateReportsInPeriod"(controller:"report", action:"generateReportsInPeriod") "/ws/project/findByName"(controller: "project"){ action = [GET:"findByName"] } "/ws/project/importProjectsFromSciStarter"(controller: "project", action: "importProjectsFromSciStarter") diff --git a/grails-app/services/au/org/ala/ecodata/ManagementUnitService.groovy b/grails-app/services/au/org/ala/ecodata/ManagementUnitService.groovy index b4e08f3fd..43838b3af 100644 --- a/grails-app/services/au/org/ala/ecodata/ManagementUnitService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ManagementUnitService.groovy @@ -192,45 +192,6 @@ class ManagementUnitService { return [count: countOfReports, downloadId: downloadId] } - /** - * - * @param startDate - * @param endDate - * @param reportDownloadBaseUrl Base url of downloading generated report - * @param senderEmail - * @param systemEmail - * @param receiverEmail - * @return - */ - Map generateReportsInPeriods(String startDate, String endDate, String reportDownloadBaseUrl, String senderEmail, String systemEmail, String receiverEmail, boolean isSummary ){ - List reports = getReportingActivities(startDate,endDate) - int countOfReports = reports.sum{it.activities?.count{it.progress!=Activity.PLANNED}} - - Map params = [:] - params.fileExtension = "xlsx" - params.reportDownloadBaseUrl = reportDownloadBaseUrl - params.senderEmail = senderEmail - params.systemEmail = systemEmail - params.email = receiverEmail - - Closure doDownload = { File file -> - XlsExporter exporter = new XlsExporter(file.absolutePath) - ManagementUnitXlsExporter muXlsExporter = new ManagementUnitXlsExporter(exporter) - muXlsExporter.export(reports, isSummary) - exporter.sizeColumns() - exporter.save() - } - String downloadId = downloadService.generateReports(params, doDownload) - Map message =[:] - if (countOfReports>0){ - message = [message:"Your will receive an email notification when report is generated", details:downloadId] - }else{ - message = [message:"Your download will be emailed to you when it is complete.

WARNING, the period you requested may not have reports.", details: downloadId] - } - return message - } - - /** * Get reports of all management units in a period * diff --git a/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy b/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy index 019ffc223..b3155549e 100644 --- a/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy +++ b/grails-app/services/au/org/ala/ecodata/OrganisationService.groovy @@ -1,11 +1,10 @@ package au.org.ala.ecodata - import com.mongodb.client.model.Filters import grails.validation.ValidationException import org.bson.conversions.Bson -import static au.org.ala.ecodata.Status.* +import static au.org.ala.ecodata.Status.DELETED /** * Works with Organisations, mostly CRUD operations at this point. @@ -18,6 +17,8 @@ class OrganisationService { static transactional = 'mongo' def commonService, projectService, userService, permissionService, documentService, collectoryService, messageSource, emailService, grailsApplication + ReportingService reportingService + ActivityService activityService def get(String id, levelOfDetail = [], includeDeleted = false) { Organisation organisation @@ -174,12 +175,15 @@ class OrganisationService { * significant memory and performance overhead when dealing with so many entities * at once. * @param action the action to be performed on each Organisation. + * @param filters list of filters */ - void doWithAllOrganisations(Closure action) { + void doWithAllOrganisations(Closure action, List filters = []) { // Due to various memory & performance issues with GORM mongo plugin 1.3, this method uses the native API. def collection = Organisation.getCollection() //DBObject siteQuery = new QueryBuilder().start('status').notEquals(DELETED).get() - Bson query = Filters.ne("status", DELETED); + Bson query = Filters.ne("status", DELETED) + filters.add(query) + query = Filters.and(filters) def results = collection.find(query).batchSize(100) results.each { dbObject -> @@ -187,4 +191,30 @@ class OrganisationService { } } + /** + * Get reports of all management units in a period + * + * @param start + * @param end + * @param hubId + * @return + */ + List getReportingActivities (String startDate, String endDate, String hubId) { + List organisationDetails = [] + doWithAllOrganisations({ org -> + List reports = reportingService.search([organisationId: org.organisationId, dateProperty:'toDate', startDate:startDate, endDate:endDate]) + + List activities = activityService.search([activityId:reports.activityId], ['all']) + + Map result = new HashMap(org) + result.reports = reports + result.activities = activities + + organisationDetails << result + + }, [Filters.eq("hubId", hubId)]) + + organisationDetails + } + } diff --git a/grails-app/services/au/org/ala/ecodata/ReportService.groovy b/grails-app/services/au/org/ala/ecodata/ReportService.groovy index 2ed8530ab..3b3bacd9b 100644 --- a/grails-app/services/au/org/ala/ecodata/ReportService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ReportService.groovy @@ -17,6 +17,7 @@ import static au.org.ala.ecodata.ElasticIndex.HOMEPAGE_INDEX */ @Slf4j class ReportService { + static final String MU = 'managementUnit', ORG = 'organisation' ActivityService activityService ElasticSearchService elasticSearchService @@ -26,6 +27,9 @@ class ReportService { MetadataService metadataService UserService userService AuthService authService + DownloadService downloadService + ManagementUnitService managementUnitService + OrganisationService organisationService def findScoresByLabel(List labels) { Score.findAllByLabelInList(labels) @@ -463,4 +467,63 @@ class ReportService { List activityIds = Report.findAllByManagementUnitIdInList(muIds.toList()).activityId Date[] period = activityService.getPeriod(activityIds) } + + /** + * Generate report for entity + * @param startDate + * @param endDate + * @param reportDownloadBaseUrl Base url of downloading generated report + * @param senderEmail + * @param systemEmail + * @param receiverEmail + * @param entity managementUnit or organisation + * @param hubId + * @return + */ + Map generateReportsInPeriods(String startDate, String endDate, String reportDownloadBaseUrl, String senderEmail, String systemEmail, String receiverEmail, boolean isSummary, String entity, String hubId ){ + List reports + switch (entity) { + case ORG: + reports = organisationService.getReportingActivities(startDate, endDate, hubId) + break + case MU: + default: + reports = managementUnitService.getReportingActivities(startDate, endDate) + break + } + + int countOfReports = reports ? reports.sum{it.activities?.count{it.progress!=Activity.PLANNED}} : 0 + + Map params = [:] + params.fileExtension = "xlsx" + params.reportDownloadBaseUrl = reportDownloadBaseUrl + params.senderEmail = senderEmail + params.systemEmail = systemEmail + params.email = receiverEmail + + Closure doDownload = { File file -> + XlsExporter exporter = new XlsExporter(file.absolutePath) + switch (entity) { + case ORG: + OrganisationXlsExporter orgXlsExporter = new OrganisationXlsExporter(exporter, [], [:]) + orgXlsExporter.export(reports, isSummary) + break + case MU: + default: + ManagementUnitXlsExporter muXlsExporter = new ManagementUnitXlsExporter(exporter) + muXlsExporter.export(reports, isSummary) + break + } + exporter.sizeColumns() + exporter.save() + } + String downloadId = downloadService.generateReports(params, doDownload) + Map message =[:] + if (countOfReports>0){ + message = [message:"Your will receive an email notification when report is generated", details:downloadId] + }else{ + message = [message:"Your download will be emailed to you when it is complete.

WARNING, the period you requested may not have reports.", details: downloadId] + } + return message + } } diff --git a/src/main/groovy/au/org/ala/ecodata/reporting/OrganisationXlsExporter.groovy b/src/main/groovy/au/org/ala/ecodata/reporting/OrganisationXlsExporter.groovy index e45dd25ad..a408d60ee 100644 --- a/src/main/groovy/au/org/ala/ecodata/reporting/OrganisationXlsExporter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/reporting/OrganisationXlsExporter.groovy @@ -1,6 +1,7 @@ package au.org.ala.ecodata.reporting - +import au.org.ala.ecodata.DateUtil +import au.org.ala.ecodata.Report import org.apache.commons.logging.Log import org.apache.commons.logging.LogFactory import pl.touk.excel.export.XlsxExporter @@ -12,6 +13,8 @@ import pl.touk.excel.export.multisheet.AdditionalSheet class OrganisationXlsExporter extends TabbedExporter { static Log log = LogFactory.getLog(OrganisationXlsExporter.class) + // Avoids name clashes for fields that appear in organisation and activities + private static final String REPORT_PREFIX = 'report_' List commonOrganisationHeaders = ['Organisation ID', 'Name'] List commonOrganisationProperties = ['organisationId', 'name'] @@ -28,11 +31,50 @@ class OrganisationXlsExporter extends TabbedExporter { List reportDataHeaders = commonOrganisationHeaders + ['Report', 'Stage from', 'Stage to', 'Data Entry Progress', 'Report Status'] List reportDataProperties = commonOrganisationProperties + ['reportName', 'fromDate', 'toDate', 'progress', 'publicationStatus'] + List reportPropertiesMinimumSet = ['reportId', 'reportName', 'reportDescription', 'fromDate', 'toDate', 'financialYear'] + List activityHeaders = ['Activity Type','Activity Description','Activity Progress', 'Activity Last Updated' ] + List activityProperties = ['type','description','progress', 'lastUpdated'] + List commonActivityHeadersSummary = ["Organisation ID",'Organisation Name','Organisation ABN', 'Report ID', 'Report name', 'Report Description', 'From Date', 'To Date', 'Financial Year', 'Current Report Status', 'Date of status change', 'Changed by'] + List commonActivityHeaders = commonActivityHeadersSummary + activityHeaders + List commonActivityPropertiesSummary = ["organisationId",'organisationName','organisationABN', REPORT_PREFIX+'reportId', REPORT_PREFIX+'reportName', REPORT_PREFIX+'reportDescription', REPORT_PREFIX+'fromDate', REPORT_PREFIX+'toDate', REPORT_PREFIX+'financialYear', REPORT_PREFIX+'reportStatus', REPORT_PREFIX+'dateChanged', REPORT_PREFIX+'changedBy'] + List commonActivityProperties = commonActivityPropertiesSummary + + activityProperties.collect { + ACTIVITY_DATA_PREFIX+it + } + public OrganisationXlsExporter(XlsxExporter exporter, List tabsToExport, Map documentMap) { super(exporter, tabsToExport, documentMap) } + void export(List orgs, boolean isSummary = false) { + if(orgs.size() > 0) { + orgs.each { Map org -> + org.activities.each { Map activity -> + Report report = org.reports.find {it.activityId == activity.activityId} + if (report){ + activity['organisationId'] = org.organisationId + activity['organisationName'] = org.name + activity['organisationAbn'] = org.abn + + Map reportData = getReportSummaryInfo(report) + reportPropertiesMinimumSet.each { String prop -> + reportData[REPORT_PREFIX + prop] = reportData[prop] + } + reportData.putAll(extractCurrentReportStatus(report).collectEntries { k, v -> [REPORT_PREFIX + k, v] }) + activity.putAll(reportData) + + } + exportReport(activity, isSummary) + + } + } + }else{ + //Create a standard empty sheet to avoid malformed xslx + createEmptySheet("Organisations") + } + } + public void export(Map organisation) { exportOrganisation(organisation) exportReports(organisation) @@ -40,6 +82,34 @@ class OrganisationXlsExporter extends TabbedExporter { exportPerformanceAssessmentReport(organisation) } + protected Map getReportSummaryInfo(Report report) { + [stage:report?.name, reportName:report?.name, reportDescription:report?.description, reportId:report?.reportId, reportType:report?.generatedBy, fromDate:report?.fromDate, toDate:report?.toDate, financialYear: report ? DateUtil.getFinancialYearBasedOnEndDate(report.toDate) : ""] + } + + private void exportReport(Map activity, boolean isSummary = false){ + Map activityCommonData = convertActivityData(activity) + if (isSummary) { + exportActivity(commonActivityHeadersSummary, commonActivityPropertiesSummary, activityCommonData, activity, false, isSummary) + } else { + exportActivity(commonActivityHeaders, commonActivityProperties, activityCommonData, activity, false) + } + } + + /** + * Add activity prefix to each property to avoid name conflicts + * Combine Organisation info as well + * @param activity + * @return + */ + private Map convertActivityData(Map activity) { + activity.collectEntries{k,v -> + if (!k.startsWith('organisation') && !k.startsWith('report')) + [ACTIVITY_DATA_PREFIX+k, v] + else + [k, v] + } + } + private void exportOrganisation(Map organisation) { String orgSheetName = 'Organisations' if (shouldExport(orgSheetName)) {