Merge pull request #926 from AtlasOfLivingAustralia/dev

Preparing v4.5

build.gradle

@@ -20,7 +20,7 @@ plugins {
     id "com.gorylenko.gradle-git-properties" version "2.4.1"
 }
 
-version "4.4"
+version "4.5-SPECIES-SNAPSHOT"
 group "au.org.ala"
 description "Ecodata"
 

grails-app/conf/application.groovy

@@ -106,6 +106,9 @@ if (!google.geocode.url) {
 if (!temp.file.cleanup.days) {
     temp.file.cleanup.days = 1
 }
+if(!paratoo.location.excluded) {
+    paratoo.location.excluded = ['location.vegetation-association-nvis']
+}
 access.expiry.maxEmails=500
 
 
@@ -531,6 +534,7 @@ ecodata.documentation.exampleProjectUrl = 'http://ecodata-test.ala.org.au/ws/act
 // Used by ParatooService to sync available protocols
 paratoo.core.baseUrl = 'https://dev.core-api.monitor.tern.org.au/api'
 paratoo.excludeInterventionProtocols = true
+paratoo.core.documentationUrl = '/documentation/swagger.json'
 
 auth.baseUrl = 'https://auth-test.ala.org.au'
 userDetails.web.url = "${auth.baseUrl}/userdetails/"
@@ -1391,3 +1395,94 @@ elasticsearch {
     username = 'elastic'
     password = 'password'
 }
+
+// paratoo / monitor
+
+paratoo.defaultPlotLayoutDataModels =  [
+                [
+                        dataType: "geoMap",
+                        name: "plot_layout",
+                        validate: "required"
+                ],
+                [
+                        dataType: "list",
+                        name: "plot_visit",
+                        validate: "required",
+                        columns: [
+                                [
+                                        dataType: "date",
+                                        name: "end_date",
+                                        dwcAttribute: "eventDate"
+                                ],
+                                [
+                                        dataType: "text",
+                                        name: "visit_field_name"
+                                ],
+                                [
+                                        dataType: "date",
+                                        name: "start_date",
+                                        dwcAttribute: "eventDate"
+                                ]
+                        ]
+                ]
+        ]
+
+paratoo.defaultPlotLayoutViewModels = [
+                [
+                        type: "row",
+                        items: [
+                                [
+                                        type: "col",
+                                        items: [
+                                                [
+                                                        type: "section",
+                                                        title: "Plot Visit",
+                                                        preLabel: "Plot Visit",
+                                                        boxed: true,
+                                                        items: [
+                                                                [
+                                                                        type: "repeat",
+                                                                        source: "plot_visit",
+                                                                        userAddedRows: false,
+                                                                        items: [
+                                                                                [
+                                                                                        type: "row",
+                                                                                        class: "output-section",
+                                                                                        items: [
+                                                                                                [
+                                                                                                        type: "col",
+                                                                                                        items: [
+                                                                                                                [
+                                                                                                                        type: "date",
+                                                                                                                        source: "end_date",
+                                                                                                                        preLabel: "End Date"
+                                                                                                                ],
+                                                                                                                [
+                                                                                                                        type: "text",
+                                                                                                                        source: "visit_field_name",
+                                                                                                                        preLabel: "Visit Field Name"
+                                                                                                                ],
+                                                                                                                [
+                                                                                                                        type: "date",
+                                                                                                                        source: "start_date",
+                                                                                                                        preLabel: "Start Date"
+                                                                                                                ]
+                                                                                                        ]
+                                                                                                ]
+                                                                                        ]
+                                                                                ]
+                                                                        ]
+                                                                ]
+                                                        ]
+                                                ],
+                                                [
+                                                        type: "geoMap",
+                                                        source: "plot_layout",
+                                                        orientation: "vertical"
+                                                ]
+                                        ]
+                                ]
+                        ]
+                ]
+        ]
+

grails-app/conf/data/mapping.json

@@ -60,6 +60,10 @@
         "fundingType": {
           "type" : "keyword"
         },
+        "fundingVerificationDate": {
+          "type": "date",
+          "ignore_malformed": true
+        },
         "status": {
           "type" : "keyword"
         },

grails-app/controllers/au/org/ala/ecodata/AdminController.groovy

@@ -1,5 +1,8 @@
 package au.org.ala.ecodata
 
+import au.org.ala.ecodata.paratoo.ParatooCollection
+import au.org.ala.ecodata.paratoo.ParatooProject
+import au.org.ala.ecodata.paratoo.ParatooProtocolConfig
 import au.org.ala.web.AlaSecured
 import grails.converters.JSON
 import grails.util.Environment
@@ -725,4 +728,58 @@ class AdminController {
         render errors as JSON
     }
 
+    /**
+     * Re-fetch data from Paratoo. Helpful when data could not be parsed correctly the first time.
+     *
+     * @return
+     */
+    @AlaSecured(["ROLE_ADMIN"])
+    def reSubmitDataSet() {
+        String projectId = params.id
+        String dataSetId = params.dataSetId
+        String userId = params.userId ?: userService.getCurrentUser().userId
+        if (!projectId || !dataSetId || !userId) {
+            render text: [message: "Bad request"] as JSON, status: HttpStatus.SC_BAD_REQUEST
+            return
+        }
+
+        ParatooCollection collection = new ParatooCollection(orgMintedUUID: dataSetId, coreProvenance:  [:])
+        List<ParatooProject> projects = paratooService.userProjects(userId)
+        ParatooProject project = projects.find {it.project.projectId == projectId }
+        if (project) {
+            paratooService.submitCollection(collection, project, userId)
+            render text: [message: "Submitted request to fetch data for dataSet $dataSetId in project $projectId by user $userId"] as JSON, status: HttpStatus.SC_OK, contentType: 'application/json'
+        }
+        else {
+            render text: [message: "Project not found"] as JSON, status: HttpStatus.SC_NOT_FOUND
+        }
+    }
+
+
+    /**
+     * Helper function to check the form generated for a protocol during the sync operation.
+     * Usual step is to update Paratoo config in DB. Use this function to check the form generated.
+     * @return
+     */
+    @AlaSecured(["ROLE_ADMIN"])
+    def checkActivityFormForProtocol() {
+        String protocolId = params.id
+        List protocols = paratooService.getProtocolsFromParatoo()
+        Map protocol = protocols.find { it.attributes.identifier == protocolId }
+        if (!protocol) {
+            render text: [message: "Protocol not found"] as JSON, status: HttpStatus.SC_NOT_FOUND, contentType: 'application/json'
+            return
+        }
+
+        Map documentation = paratooService.getParatooSwaggerDocumentation()
+        ParatooProtocolConfig config = paratooService.getProtocolConfig(protocolId)
+        if (!config) {
+           render text: [message: "Protocol config not found"] as JSON, status: HttpStatus.SC_NOT_FOUND, contentType: 'application/json'
+            return
+        }
+
+        Map template = paratooService.buildTemplateForProtocol(protocol, documentation, config)
+        render text: template as JSON, status: HttpStatus.SC_OK, contentType: 'application/json'
+    }
+
 }

grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy

@@ -1,12 +1,7 @@
 package au.org.ala.ecodata
 
 import au.ala.org.ws.security.SkipApiKeyCheck
-import au.org.ala.ecodata.paratoo.ParatooCollection
-import au.org.ala.ecodata.paratoo.ParatooCollectionId
-import au.org.ala.ecodata.paratoo.ParatooPlotSelection
-import au.org.ala.ecodata.paratoo.ParatooPlotSelectionData
-import au.org.ala.ecodata.paratoo.ParatooProject
-import au.org.ala.ecodata.paratoo.ParatooToken
+import au.org.ala.ecodata.paratoo.*
 import groovy.util.logging.Slf4j
 import io.swagger.v3.oas.annotations.OpenAPIDefinition
 import io.swagger.v3.oas.annotations.Operation
@@ -33,7 +28,7 @@ import javax.ws.rs.PUT
 import javax.ws.rs.Path
 // Requiring these scopes will guarantee we can get a valid userId out of the process.
 @Slf4j
-@au.ala.org.ws.security.RequireApiKey(scopes=["profile", "openid"])
+@au.ala.org.ws.security.RequireApiKey(scopes = ["profile", "openid"])
 @OpenAPIDefinition(
         info = @Info(
                 title = "Ecodata APIs",
@@ -55,17 +50,17 @@ import javax.ws.rs.Path
                 type = SecuritySchemeType.OAUTH2,
                 flows = @OAuthFlows(
                         clientCredentials = @OAuthFlow(
-                            authorizationUrl = "https://auth-test.ala.org.au/cas/oidc/authorize",
-                            tokenUrl = "https://auth-test.ala.org.au/cas/oidc/token",
-                            refreshUrl = "https://auth-test.ala.org.au/cas/oidc/refresh",
-                            scopes = [
-                                    @OAuthScope(name="openid"),
-                                    @OAuthScope(name="profile"),
-                                    @OAuthScope(name="ala", description = "CAS scope"),
-                                    @OAuthScope(name="roles", description = "CAS scope"),
-                                    @OAuthScope(name="ala/attrs", description = "Cognito scope"),
-                                    @OAuthScope(name="ala/roles", description = "Cognito scope")
-                            ]
+                                authorizationUrl = "https://auth-test.ala.org.au/cas/oidc/authorize",
+                                tokenUrl = "https://auth-test.ala.org.au/cas/oidc/token",
+                                refreshUrl = "https://auth-test.ala.org.au/cas/oidc/refresh",
+                                scopes = [
+                                        @OAuthScope(name = "openid"),
+                                        @OAuthScope(name = "profile"),
+                                        @OAuthScope(name = "ala", description = "CAS scope"),
+                                        @OAuthScope(name = "roles", description = "CAS scope"),
+                                        @OAuthScope(name = "ala/attrs", description = "Cognito scope"),
+                                        @OAuthScope(name = "ala/roles", description = "Cognito scope")
+                                ]
                         )
                 ),
                 scheme = "bearer"
@@ -250,7 +245,7 @@ class ParatooController {
             error(collectionId.errors)
         } else {
             String userId = userService.currentUserDetails.userId
-            boolean hasProtocol = paratooService.protocolWriteCheck(userId, collectionId.surveyId.projectId, collectionId.surveyId.protocol.id)
+            boolean hasProtocol = paratooService.protocolWriteCheck(userId, collectionId.projectId, collectionId.protocolId)
             if (hasProtocol) {
                 Map mintResults = paratooService.mintCollectionId(userId, collectionId)
                 if (mintResults.error) {
@@ -282,24 +277,28 @@ class ParatooController {
 
         if (log.isDebugEnabled()) {
             log.debug("ParatooController::submitCollection")
-            log.debug(request.JSON.toString())
         }
         if (collection.hasErrors()) {
             error(collection.errors)
         } else {
             String userId = userService.currentUserDetails.userId
-            Map dataSet = paratooService.findDataSet(userId, collection.orgMintedIdentifier)
-
-            boolean hasProtocol = paratooService.protocolWriteCheck(userId, dataSet.project.id, collection.protocol.id)
-            if (hasProtocol) {
-                Map result = paratooService.submitCollection(collection, dataSet.project)
-                if (!result.error) {
-                    respond([success: true])
+            Map dataSet = paratooService.findDataSet(userId, collection.orgMintedUUID)
+            if (dataSet?.dataSet?.surveyId) {
+                ParatooCollectionId collectionId = ParatooCollectionId.fromMap(dataSet.dataSet.surveyId)
+                boolean hasProtocol = paratooService.protocolWriteCheck(userId, dataSet.project.id, collectionId.protocolId)
+                if (hasProtocol) {
+                    Map result = paratooService.submitCollection(collection, dataSet.project)
+                    if (!result.updateResult.error) {
+                        respond([success: true])
+                    } else {
+                        error(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.updateResult.error)
+                    }
                 } else {
-                    error(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.error)
+                    error(HttpStatus.SC_FORBIDDEN, "Project / protocol combination not available")
                 }
+
             } else {
-                error(HttpStatus.SC_FORBIDDEN, "Project / protocol combination not available")
+                error(HttpStatus.SC_NOT_FOUND, "No data set found with orgMintedUUID=${collection.orgMintedUUID}")
             }
         }
     }
@@ -394,8 +393,8 @@ class ParatooController {
                 plotSelections.addAll(it.plots)
             }
         }
-        plotSelections = plotSelections.unique {it.siteId} ?: []
-        respond plots:plotSelections
+        plotSelections = plotSelections.unique { it.siteId } ?: []
+        respond plots: plotSelections
     }
 
     private def addOrUpdatePlotSelection(ParatooPlotSelection plotSelection) {
@@ -440,7 +439,7 @@ class ParatooController {
             ),
             tags = "Org Interface"
     )
-    def updateProjectSites(@Parameter(name = "id", description = "Project id", required = true, in = ParameterIn.PATH, schema = @Schema(type = "string"))String id) {
+    def updateProjectSites(@Parameter(name = "id", description = "Project id", required = true, in = ParameterIn.PATH, schema = @Schema(type = "string")) String id) {
         String userId = userService.currentUserDetails.userId
         List projects = paratooService.userProjects(userId)
         ParatooProject project = projects?.find { it.id == id }
@@ -453,20 +452,19 @@ class ParatooController {
         Map result = paratooService.updateProjectSites(project, data.data, projects)
 
         if (result?.error) {
-            respond([message:result.error], status:HttpStatus.SC_INTERNAL_SERVER_ERROR)
-        }
-        else {
-            respond(buildUpdateProjectSitesResponse(id, data.data), status:HttpStatus.SC_OK)
+            respond([message: result.error], status: HttpStatus.SC_INTERNAL_SERVER_ERROR)
+        } else {
+            respond(buildUpdateProjectSitesResponse(id, data.data), status: HttpStatus.SC_OK)
         }
     }
 
     private static Map buildUpdateProjectSitesResponse(String id, Map data) {
         [
-            "data": [
-                    "id": id,
-                    "attributes": data
-            ],
-            meta: [:]
+                "data": [
+                        "id"        : id,
+                        "attributes": data
+                ],
+                meta  : [:]
         ]
     }
 

grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy

@@ -366,6 +366,17 @@ class ProjectController {
         }
     }
 
+    @RequireApiKey
+    def fetchDataSetRecords (String projectId, String dataSetId) {
+        if (projectId && dataSetId) {
+            List records = projectService.fetchDataSetRecords(projectId, dataSetId)
+            render text: records as JSON, contentType: 'application/json'
+        }
+        else {
+            render status: 400, text: "projectId and dataSetId are required parameters"
+        }
+    }
+
     def importProjectsFromSciStarter(){
         Integer count = projectService.importProjectsFromSciStarter()?:0
         render(text: [count: count] as JSON, contentType: 'application/json');

grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy

@@ -194,6 +194,7 @@ class UrlMappings {
         "/ws/project/getDataCollectionWhiteList"(controller: "project"){ action = [GET:"getDataCollectionWhiteList"] }
         "/ws/project/getBiocollectFacets"(controller: "project"){ action = [GET:"getBiocollectFacets"] }
         "/ws/project/getDefaultFacets"(controller: "project", action: "getDefaultFacets")
+        "/ws/project/$projectId/dataSet/$dataSetId/records"(controller: "project", action: "fetchDataSetRecords")
         "/ws/admin/initiateSpeciesRematch"(controller: "admin", action: "initiateSpeciesRematch")
 
         "/ws/document/download"(controller:"document", action:"download")