From 151d9b7ba8953226b78224af65d248017be98742 Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 30 Oct 2023 09:43:52 +1100 Subject: [PATCH 01/52] - initial merit commit --- .../org/ala/ecodata/ProjectController.groovy | 11 + .../au/org/ala/ecodata/UrlMappings.groovy | 1 + .../domain/au/org/ala/ecodata/Record.groovy | 9 + .../au/org/ala/ecodata/ParatooService.groovy | 92 +++++- .../au/org/ala/ecodata/ProjectService.groovy | 15 + .../paratoo/ParatooProtocolConfig.groovy | 111 ++++++- .../paratoo/ParatooProtocolConfigSpec.groovy | 49 ++- .../opportunisticSurveyObservations.json | 310 ++++++++++++++++++ 8 files changed, 582 insertions(+), 16 deletions(-) create mode 100644 src/test/resources/paratoo/opportunisticSurveyObservations.json diff --git a/grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy b/grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy index 9256bcf20..4a0263066 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ProjectController.groovy @@ -414,6 +414,17 @@ class ProjectController { } } + @RequireApiKey + def fetchDataSetRecords (String projectId, String dataSetId) { + if (projectId && dataSetId) { + List records = projectService.fetchDataSetRecords(projectId, dataSetId) + render text: records as JSON, contentType: 'application/json' + } + else { + render status: 400, text: "projectId and dataSetId are required parameters" + } + } + def importProjectsFromSciStarter(){ Integer count = projectService.importProjectsFromSciStarter()?:0 render(text: [count: count] as JSON, contentType: 'application/json'); diff --git a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy index cfde00677..3bbfaf0b6 100644 --- a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy @@ -193,6 +193,7 @@ class UrlMappings { "/ws/project/getDataCollectionWhiteList"(controller: "project"){ action = [GET:"getDataCollectionWhiteList"] } "/ws/project/getBiocollectFacets"(controller: "project"){ action = [GET:"getBiocollectFacets"] } "/ws/project/getDefaultFacets"(controller: "project", action: "getDefaultFacets") + "/ws/project/$projectId/dataSet/$dataSetId/records"(controller: "project", action: "fetchDataSetRecords") "/ws/admin/initiateSpeciesRematch"(controller: "admin", action: "initiateSpeciesRematch") "/ws/document/download"(controller:"document", action:"download") diff --git a/grails-app/domain/au/org/ala/ecodata/Record.groovy b/grails-app/domain/au/org/ala/ecodata/Record.groovy index 377c91f2c..af73a8619 100644 --- a/grails-app/domain/au/org/ala/ecodata/Record.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Record.groovy @@ -14,17 +14,22 @@ class Record { activityId index: true projectActivityId index: true lastUpdated index: true + dataSetId index: true version false } ObjectId id String projectId //ID of the project within ecodata String projectActivityId + String dataSetId String activityId String occurrenceID String outputSpeciesId // reference to output species outputSpeciesId. String userId String eventDate //should be a date in "yyyy-MM-dd" or "2014-11-24T04:55:48+11:00" format + String scientificName + String name + String vernacularName Double decimalLatitude Double decimalLongitude Double generalizedDecimalLatitude @@ -66,6 +71,10 @@ class Record { outputItemId nullable: true status nullable: true outputSpeciesId nullable: true + dataSetId nullable: true + name nullable: true + vernacularName nullable: true + scientificName nullable: true } String getRecordNumber(sightingsUrl){ diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 291d789f5..2d42615b5 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -1,19 +1,10 @@ package au.org.ala.ecodata - -import au.org.ala.ecodata.paratoo.ParatooCollection -import au.org.ala.ecodata.paratoo.ParatooCollectionId -import au.org.ala.ecodata.paratoo.ParatooMintedIdentifier -import au.org.ala.ecodata.paratoo.ParatooProject -import au.org.ala.ecodata.paratoo.ParatooProtocolConfig -import au.org.ala.ecodata.paratoo.ParatooSurveyId +import au.org.ala.ecodata.paratoo.* import au.org.ala.ws.tokens.TokenService import grails.converters.JSON import grails.core.GrailsApplication import groovy.util.logging.Slf4j - -import java.net.http.HttpHeaders - /** * Supports the implementation of the paratoo "org" interface */ @@ -173,10 +164,13 @@ class ParatooService { ParatooProtocolConfig config = getProtocolConfig(surveyId.protocol.id) Map surveyData = retrieveSurveyData(surveyId, config) + List surveyObservations = retrieveSurveyObservations(surveyId, config) if (surveyData) { // If we are unable to create a site, null will be returned - assigning a null siteId is valid. dataSet.siteId = createSiteFromSurveyData(surveyData, collection, surveyId, project.project, config) + List species = createSpeciesFromSurveyData(surveyObservations, collection, config, dataSet ) + dataSet.areSpeciesRecorded = species?.size() > 0 dataSet.startDate = config.getStartDate(surveyData) dataSet.endDate = config.getEndDate(surveyData) } @@ -223,6 +217,48 @@ class ParatooService { [dataSet:dataSet, project:project] } + private List createSpeciesFromSurveyData(List surveyObservations, ParatooCollection collection, ParatooProtocolConfig config, Map dataSet) { + // delete records + Record.where { + dataSetId == dataSet.dataSetId + }.deleteAll() + + createRecords(surveyObservations, config, collection, dataSet) + } + + private static List createRecords (List surveyObservations, ParatooProtocolConfig config, ParatooCollection collection, Map dataSet) { + List result = [] + surveyObservations?.each { observation -> + def obs = transformSpeciesObservation(observation, config, collection, dataSet) + def record = new Record(obs) + try { + record.save(flush: true, failOnError: true) + result.add(record) + } catch (Exception e) { + log.error("Error saving record: ${record.name} ${record.projectId}", e) + } + } + + result + } + + private static Map transformSpeciesObservation (Map observation, ParatooProtocolConfig config, ParatooSurveyId surveyId, ParatooCollection collection, Map dataSet) { + def lat = config.getDecimalLatitude(observation), lng = config.getDecimalLongitude(observation) + Map result = [ + dataSetId: dataSet.dataSetId, + projectId: surveyId.projectId, + eventDate: config.getEventDate(observation), + decimalLatitude: lat, + decimalLongitude: lng, + individualCount: config.getIndividualCount(observation), +// numberOfOrganisms: config.getNumberOfOrganisms(observation), + recordedBy: config.getRecordedBy(observation) + ] + + result << config.parseSpecies(config.getSpecies(observation)) + result + } + private String createSiteFromSurveyData(Map surveyData, ParatooCollection collection, ParatooSurveyId surveyId, Project project, ParatooProtocolConfig config) { String siteId = null // Create a site representing the location of the collection @@ -379,11 +415,47 @@ class ParatooService { survey } + List retrieveSurveyObservations(ParatooSurveyId surveyId, ParatooProtocolConfig config) { + + String apiEndpoint = config.observationEndpoint + String accessToken = tokenService.getAuthToken(true) + if (!accessToken?.startsWith('Bearer')) { + accessToken = 'Bearer '+accessToken + } + Map authHeader = [(MONITOR_AUTH_HEADER):accessToken] + + if (!accessToken) { + throw new RuntimeException("Unable to get access token") + } + int start = 0 + int limit = 10 + + + String url = paratooBaseUrl+'/'+apiEndpoint + String query = buildSurveyQueryString(start, limit) + Map response = webService.getJson(url+query, null, authHeader, false) + List data = config.findObservationsBelongingToSurvey(response.data, surveyId) ?: [] + int total = response.meta?.pagination?.total ?: 0 + while (!data && start+limit < total) { + start += limit + + query = buildSurveyQueryString(start, limit) + response = webService.getJson(url+query, null, authHeader, false) + data.addAll(config.findObservationsBelongingToSurvey(response.data, surveyId)) + } + + data + } + private static Map findMatchingSurvey(ParatooSurveyId surveyId, List data, ParatooProtocolConfig config) { data?.find { config.matches(it, surveyId) } } + private static List findSurveyData(ParatooSurveyId surveyId, Map surveyData, ParatooProtocolConfig config) { + surveyData?.data?.findAll { config.matches(it) } + } + Map plotSelections(String userId, Map plotSelectionData) { List projects = userProjects(userId) diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy index a3dcdcd2a..7a5b00344 100644 --- a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy @@ -44,6 +44,7 @@ class ProjectService { OrganisationService organisationService UserService userService ActivityFormService activityFormService + RecordService recordService /* def getCommonService() { grailsApplication.mainContext.commonService @@ -1034,4 +1035,18 @@ class ProjectService { count > 0 } + List fetchDataSetRecords (String projectId, String dataSetId) { + int batchSize = 10, count = 10, offset = 0 + List records = [] + while (batchSize == count) { + def response = Record.findAllByProjectIdAndDataSetId(projectId, dataSetId, [max: batchSize, offset: offset]) + count = records.size() + response = response.collect { recordService.toMap(it) } + records.addAll(response) + offset += count + } + + records + } + } \ No newline at end of file diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index c7030b693..692599754 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -4,6 +4,8 @@ import au.org.ala.ecodata.DateUtil import au.org.ala.ecodata.metadata.PropertyAccessor import groovy.util.logging.Slf4j +import java.util.regex.Matcher + /** * Configuration about how to work with a Paratoo/Monitor protocol */ @@ -11,6 +13,8 @@ import groovy.util.logging.Slf4j class ParatooProtocolConfig { String apiEndpoint + String observationEndpoint + String surveyType boolean usesPlotLayout = true String geometryType = 'Polygon' @@ -18,6 +22,12 @@ class ParatooProtocolConfig { String startDatePath = 'attributes.start_date_time' String endDatePath = 'attributes.end_date_time' String surveyIdPath = 'attributes.surveyId' + String observationSurveyIdPath = '' + String observationSpeciesPath = 'attributes.species' + String observationRecordedByPath = 'attributes.observers.observer' + String observationIndividualCountPath = 'attributes.number_of_individuals' + String observationEventDatePath = 'attributes.date_time' + String observationGeometryPath = 'attributes.location' String plotLayoutIdPath = 'attributes.plot_visit.data.attributes.plot_layout.data.id' String plotLayoutPointsPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_points' String plotSelectionPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_selection.data.attributes' @@ -46,19 +56,82 @@ class ParatooProtocolConfig { getProperty(surveyData, surveyIdPath) } + String getSpecies (Map observation) { + getProperty(observation, observationSpeciesPath) + } + + Map parseSpecies (String species) { + extractSpeciesName(species) + } + + List findObservationsBelongingToSurvey (List surveyData, ParatooSurveyId surveyId) { + surveyData?.findAll { Map data -> + Map dataLinkedToSurvey = getSurveyIdOfObservation( data ) + surveyEqualityTest(dataLinkedToSurvey, surveyId) + } + } + + Map getSurveyIdOfObservation (Map data) { + getProperty(data, observationSurveyIdPath) + } + + Map getEventDate (Map data) { + getProperty(data, observationEventDatePath) + } + + Double getDecimalLatitude (Map data) { + def shape = extractObservationSiteDataFromPath(data) + switch (shape.type) { + case 'Point': + return shape.coordinates[1] + } + } + + Double getDecimalLongitude (Map data) { + def shape = extractObservationSiteDataFromPath(data) + switch (shape.type) { + case 'Point': + return shape.coordinates[0] + } + + } + + Integer getIndividualCount (Map data) { + getProperty(data, observationIndividualCountPath) + } + + String getRecordedBy (Map data) { + def result = getProperty(data, observationRecordedByPath) + if (result instanceof List) { + return result.join(', ') + } + else if (result instanceof String) { + return result + } + } + + private Map extractObservationSiteDataFromPath(Map surveyData) { + def geometryData = getProperty(surveyData, observationGeometryPath) + extractGeometryFromSiteData(geometryData) + } + private Map extractSiteDataFromPath(Map surveyData) { - Map geometry = null def geometryData = getProperty(surveyData, geometryPath) + extractGeometryFromSiteData(geometryData) + } + + private Map extractGeometryFromSiteData(geometryData) { + Map geometry = null if (geometryData) { switch (geometryType) { case 'Point': - geometry = [type:'Point', coordinates:[geometryData.lng, geometryData.lat]] + geometry = [type: 'Point', coordinates: [geometryData.lng, geometryData.lat]] break } + } else { + log.warn("Unable to get spatial data from survey: " + apiEndpoint + ", " + geometryPath) } - else { - log.warn("Unable to get spatial data from survey: "+apiEndpoint+", "+geometryPath) - } + geometry } @@ -94,6 +167,10 @@ class ParatooProtocolConfig { boolean matches(Map surveyData, ParatooSurveyId surveyId) { Map tmpSurveyId = getSurveyId(surveyData) + surveyEqualityTest(tmpSurveyId, surveyId) + } + + static boolean surveyEqualityTest(Map tmpSurveyId, ParatooSurveyId surveyId) { tmpSurveyId.surveyType == surveyId.surveyType && tmpSurveyId.time == surveyId.timeAsISOString() && tmpSurveyId.randNum == surveyId.randNum @@ -173,4 +250,28 @@ class ParatooProtocolConfig { ] geoJson } + + static Map extractSpeciesName (String species) { + Map result + Matcher m = (species =~ /([^\[]*)\[([^\]]*)\]\s*\(scientific: ([^\)]*)\)/) + if (m.matches()) { + def scientificName = m.group(3).trim() + def commonName = m.group(1)?.trim() + def name = "$scientificName ($commonName)" + result = [ + vernacularName: commonName, + scientificName: scientificName, + name: name + ] + } + else { + result = [ + vernacularName: species, + scientificName: species, + name: species + ] + } + + result + } } diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index 62c883363..345ae69f8 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -2,11 +2,12 @@ package au.org.ala.ecodata.paratoo import grails.converters.JSON import groovy.json.JsonSlurper -import org.geotools.geojson.geom.GeometryJSON import org.grails.web.converters.marshaller.json.CollectionMarshaller import org.grails.web.converters.marshaller.json.MapMarshaller import spock.lang.Specification +import static org.joda.time.DateTime.parse + class ParatooProtocolConfigSpec extends Specification { def setup() { @@ -19,6 +20,11 @@ class ParatooProtocolConfigSpec extends Specification { new JsonSlurper().parse(url) } + private Map readSurveyObservations(String name) { + URL url = getClass().getResource("/paratoo/${name}.json") + new JsonSlurper().parse(url) + } + def "The vegetation-mapping-survey can be used with this config"() { setup: @@ -79,4 +85,45 @@ class ParatooProtocolConfigSpec extends Specification { properties:["name":"SATFLB0001 - Control (100 x 100)", externalId:4, description:"SATFLB0001 - Control (100 x 100)", notes:"some comment"]] } + + def "The observations from opportunistic-survey can be filtered" () { + setup: + Map surveyObservations = readSurveyObservations('opportunisticSurveyObservations') + Map opportunisticSurveyConfig = [ + apiEndpoint:'opportunistic-surveys', + observationEndpoint: 'opportunistic-observations', + surveyType: 'opportunistic-survey', + usesPlotLayout:false, + geometryType: 'Point', + startDatePath: 'attributes.startdate', + endDatePath: 'attributes.updatedAt', + observationSurveyIdPath: 'attributes.opportunistic_survey.data.attributes.surveyId' + ] + ParatooProtocolConfig config = new ParatooProtocolConfig(opportunisticSurveyConfig) + ParatooSurveyId paratooSurveyId = new ParatooSurveyId( + surveyType: 'opportunistic-survey', + time: parse("2023-10-24T00:59:48.456Z").toDate(), + randNum: 80454523, + projectId: '0d02b422-5bf7-495f-b9f2-fa0a3046937f', + protocol: new ParatooProtocolId(id: "068d17e8-e042-ae42-1e42-cff4006e64b0", version: 1) + ) + List data = config.findObservationsBelongingToSurvey(surveyObservations.data, paratooSurveyId) + + expect: + data.size() == 1 + data[0].attributes.observation_id == 'OPP001' + + when: + Map species = config.parseSpecies(config.getSpecies(data[0])) + + then: + species.scientificName == "Dromaius novaehollandiae" + species.name == "Dromaius novaehollandiae (Emu)" + species.vernacularName == "Emu" + config.getDecimalLatitude(data[0]) == -35.272 + config.getDecimalLongitude(data[0]) == 149.116 + config.getIndividualCount(data[0]) == 1 + config.getRecordedBy(data[0]) == "xyz, abc" + + } } diff --git a/src/test/resources/paratoo/opportunisticSurveyObservations.json b/src/test/resources/paratoo/opportunisticSurveyObservations.json new file mode 100644 index 000000000..8253c6361 --- /dev/null +++ b/src/test/resources/paratoo/opportunisticSurveyObservations.json @@ -0,0 +1,310 @@ +{ + "data": [ + { + "id": 1, + "attributes": { + "observation_id": "OPP001", + "date_time": "2023-10-24T00:59:48.532Z", + "species": "Emu [sp] (scientific: Dromaius novaehollandiae)", + "confident": true, + "number_of_individuals": 1, + "other_observation_method_tier_2": null, + "observation_notes": null, + "add_additional_fauna_data": false, + "createdAt": "2023-10-24T01:01:56.651Z", + "updatedAt": "2023-10-24T01:01:56.651Z", + "media": [], + "location": { + "id": 40, + "lat": -35.272, + "lng": 149.116 + }, + "taxa_type": { + "data": { + "id": 4, + "attributes": { + "symbol": "BI", + "label": "Bird", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/f9874c91-f61d-5b74-90d1-aa71d3805b45", + "createdAt": "2023-10-23T05:56:50.068Z", + "updatedAt": "2023-10-23T05:56:50.068Z" + } + } + }, + "exact_or_estimate": { + "data": { + "id": 2, + "attributes": { + "symbol": "Estimate", + "label": "Estimate", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/68c919a7-5759-5169-92dd-a99b6b7be407", + "createdAt": "2023-10-23T05:56:09.675Z", + "updatedAt": "2023-10-23T05:56:09.675Z" + } + } + }, + "observation_method_tier_1": { + "data": null + }, + "observation_method_tier_2": { + "data": null + }, + "observation_method_tier_3": { + "data": null + }, + "observers": [ + { + "id": 7, + "observer": "xyz" + }, + { + "id": 8, + "observer": "abc" + } + ], + "habitat_description": { + "data": { + "id": 10, + "attributes": { + "symbol": "MVG10", + "label": "Other Forests and Woodlands", + "description": "", + "uri": "", + "createdAt": "2023-10-23T05:56:26.236Z", + "updatedAt": "2023-10-23T05:56:26.236Z" + } + } + }, + "growth_form_and_life_stage": [], + "selections_option": [], + "vouchered_specimens": [], + "opportunistic_survey": { + "data": { + "id": 4, + "attributes": { + "surveyId": { + "time": "2023-10-24T00:59:48.456Z", + "randNum": 80454523, + "protocol": { + "id": "068d17e8-e042-ae42-1e42-cff4006e64b0", + "version": 1 + }, + "projectId": "0d02b422-5bf7-495f-b9f2-fa0a3046937f", + "surveyType": "opportunistic-survey" + }, + "startdate": null, + "createdAt": "2023-10-24T01:01:56.529Z", + "updatedAt": "2023-10-24T01:01:56.529Z" + } + } + } + } + }, + { + "id": 1, + "attributes": { + "observation_id": "OPP002", + "date_time": "2023-10-26T00:59:48.532Z", + "species": "Emu [sp] (scientific: Dromaius novaehollandiae)", + "confident": true, + "number_of_individuals": 1, + "other_observation_method_tier_2": null, + "observation_notes": null, + "add_additional_fauna_data": false, + "createdAt": "2023-10-24T01:01:56.651Z", + "updatedAt": "2023-10-24T01:01:56.651Z", + "media": [], + "location": { + "id": 40, + "lat": -35.272, + "lng": 149.116 + }, + "taxa_type": { + "data": { + "id": 4, + "attributes": { + "symbol": "BI", + "label": "Bird", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/f9874c91-f61d-5b74-90d1-aa71d3805b45", + "createdAt": "2023-10-23T05:56:50.068Z", + "updatedAt": "2023-10-23T05:56:50.068Z" + } + } + }, + "exact_or_estimate": { + "data": { + "id": 2, + "attributes": { + "symbol": "Estimate", + "label": "Estimate", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/68c919a7-5759-5169-92dd-a99b6b7be407", + "createdAt": "2023-10-23T05:56:09.675Z", + "updatedAt": "2023-10-23T05:56:09.675Z" + } + } + }, + "observation_method_tier_1": { + "data": null + }, + "observation_method_tier_2": { + "data": null + }, + "observation_method_tier_3": { + "data": null + }, + "observers": [ + { + "id": 7, + "observer": "xyz" + } + ], + "habitat_description": { + "data": { + "id": 10, + "attributes": { + "symbol": "MVG10", + "label": "Other Forests and Woodlands", + "description": "", + "uri": "", + "createdAt": "2023-10-23T05:56:26.236Z", + "updatedAt": "2023-10-23T05:56:26.236Z" + } + } + }, + "growth_form_and_life_stage": [], + "selections_option": [], + "vouchered_specimens": [], + "opportunistic_survey": { + "data": { + "id": 4, + "attributes": { + "surveyId": { + "time": "2023-10-24T00:59:48.456Z", + "randNum": 80775523, + "protocol": { + "id": "068d17e8-e042-ae42-1e42-cff4006e64b0", + "version": 1 + }, + "projectId": "0d02b422-5bf7-495f-b9f2-fa0a3046937f", + "surveyType": "opportunistic-survey" + }, + "startdate": null, + "createdAt": "2023-10-24T01:01:56.529Z", + "updatedAt": "2023-10-24T01:01:56.529Z" + } + } + } + } + }, + { + "id": 1, + "attributes": { + "observation_id": "OPP003", + "date_time": "2023-10-25T00:59:48.532Z", + "species": "Emu [sp] (scientific: Dromaius novaehollandiae)", + "confident": true, + "number_of_individuals": 1, + "other_observation_method_tier_2": null, + "observation_notes": null, + "add_additional_fauna_data": false, + "createdAt": "2023-10-24T01:01:56.651Z", + "updatedAt": "2023-10-24T01:01:56.651Z", + "media": [], + "location": { + "id": 40, + "lat": -35.272, + "lng": 149.116 + }, + "taxa_type": { + "data": { + "id": 4, + "attributes": { + "symbol": "BI", + "label": "Bird", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/f9874c91-f61d-5b74-90d1-aa71d3805b45", + "createdAt": "2023-10-23T05:56:50.068Z", + "updatedAt": "2023-10-23T05:56:50.068Z" + } + } + }, + "exact_or_estimate": { + "data": { + "id": 2, + "attributes": { + "symbol": "Estimate", + "label": "Estimate", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/68c919a7-5759-5169-92dd-a99b6b7be407", + "createdAt": "2023-10-23T05:56:09.675Z", + "updatedAt": "2023-10-23T05:56:09.675Z" + } + } + }, + "observation_method_tier_1": { + "data": null + }, + "observation_method_tier_2": { + "data": null + }, + "observation_method_tier_3": { + "data": null + }, + "observers": [ + { + "id": 7, + "observer": "xyz" + } + ], + "habitat_description": { + "data": { + "id": 10, + "attributes": { + "symbol": "MVG10", + "label": "Other Forests and Woodlands", + "description": "", + "uri": "", + "createdAt": "2023-10-23T05:56:26.236Z", + "updatedAt": "2023-10-23T05:56:26.236Z" + } + } + }, + "growth_form_and_life_stage": [], + "selections_option": [], + "vouchered_specimens": [], + "opportunistic_survey": { + "data": { + "id": 4, + "attributes": { + "surveyId": { + "time": "2023-10-24T00:59:48.456Z", + "randNum": 804993, + "protocol": { + "id": "068d17e8-e042-ae42-1e42-cff4006e64b0", + "version": 1 + }, + "projectId": "0d02b422-5bf7-495f-b9f2-fa0a3046937f", + "surveyType": "opportunistic-survey" + }, + "startdate": null, + "createdAt": "2023-10-24T01:01:56.529Z", + "updatedAt": "2023-10-24T01:01:56.529Z" + } + } + } + } + } + ], + "meta": { + "pagination": { + "page": 1, + "pageSize": 1000, + "pageCount": 1, + "total": 3 + } + } +} \ No newline at end of file From a3e6fd367e89abf7781567c568a3af88c4646a0a Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 15 Dec 2023 15:23:12 +1100 Subject: [PATCH 02/52] #902 - enabling access to harvest APIs via JWT token --- .../org/ala/ecodata/ApiKeyInterceptor.groovy | 4 +- .../org/ala/ecodata/HarvestController.groovy | 137 ++++++++++++++++++ .../org/ala/ecodata/RecordController.groovy | 133 +---------------- .../au/org/ala/ecodata/UrlMappings.groovy | 4 +- .../au/org/ala/ecodata/UserService.groovy | 4 +- 5 files changed, 145 insertions(+), 137 deletions(-) create mode 100644 grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy diff --git a/grails-app/controllers/au/org/ala/ecodata/ApiKeyInterceptor.groovy b/grails-app/controllers/au/org/ala/ecodata/ApiKeyInterceptor.groovy index 0ce1e1b56..f91f15a71 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ApiKeyInterceptor.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ApiKeyInterceptor.groovy @@ -21,7 +21,7 @@ class ApiKeyInterceptor { public ApiKeyInterceptor() { // These controllers use JWT authorization instead - matchAll().excludes(controller: 'graphql').excludes(controller: 'paratoo') + matchAll().excludes(controller: 'graphql').excludes(controller: 'paratoo').excludes(controller: 'harvest') } boolean before() { @@ -29,7 +29,7 @@ class ApiKeyInterceptor { Class controllerClass = controller?.clazz // The "excludes" configuration in the constructor isn't working - if (controllerClass == ParatooController.class) { + if ( [ParatooController.class, HarvestController.class].contains( controllerClass ) ) { return true } diff --git a/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy b/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy new file mode 100644 index 000000000..98485525e --- /dev/null +++ b/grails-app/controllers/au/org/ala/ecodata/HarvestController.groovy @@ -0,0 +1,137 @@ +package au.org.ala.ecodata + +import grails.converters.JSON +import org.apache.http.HttpStatus + +import java.text.SimpleDateFormat; + +@au.ala.org.ws.security.RequireApiKey +class HarvestController { + + RecordService recordService + ProjectService projectService + ProjectActivityService projectActivityService + UserService userService + + /** + * List of supported data resource id available for harvesting. + * Note: Data Provider must be BioCollect or MERIT + * + * @param max = number + * @param offset = number + * @param order = lastUpdated + * @param sort = asc | desc + * + */ + def listHarvestDataResource() { + def result, error + try { + if (params.max && !params.max?.isNumber()) { + error = "Invalid parameter max" + } else if (params.offset && !params.offset?.isNumber()) { + error = "Invalid parameter offset" + } else if (params.sort && params.sort != "asc" && params.sort != "desc") { + error = "Invalid parameter sort" + } else if (params.order && params.order != "lastUpdated") { + error = "Invalid parameter order (Expected: lastUpdated)" + } + + if (!error) { + def pagination = [ + max : params.max ?: 10, + offset: params.offset ?: 0, + order : params.order ?: 'asc', + sort : params.sort ?: 'lastUpdated' + ] + + result = projectService.listProjectForAlaHarvesting(pagination) + + } else { + response.status = HttpStatus.SC_BAD_REQUEST + result = [status: 'error', error: error] + } + + } catch (Exception ex) { + response.status = HttpStatus.SC_INTERNAL_SERVER_ERROR + result << [status: 'error', error: "Unexpected error."] + } + + response.setContentType("application/json") + render result as JSON + } + + /** + * List records associated with the given data resource id + * Data Provider must be BioCollect or MERIT + * @param id dataResourceId + * @param max = number + * @param offset = number + * @param order = lastUpdated + * @param sort = asc | desc | default:asc + * @param lastUpdated = date | dd/MM/yyyy | default:null + * @param status = active | deleted | default:active + * + */ + def listRecordsForDataResourceId (){ + def result = [], error, project + Date lastUpdated = null + try { + if(!params.id) { + error = "Invalid data resource id" + } else if (params.max && !params.max.isNumber()) { + error = "Invalid max parameter vaue" + } else if (params.offset && !params.offset.isNumber()) { + error = "Invalid offset parameter vaue" + } else if (params.order && params.order != "asc" && params.order != "desc") { + error = "Invalid order parameter value (expected: asc, desc)" + } else if (params.sort && params.sort != "lastUpdated") { + error = "Invalid sort parameter value (expected: lastUpdated)" + } else if (params.status && params.status != "active" && params.status != "deleted") { + error = "Invalid status parameter value (expected: active or deleted)" + } else if(params.id){ + project = projectService.getByDataResourceId(params.id, 'active', 'basic') + error = !project ? 'No valid project found for the given data resource id' : !project.alaHarvest ? "Harvest disabled for data resource id - ${params.id}" : '' + } + + if (params.lastUpdated) { + try{ + def df = new SimpleDateFormat("dd/MM/yyyy") + lastUpdated = df.parse(params.lastUpdated) + } catch (Exception ex) { + error = "Invalid lastUpdated format (Expected date format - Example: dd/MM/yyyy" + } + } + + if (!error && project) { + def args = [ + max : params.max ?: 10, + offset : params.offset ?: 0, + order : params.order ?: 'asc', + sort : params.sort ?: 'lastUpdated', + status : params.status ?: 'active', + projectId: project.projectId + ] + + List restrictedProjectActivities = projectActivityService.listRestrictedProjectActivityIds(null, params.id) + log.debug("Retrieving results...") + result = recordService.listByProjectId(args, lastUpdated, restrictedProjectActivities) + result?.list?.each { + it.projectName = project?.name + it.license = recordService.getLicense(it) + } + } else { + response.status = HttpStatus.SC_BAD_REQUEST + log.error(error.toString()) + result = [status: 'error', error: error] + } + + } catch (Exception ex) { + response.status = HttpStatus.SC_INTERNAL_SERVER_ERROR + log.error(ex.toString()) + result << [status: 'error', error: "Unexpected error."] + } + + response.setContentType("application/json") + render result as JSON + } +} \ No newline at end of file diff --git a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy index 2e190a36d..108db9c16 100644 --- a/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/RecordController.groovy @@ -1,18 +1,13 @@ package au.org.ala.ecodata -import org.apache.http.HttpStatus - -import java.text.SimpleDateFormat - -import static au.org.ala.ecodata.Status.* -import static javax.servlet.http.HttpServletResponse.* - import grails.converters.JSON import groovy.json.JsonSlurper import org.apache.commons.codec.binary.Base64 import org.springframework.web.multipart.MultipartFile import org.springframework.web.multipart.MultipartHttpServletRequest +import static au.org.ala.ecodata.Status.DELETED +import static javax.servlet.http.HttpServletResponse.* /** * Controller for record CRUD operations with support for handling images. */ @@ -30,130 +25,6 @@ class RecordController { def index() {} - /** - * List of supported data resource id available for harvesting. - * Note: Data Provider must be BioCollect or MERIT - * - * @param max = number - * @param offset = number - * @param order = lastUpdated - * @param sort = asc | desc - * - */ - @PreAuthorise - def listHarvestDataResource() { - def result, error - try { - if (params.max && !params.max?.isNumber()) { - error = "Invalid parameter max" - } else if (params.offset && !params.offset?.isNumber()) { - error = "Invalid parameter offset" - } else if (params.sort && params.sort != "asc" && params.sort != "desc") { - error = "Invalid parameter sort" - } else if (params.order && params.order != "lastUpdated") { - error = "Invalid parameter order (Expected: lastUpdated)" - } - - if (!error) { - def pagination = [ - max : params.max ?: 10, - offset: params.offset ?: 0, - order : params.order ?: 'asc', - sort : params.sort ?: 'lastUpdated' - ] - - result = projectService.listProjectForAlaHarvesting(pagination) - - } else { - response.status = HttpStatus.SC_BAD_REQUEST - result = [status: 'error', error: error] - } - - } catch (Exception ex) { - response.status = HttpStatus.SC_INTERNAL_SERVER_ERROR - result << [status: 'error', error: "Unexpected error."] - } - - response.setContentType("application/json") - render result as JSON - } - - /** - * List records associated with the given data resource id - * Data Provider must be BioCollect or MERIT - * @param id dataResourceId - * @param max = number - * @param offset = number - * @param order = lastUpdated - * @param sort = asc | desc | default:asc - * @param lastUpdated = date | dd/MM/yyyy | default:null - * @param status = active | deleted | default:active - * - */ - @PreAuthorise - def listRecordsForDataResourceId (){ - def result = [], error, project - Date lastUpdated = null - try { - if(!params.id) { - error = "Invalid data resource id" - } else if (params.max && !params.max.isNumber()) { - error = "Invalid max parameter vaue" - } else if (params.offset && !params.offset.isNumber()) { - error = "Invalid offset parameter vaue" - } else if (params.order && params.order != "asc" && params.order != "desc") { - error = "Invalid order parameter value (expected: asc, desc)" - } else if (params.sort && params.sort != "lastUpdated") { - error = "Invalid sort parameter value (expected: lastUpdated)" - } else if (params.status && params.status != "active" && params.status != "deleted") { - error = "Invalid status parameter value (expected: active or deleted)" - } else if(params.id){ - project = projectService.getByDataResourceId(params.id, 'active', 'basic') - error = !project ? 'No valid project found for the given data resource id' : !project.alaHarvest ? "Harvest disabled for data resource id - ${params.id}" : '' - } - - if (params.lastUpdated) { - try{ - def df = new SimpleDateFormat("dd/MM/yyyy") - lastUpdated = df.parse(params.lastUpdated) - } catch (Exception ex) { - error = "Invalid lastUpdated format (Expected date format - Example: dd/MM/yyyy" - } - } - - if (!error && project) { - def args = [ - max : params.max ?: 10, - offset : params.offset ?: 0, - order : params.order ?: 'asc', - sort : params.sort ?: 'lastUpdated', - status : params.status ?: 'active', - projectId: project.projectId - ] - - List restrictedProjectActivities = projectActivityService.listRestrictedProjectActivityIds(null, params.id) - log.debug("Retrieving results...") - result = recordService.listByProjectId(args, lastUpdated, restrictedProjectActivities) - result?.list?.each { - it.projectName = project?.name - it.license = recordService.getLicense(it) - } - } else { - response.status = HttpStatus.SC_BAD_REQUEST - log.error(error.toString()) - result = [status: 'error', error: error] - } - - } catch (Exception ex) { - response.status = HttpStatus.SC_INTERNAL_SERVER_ERROR - log.error(ex.toString()) - result << [status: 'error', error: "Unexpected error."] - } - - response.setContentType("application/json") - render result as JSON - } - /** * Exports all active Records with lat/lng coords into a .csv suitable for use by the Biocache to create occurrence records. diff --git a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy index 3bbfaf0b6..05ccc906f 100644 --- a/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/UrlMappings.groovy @@ -14,8 +14,8 @@ class UrlMappings { "/ws/record/images"(controller: "record"){ action = [GET:"listRecordWithImages"] } "/ws/record/images/"(controller: "record"){ action = [GET:"listRecordWithImages"] } "/ws/record/getRecordForOutputSpeciesId/"(controller: "record", action: "getRecordForOutputSpeciesId") - "/ws/record/listHarvestDataResource" (controller: "record", action: "listHarvestDataResource") - "/ws/record/listRecordsForDataResourceId" (controller: "record", action: "listRecordsForDataResourceId") //dataResourceId + "/ws/record/listHarvestDataResource" (controller: "harvest", action: "listHarvestDataResource") + "/ws/record/listRecordsForDataResourceId" (controller: "harvest", action: "listRecordsForDataResourceId") //dataResourceId "/ws/record/$id"(controller: "record"){ action = [GET:"get", PUT:"update", DELETE:"delete", POST:"update"] } diff --git a/grails-app/services/au/org/ala/ecodata/UserService.groovy b/grails-app/services/au/org/ala/ecodata/UserService.groovy index 011bbee64..4ed7693a5 100644 --- a/grails-app/services/au/org/ala/ecodata/UserService.groovy +++ b/grails-app/services/au/org/ala/ecodata/UserService.groovy @@ -1,10 +1,8 @@ package au.org.ala.ecodata import au.org.ala.userdetails.UserDetailsClient -import au.org.ala.userdetails.UserDetailsFromIdListRequest import au.org.ala.web.AuthService import grails.core.GrailsApplication -import grails.plugin.cache.Cacheable class UserService { @@ -104,6 +102,7 @@ class UserService { * @param username * @param authKey */ + @Deprecated String authorize(userName, authKey) { String userId = "" @@ -131,6 +130,7 @@ class UserService { * @param username * @param password */ + @Deprecated def getUserKey(String username, String password) { webService.doPostWithParams(grailsApplication.config.getProperty('authGetKeyUrl'), [userName: username, password: password], true) } From a11aa78b1c7b7511b425e1f092a64eacd303ac52 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 15 Dec 2023 15:25:25 +1100 Subject: [PATCH 03/52] #3049 - temporarily added individualsOrGroups field --- grails-app/domain/au/org/ala/ecodata/Record.groovy | 2 ++ .../services/au/org/ala/ecodata/ParatooService.groovy | 2 +- .../ala/ecodata/paratoo/ParatooProtocolConfig.groovy | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/grails-app/domain/au/org/ala/ecodata/Record.groovy b/grails-app/domain/au/org/ala/ecodata/Record.groovy index af73a8619..800c1513c 100644 --- a/grails-app/domain/au/org/ala/ecodata/Record.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Record.groovy @@ -42,6 +42,7 @@ class Record { String outputId String json Integer outputItemId + String individualsOrGroups = 'Individuals' String status = ACTIVE static transients = ['recordNumber'] @@ -75,6 +76,7 @@ class Record { name nullable: true vernacularName nullable: true scientificName nullable: true + individualsOrGroups nullable: true, inList: ['Individuals', 'Groups'] } String getRecordNumber(sightingsUrl){ diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 2d42615b5..bffe81bac 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -251,7 +251,7 @@ class ParatooService { decimalLatitude: lat, decimalLongitude: lng, individualCount: config.getIndividualCount(observation), -// numberOfOrganisms: config.getNumberOfOrganisms(observation), + individualsOrGroups: config.getIndividualOrGroup(observation), recordedBy: config.getRecordedBy(observation) ] diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 692599754..101ae8eb0 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -100,6 +100,16 @@ class ParatooProtocolConfig { getProperty(data, observationIndividualCountPath) } + /** + * todo: this is a temporary hack to get around the fact that the survey doesn't have a field for this and it has to + * be interpreted from the protocol + * @param data + * @return + */ + String getIndividualOrGroup (Map data) { + 'Individuals' + } + String getRecordedBy (Map data) { def result = getProperty(data, observationRecordedByPath) if (result instanceof List) { From f9e4487e70294b55079bd9528e6582a9a501a61f Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 20 Mar 2024 08:24:47 +1100 Subject: [PATCH 04/52] Updates to the org interface, added grantID to project payload #823 --- .../org/ala/ecodata/ParatooController.groovy | 73 ++++++++-------- .../au/org/ala/ecodata/ParatooService.groovy | 84 ++++++++++--------- grails-app/views/paratoo/_paratooProject.gson | 1 + .../ecodata/paratoo/ParatooCollection.groovy | 15 +--- .../paratoo/ParatooCollectionId.groovy | 50 ++++++++++- .../ala/ecodata/paratoo/ParatooProject.groovy | 2 + .../paratoo/ParatooProtocolConfig.groovy | 14 ++-- .../ecodata/paratoo/ParatooProvenance.groovy | 24 ++++++ .../paratoo/ParatooSurveyDetails.groovy | 22 +++++ .../paratoo/ParatooSurveyMetadata.groovy | 25 ++++++ .../ala/ecodata/ParatooControllerSpec.groovy | 57 +++++++------ .../ala/ecodata/ParatooJsonViewSpec.groovy | 8 +- .../org/ala/ecodata/ParatooServiceSpec.groovy | 64 ++++++++++---- .../mintCollectionIdBasalAreaPayload.json | 24 ++++++ .../paratoo/mintCollectionIdPayload.json | 21 +++++ 15 files changed, 339 insertions(+), 145 deletions(-) create mode 100644 src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProvenance.groovy create mode 100644 src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyDetails.groovy create mode 100644 src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyMetadata.groovy create mode 100644 src/test/resources/paratoo/mintCollectionIdBasalAreaPayload.json create mode 100644 src/test/resources/paratoo/mintCollectionIdPayload.json diff --git a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy index 0042a736f..3f5b100d2 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy @@ -31,9 +31,10 @@ import javax.ws.rs.GET import javax.ws.rs.POST import javax.ws.rs.PUT import javax.ws.rs.Path + // Requiring these scopes will guarantee we can get a valid userId out of the process. @Slf4j -@au.ala.org.ws.security.RequireApiKey(scopes=["profile", "openid"]) +@au.ala.org.ws.security.RequireApiKey(scopes = ["profile", "openid"]) @OpenAPIDefinition( info = @Info( title = "Ecodata APIs", @@ -55,17 +56,17 @@ import javax.ws.rs.Path type = SecuritySchemeType.OAUTH2, flows = @OAuthFlows( clientCredentials = @OAuthFlow( - authorizationUrl = "https://auth-test.ala.org.au/cas/oidc/authorize", - tokenUrl = "https://auth-test.ala.org.au/cas/oidc/token", - refreshUrl = "https://auth-test.ala.org.au/cas/oidc/refresh", - scopes = [ - @OAuthScope(name="openid"), - @OAuthScope(name="profile"), - @OAuthScope(name="ala", description = "CAS scope"), - @OAuthScope(name="roles", description = "CAS scope"), - @OAuthScope(name="ala/attrs", description = "Cognito scope"), - @OAuthScope(name="ala/roles", description = "Cognito scope") - ] + authorizationUrl = "https://auth-test.ala.org.au/cas/oidc/authorize", + tokenUrl = "https://auth-test.ala.org.au/cas/oidc/token", + refreshUrl = "https://auth-test.ala.org.au/cas/oidc/refresh", + scopes = [ + @OAuthScope(name = "openid"), + @OAuthScope(name = "profile"), + @OAuthScope(name = "ala", description = "CAS scope"), + @OAuthScope(name = "roles", description = "CAS scope"), + @OAuthScope(name = "ala/attrs", description = "Cognito scope"), + @OAuthScope(name = "ala/roles", description = "Cognito scope") + ] ) ), scheme = "bearer" @@ -250,7 +251,7 @@ class ParatooController { error(collectionId.errors) } else { String userId = userService.currentUserDetails.userId - boolean hasProtocol = paratooService.protocolWriteCheck(userId, collectionId.surveyId.projectId, collectionId.surveyId.protocol.id) + boolean hasProtocol = paratooService.protocolWriteCheck(userId, collectionId.projectId, collectionId.protocolId) if (hasProtocol) { Map mintResults = paratooService.mintCollectionId(userId, collectionId) if (mintResults.error) { @@ -288,18 +289,23 @@ class ParatooController { error(collection.errors) } else { String userId = userService.currentUserDetails.userId - Map dataSet = paratooService.findDataSet(userId, collection.orgMintedIdentifier) - - boolean hasProtocol = paratooService.protocolWriteCheck(userId, dataSet.project.id, collection.protocol.id) - if (hasProtocol) { - Map result = paratooService.submitCollection(collection, dataSet.project) - if (!result.error) { - respond([success: true]) + Map dataSet = paratooService.findDataSet(userId, collection.orgMintedUUID) + if (dataSet?.dataSet?.surveyId) { + ParatooCollectionId collectionId = ParatooCollectionId.fromMap(dataSet.dataSet.surveyId) + boolean hasProtocol = paratooService.protocolWriteCheck(userId, dataSet.project.id, collectionId.protocolId) + if (hasProtocol) { + Map result = paratooService.submitCollection(collection, dataSet.project) + if (!result.error) { + respond([success: true]) + } else { + error(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.error) + } } else { - error(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.error) + error(HttpStatus.SC_FORBIDDEN, "Project / protocol combination not available") } + } else { - error(HttpStatus.SC_FORBIDDEN, "Project / protocol combination not available") + error(HttpStatus.SC_NOT_FOUND, "No data set found with orgMintedUUID=${collection.orgMintedUUID}") } } } @@ -394,8 +400,8 @@ class ParatooController { plotSelections.addAll(it.plots) } } - plotSelections = plotSelections.unique {it.siteId} ?: [] - respond plots:plotSelections + plotSelections = plotSelections.unique { it.siteId } ?: [] + respond plots: plotSelections } private def addOrUpdatePlotSelection(ParatooPlotSelection plotSelection) { @@ -440,7 +446,7 @@ class ParatooController { ), tags = "Org Interface" ) - def updateProjectSites(@Parameter(name = "id", description = "Project id", required = true, in = ParameterIn.PATH, schema = @Schema(type = "string"))String id) { + def updateProjectSites(@Parameter(name = "id", description = "Project id", required = true, in = ParameterIn.PATH, schema = @Schema(type = "string")) String id) { String userId = userService.currentUserDetails.userId List projects = paratooService.userProjects(userId) ParatooProject project = projects?.find { it.id == id } @@ -453,20 +459,19 @@ class ParatooController { Map result = paratooService.updateProjectSites(project, data.data, projects) if (result?.error) { - respond([message:result.error], status:HttpStatus.SC_INTERNAL_SERVER_ERROR) - } - else { - respond(buildUpdateProjectSitesResponse(id, data.data), status:HttpStatus.SC_OK) + respond([message: result.error], status: HttpStatus.SC_INTERNAL_SERVER_ERROR) + } else { + respond(buildUpdateProjectSitesResponse(id, data.data), status: HttpStatus.SC_OK) } } private static Map buildUpdateProjectSitesResponse(String id, Map data) { [ - "data": [ - "id": id, - "attributes": data - ], - meta: [:] + "data": [ + "id" : id, + "attributes": data + ], + meta : [:] ] } diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 8177dcce5..0917c3a1c 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -3,19 +3,15 @@ package au.org.ala.ecodata import au.org.ala.ecodata.paratoo.ParatooCollection import au.org.ala.ecodata.paratoo.ParatooCollectionId -import au.org.ala.ecodata.paratoo.ParatooMintedIdentifier import au.org.ala.ecodata.paratoo.ParatooPlotSelectionData import au.org.ala.ecodata.paratoo.ParatooProject import au.org.ala.ecodata.paratoo.ParatooProtocolConfig -import au.org.ala.ecodata.paratoo.ParatooProtocolId import au.org.ala.ecodata.paratoo.ParatooSurveyId import au.org.ala.ws.tokens.TokenService import grails.converters.JSON import grails.core.GrailsApplication import groovy.util.logging.Slf4j -import java.net.http.HttpHeaders - /** * Supports the implementation of the paratoo "org" interface */ @@ -138,13 +134,17 @@ class ParatooService { } Map mintCollectionId(String userId, ParatooCollectionId paratooCollectionId) { - String projectId = paratooCollectionId.surveyId.projectId + String projectId = paratooCollectionId.projectId Project project = Project.findByProjectId(projectId) + // Update the identifier properties as per the org contract and save the data to the data set + String dataSetId = Identifiers.getNew(true, '') + paratooCollectionId.eventTime = new Date() + paratooCollectionId.survey_metadata.orgMintedUUID = dataSetId + paratooCollectionId.userId = project.organisationId // using the organisation as the owner of the data Map dataSet = mapParatooCollectionId(paratooCollectionId, project) - dataSet.progress = Activity.PLANNED - String dataSetName = buildName(paratooCollectionId.surveyId, project) - dataSet.name = dataSetName + + dataSet.surveyId = paratooCollectionId.toMap() // No codec to save this to mongo if (!project.custom) { project.custom = [:] @@ -152,15 +152,9 @@ class ParatooService { if (!project.custom.dataSets) { project.custom.dataSets = [] } - ParatooMintedIdentifier orgMintedIdentifier = new ParatooMintedIdentifier( - surveyId: paratooCollectionId.surveyId, - eventTime: new Date(), - userId: userId, - projectId: projectId, - system: "MERIT", - version: metadataService.getVersion() - ) - dataSet.orgMintedIdentifier = orgMintedIdentifier.encodeAsMintedCollectionId() + + dataSet.orgMintedIdentifier = paratooCollectionId.encodeAsOrgMintedIdentifier() + log.info "Minting identifier for Monitor collection: ${paratooCollectionId}: ${dataSet.orgMintedIdentifier}" project.custom.dataSets << dataSet Map result = projectService.update([custom:project.custom], projectId, false) @@ -171,24 +165,26 @@ class ParatooService { result } - private static String buildName(ParatooSurveyId surveyId, Project project) { - ActivityForm protocolForm = ActivityForm.findByExternalId(surveyId.protocol.id) - String dataSetName = protocolForm?.name + " - " + surveyId.timeAsDisplayDate() + " (" + project.name + ")" + private static String buildName(String protocolId, String displayDate, Project project) { + ActivityForm protocolForm = ActivityForm.findByExternalId(protocolId) + String dataSetName = protocolForm?.name + " - " + displayDate + " (" + project.name + ")" dataSetName } Map submitCollection(ParatooCollection collection, ParatooProject project) { - Map dataSet = project.project.custom?.dataSets?.find{it.orgMintedIdentifier == collection.orgMintedIdentifier} + Map dataSet = project.project.custom?.dataSets?.find{it.surveyId.survey_metadata?.orgMintedUUID == collection.orgMintedUUID} if (!dataSet) { - throw new RuntimeException("Unable to find data set with orgMintedIdentifier: "+collection.orgMintedIdentifier) + throw new RuntimeException("Unable to find data set with orgMintedUUID: "+collection.orgMintedUUID) } dataSet.progress = Activity.STARTED + dataSet.surveyId.coreSubmitTime = new Date() + dataSet.surveyId.survey_metadata.provenance.putAll(collection.coreProvenance) - ParatooSurveyId surveyId = ParatooSurveyId.fromMap(dataSet.surveyId) + ParatooCollectionId surveyId = ParatooCollectionId.fromMap(dataSet.surveyId) - ParatooProtocolConfig config = getProtocolConfig(surveyId.protocol.id) + ParatooProtocolConfig config = getProtocolConfig(surveyId.protocolId) Map surveyData = retrieveSurveyData(surveyId, config) if (surveyData) { @@ -197,10 +193,11 @@ class ParatooService { dataSet.startDate = config.getStartDate(surveyData) dataSet.endDate = config.getEndDate(surveyData) - createActivityFromSurveyData(surveyId, collection.orgMintedIdentifier, surveyData, config, project) + createActivityFromSurveyData(surveyId, collection.orgMintedUUID, surveyData, config, project) } else { - log.warn("Unable to retrieve survey data for: "+collection.orgMintedIdentifier) + log.warn("Unable to retrieve survey data for: "+collection.orgMintedUUID) + log.debug(surveyData) } Map result = projectService.update([custom:project.project.custom], project.id, false) @@ -208,10 +205,10 @@ class ParatooService { result } - private void createActivityFromSurveyData(ParatooSurveyId paratooSurveyId, String mintedCollectionId, Map surveyData, ParatooProtocolConfig config, ParatooProject project) { - ActivityForm form = ActivityForm.findByExternalId(paratooSurveyId.protocol.id) + private void createActivityFromSurveyData(ParatooCollectionId paratooSurveyId, String mintedCollectionId, Map surveyData, ParatooProtocolConfig config, ParatooProject project) { + ActivityForm form = ActivityForm.findByExternalId(paratooSurveyId.protocolId) if (!form) { - log.error("No activity form found for protocol: "+paratooSurveyId.protocol.id) + log.error("No activity form found for protocol: "+paratooSurveyId.protocolId) } else { Map activity = mapActivity(mintedCollectionId, paratooSurveyId, surveyData, form, config, project) @@ -220,13 +217,13 @@ class ParatooService { } - private static Map mapActivity(String mintedCollectionId, ParatooSurveyId surveyId, Map surveyData, ActivityForm activityForm, ParatooProtocolConfig config, ParatooProject project) { + private static Map mapActivity(String mintedCollectionId, ParatooCollectionId surveyId, Map surveyData, ActivityForm activityForm, ParatooProtocolConfig config, ParatooProject project) { Map activity = [:] activity.projectId = project.id activity.startDate = config.getStartDate(surveyData) activity.endDate = config.getEndDate(surveyData) activity.type = activityForm.name - activity.description = activityForm.name + " - " + surveyId.timeAsDisplayDate() + activity.description = activityForm.name + " - " + DateUtil.formatAsDisplayDate(surveyId.eventTime) Map output = [ name: 'Unstructured', @@ -261,18 +258,18 @@ class ParatooService { protocol && project.accessLevel.code >= minimumAccess } - Map findDataSet(String userId, String collectionId) { + Map findDataSet(String userId, String orgMintedUUID) { List projects = findUserProjects(userId) Map dataSet = null ParatooProject project = projects?.find { - dataSet = it.dataSets?.find { it.orgMintedIdentifier == collectionId } + dataSet = it.dataSets?.find { it.orgMintedUUID == orgMintedUUID } dataSet } [dataSet:dataSet, project:project] } - private String createSiteFromSurveyData(Map surveyData, ParatooCollection collection, ParatooSurveyId surveyId, Project project, ParatooProtocolConfig config) { + private String createSiteFromSurveyData(Map surveyData, ParatooCollection collection, ParatooCollectionId surveyId, Project project, ParatooProtocolConfig config) { String siteId = null // Create a site representing the location of the collection Map geoJson = config.getGeoJson(surveyData) @@ -412,6 +409,7 @@ class ParatooService { Map attributes = [ id:project.projectId, name:project.name, + grantID:project.grantId, accessLevel: accessLevel, project:project, projectArea: projectAreaGeoJson, @@ -421,15 +419,21 @@ class ParatooService { } - private static Map mapParatooCollectionId(ParatooCollectionId collectionId, Project project) { + private static Map mapParatooCollectionId(ParatooCollectionId paratooCollectionId, Project project) { Map dataSet = [:] - dataSet.dataSetId = Identifiers.getNew(true, '') - dataSet.surveyId = collectionId.surveyId.toMap() // No codec to save this to mongo - dataSet.protocol = collectionId.surveyId.protocol.id + dataSet.dataSetId = paratooCollectionId.survey_metadata.orgMintedUUID + dataSet.protocol = paratooCollectionId.protocolId dataSet.grantId = project.grantId dataSet.collectionApp = PARATOO_APP_NAME dataSet.dateCreated = DateUtil.format(new Date()) dataSet.lastUpdated = DateUtil.format(new Date()) + + dataSet.progress = Activity.PLANNED + String dataSetName = buildName( + paratooCollectionId.protocolId, + DateUtil.formatAsDisplayDate(paratooCollectionId.eventTime), project) + dataSet.name = dataSetName + dataSet } @@ -437,7 +441,7 @@ class ParatooService { "?populate=deep&sort=updatedAt&pagination[start]=$start&pagination[limit]=$limit" } - Map retrieveSurveyData(ParatooSurveyId surveyId, ParatooProtocolConfig config) { + Map retrieveSurveyData(ParatooCollectionId surveyId, ParatooProtocolConfig config) { String apiEndpoint = config.getApiEndpoint(surveyId) @@ -471,7 +475,7 @@ class ParatooService { } - private static Map findMatchingSurvey(ParatooSurveyId surveyId, List data, ParatooProtocolConfig config) { + private static Map findMatchingSurvey(ParatooCollectionId surveyId, List data, ParatooProtocolConfig config) { data?.find { config.matches(it, surveyId) } } diff --git a/grails-app/views/paratoo/_paratooProject.gson b/grails-app/views/paratoo/_paratooProject.gson index 703c836e5..81adbf773 100644 --- a/grails-app/views/paratoo/_paratooProject.gson +++ b/grails-app/views/paratoo/_paratooProject.gson @@ -7,6 +7,7 @@ model { json { id project.id name project.name + grantID project.grantID protocols tmpl.paratooProtocol('protocol', project.protocols) project_area project.projectArea ? tmpl.projectArea(projectArea:project.projectArea) : null plot_selections tmpl.plot('plot', project.plots) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollection.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollection.groovy index fff87f0cd..7d80d0c9c 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollection.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollection.groovy @@ -6,18 +6,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties @JsonIgnoreProperties(['metaClass', 'errors', 'expandoMetaClass']) class ParatooCollection implements Validateable { - String orgMintedIdentifier - ParatooProtocolId protocol - String projectId - String userId - @BindingFormat("iso8601") - Date eventTime - - static constraints = { - protocol validator: { val, obj -> val.validate() } - projectId nullable: true - userId nullable: true - eventTime nullable: true - } + String orgMintedUUID + Map coreProvenance } diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollectionId.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollectionId.groovy index a64ae1494..89f55fae8 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollectionId.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollectionId.groovy @@ -1,14 +1,60 @@ package au.org.ala.ecodata.paratoo +import au.org.ala.ecodata.DateUtil import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import grails.converters.JSON +import grails.databinding.BindingFormat import grails.validation.Validateable @JsonIgnoreProperties(['metaClass', 'errors', 'expandoMetaClass']) class ParatooCollectionId implements Validateable { - ParatooSurveyId surveyId + ParatooSurveyMetadata survey_metadata + String userId + @BindingFormat("iso8601") + Date eventTime + + Date coreSubmitTime static constraints = { - surveyId validator: { val, obj -> val.validate() } + userId nullable: true + eventTime nullable: true + coreSubmitTime nullable: true + survey_metadata validator: { val, obj -> val.validate() } + } + + String getProjectId() { + survey_metadata?.survey_details?.project_id + } + + String getProtocolId() { + survey_metadata?.survey_details?.protocol_id + } + + Map toMap() { + [ + survey_metadata: survey_metadata.toMap(), + userId: userId, + eventTime: eventTimeAsISOString() + ] + } + + String eventTimeAsISOString() { + eventTime ? DateUtil.formatWithMilliseconds(eventTime) : null + } + + String encodeAsOrgMintedIdentifier() { + Map data = toMap() + String jsonString = (data as JSON).toString() + jsonString.encodeAsBase64() + } + + static ParatooCollectionId fromMap(Map map) { + + Date eventTime = map.eventTime ? DateUtil.parseWithMilliseconds(map.eventTime) : null + new ParatooCollectionId( + eventTime: eventTime, + survey_metadata: new ParatooSurveyMetadata(map.survey_metadata) + ) } } diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProject.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProject.groovy index 585958555..c046e3356 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProject.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProject.groovy @@ -17,6 +17,8 @@ class ParatooProject { String id String name + // Used when creating voucher labels + String grantID AccessLevel accessLevel Project project List protocols diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 657c0eaa3..55ca07e2c 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -27,7 +27,7 @@ class ParatooProtocolConfig { String plotSelectionPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_selection.data.attributes' String plotLayoutDimensionLabelPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_dimensions.data.attributes.label' String plotLayoutTypeLabelPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_type.data.attributes.label' - String getApiEndpoint(ParatooSurveyId surveyId) { + String getApiEndpoint(ParatooCollectionId surveyId) { apiEndpoint ?: defaultEndpoint(surveyId) } @@ -66,8 +66,8 @@ class ParatooProtocolConfig { geometry } - private static String defaultEndpoint(ParatooSurveyId surveyId) { - String apiEndpoint = surveyId.surveyType + private static String defaultEndpoint(ParatooCollectionId surveyId) { + String apiEndpoint = surveyId.survey_metadata?.survey_details?.survey_model if (!apiEndpoint.endsWith('s')) { apiEndpoint += 's' } // strapi makes the endpoint plural sometimes? @@ -96,11 +96,11 @@ class ParatooProtocolConfig { geoJson } - boolean matches(Map surveyData, ParatooSurveyId surveyId) { + boolean matches(Map surveyData, ParatooCollectionId surveyId) { Map tmpSurveyId = getSurveyId(surveyData) - tmpSurveyId.surveyType == surveyId.surveyType && - tmpSurveyId.time == surveyId.timeAsISOString() && - tmpSurveyId.uuid == surveyId.uuid + tmpSurveyId.surveyType == surveyId.survey_metadata.survey_details.survey_model && + tmpSurveyId.time == surveyId.survey_metadata.survey_details.time && + tmpSurveyId.uuid == surveyId.survey_metadata.survey_details.uuid } private Map extractSiteDataFromPlotVisit(Map survey) { diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProvenance.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProvenance.groovy new file mode 100644 index 000000000..e4c125633 --- /dev/null +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProvenance.groovy @@ -0,0 +1,24 @@ +package au.org.ala.ecodata.paratoo + +class ParatooProvenance { + + String system_app + String version_app + String version_core_documentation + + String system_core + String version_core + + String system_org + String version_org + + Map toMap() { + [ + version_app: version_app, + version_core_documentation: version_core_documentation, + system_app: system_app, + system_org: system_org, + version_org: version_org + ] + } +} diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyDetails.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyDetails.groovy new file mode 100644 index 000000000..8cca655a6 --- /dev/null +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyDetails.groovy @@ -0,0 +1,22 @@ +package au.org.ala.ecodata.paratoo + +class ParatooSurveyDetails { + String survey_model + /** ISO time, stored as a String as we don't need to use it as a Date */ + String time + String uuid + String project_id + String protocol_id + Integer protocol_version + + Map toMap() { + [ + survey_model: survey_model, + time: time, + uuid: uuid, + project_id: project_id, + protocol_id: protocol_id, + protocol_version: protocol_version + ] + } +} diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyMetadata.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyMetadata.groovy new file mode 100644 index 000000000..99413d0b3 --- /dev/null +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyMetadata.groovy @@ -0,0 +1,25 @@ +package au.org.ala.ecodata.paratoo + +import grails.validation.Validateable + +class ParatooSurveyMetadata implements Validateable { + + static constraints = { + orgMintedUUID nullable: true + } + + ParatooSurveyDetails survey_details + ParatooProvenance provenance + + /** Added by the ParatooService, other fields are supplied as input */ + String orgMintedUUID + + Map toMap() { + [ + survey_details: survey_details.toMap(), + provenance: provenance.toMap(), + orgMintedUUID: orgMintedUUID + ] + } + +} diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooControllerSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooControllerSpec.groovy index 50ab32e42..158a13389 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooControllerSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooControllerSpec.groovy @@ -3,6 +3,7 @@ package au.org.ala.ecodata import au.org.ala.ecodata.converter.ISODateBindingConverter import au.org.ala.ecodata.paratoo.ParatooProject import grails.testing.web.controllers.ControllerUnitTest +import groovy.json.JsonSlurper import org.apache.http.HttpStatus import spock.lang.Specification @@ -163,6 +164,11 @@ class ParatooControllerSpec extends Specification implements ControllerUnitTest< setup: String userId = 'u1' Map collection = buildCollectionJson() + Map collectionId = buildCollectionIdJson() + collectionId.eventTime = DateUtil.formatWithMilliseconds(new Date()) + collectionId.userId = 'system' + Map dataSet = [surveyId:collectionId] + when: request.method = "POST" @@ -171,7 +177,7 @@ class ParatooControllerSpec extends Specification implements ControllerUnitTest< then: 1 * userService.currentUserDetails >> [userId:userId] - 1 * paratooService.findDataSet(userId, collection.orgMintedIdentifier) >> [project:new ParatooProject(id:'p1'), dataSet:[:]] + 1 * paratooService.findDataSet(userId, collection.orgMintedUUID) >> [project:new ParatooProject(id:'p1'), dataSet:dataSet] 1 * paratooService.protocolWriteCheck(userId, 'p1', "guid-1") >> false and: @@ -184,7 +190,10 @@ class ParatooControllerSpec extends Specification implements ControllerUnitTest< setup: String userId = 'u1' Map collection = buildCollectionJson() - Map searchResults = [project:new ParatooProject(id:'p1'), dataSet:[:]] + Map collectionId = buildCollectionIdJson() + collectionId.eventTime = DateUtil.formatWithMilliseconds(new Date()) + collectionId.userId = 'system' + Map searchResults = [project:new ParatooProject(id:'p1'), dataSet:[surveyId:collectionId]] when: request.method = "POST" @@ -193,9 +202,9 @@ class ParatooControllerSpec extends Specification implements ControllerUnitTest< then: 1 * userService.currentUserDetails >> [userId:userId] - 1 * paratooService.findDataSet(userId, collection.orgMintedIdentifier) >> searchResults + 1 * paratooService.findDataSet(userId, collection.orgMintedUUID) >> searchResults 1 * paratooService.protocolWriteCheck(userId, 'p1', "guid-1") >> true - 1 * paratooService.submitCollection({it.orgMintedIdentifier == "c1"}, searchResults.project) >> [:] + 1 * paratooService.submitCollection({it.orgMintedUUID == "c1"}, searchResults.project) >> [:] and: response.status == HttpStatus.SC_OK @@ -207,7 +216,10 @@ class ParatooControllerSpec extends Specification implements ControllerUnitTest< setup: String userId = 'u1' Map collection = buildCollectionJson() - Map searchResults = [project:new ParatooProject(id:'p1'), dataSet:[:]] + Map collectionId = buildCollectionIdJson() + collectionId.eventTime = DateUtil.formatWithMilliseconds(new Date()) + collectionId.userId = 'system' + Map searchResults = [project:new ParatooProject(id:'p1'), dataSet:[surveyId:collectionId]] when: request.method = "POST" @@ -216,9 +228,9 @@ class ParatooControllerSpec extends Specification implements ControllerUnitTest< then: 1 * userService.currentUserDetails >> [userId:userId] - 1 * paratooService.findDataSet(userId, collection.orgMintedIdentifier) >> searchResults + 1 * paratooService.findDataSet(userId, collection.orgMintedUUID) >> searchResults 1 * paratooService.protocolWriteCheck(userId, 'p1', "guid-1") >> true - 1 * paratooService.submitCollection({it.orgMintedIdentifier == "c1"}, searchResults.project) >> [error:"Error"] + 1 * paratooService.submitCollection({it.orgMintedUUID == "c1"}, searchResults.project) >> [error:"Error"] and: response.status == HttpStatus.SC_INTERNAL_SERVER_ERROR @@ -299,31 +311,22 @@ class ParatooControllerSpec extends Specification implements ControllerUnitTest< } private Map buildCollectionIdJson() { - [ - "surveyId": [ - surveyType: "Bird", - time: "2023-01-01T00:00:00Z", - uuid: "1234", - "projectId":"p1", - "protocol": [ - "id": "guid-1", - "version": 1 - ] - ] - ] + readData("mintCollectionIdPayload") } private Map buildCollectionJson() { [ - "orgMintedIdentifier":"c1", - "projectId":"p1", - "userId": "u1", - "protocol": [ - "id": "guid-1", - "version": 1 - ], - "eventTime":"2023-01-01T00:00:00Z" + "orgMintedUUID":"c1", + "coreProvenance": [ + "system_core": "Monitor-test", + "version_core": "1" + ] ] } + private Map readData(String name) { + URL url = getClass().getResource("/paratoo/${name}.json") + new JsonSlurper().parse(url) + } + } diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooJsonViewSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooJsonViewSpec.groovy index 37eacd044..a772d479b 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooJsonViewSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooJsonViewSpec.groovy @@ -12,7 +12,7 @@ class ParatooJsonViewSpec extends Specification implements JsonViewTest { int[][] projectSpec = [[3, 1, 0], [0, 0, 1], [1, 0, 0]] as int[][] Map expectedResult = [ projects: [[ - id:"p1", name:"Project 1", protocols: [ + id:"p1", name:"Project 1", grantID:"g1", protocols: [ [id:1, identifier: "guid-1", name: "Protocol 1", version: 1, module: "module-1"], [id:2, identifier: "guid-2", name: "Protocol 2", version: 1, module: "module-2"], [id:3, identifier: "guid-3", name: "Protocol 3", version: 1, module: "module-3"]], @@ -22,11 +22,11 @@ class ParatooJsonViewSpec extends Specification implements JsonViewTest { ], role:"project_admin" ],[ - id:"p2", name:"Project 2", protocols:[], plot_selections:[], + id:"p2", name:"Project 2", grantID:"g2", protocols:[], plot_selections:[], project_area:[type:"Polygon", coordinates: DUMMY_POLYGON[0].collect{[lat:it[1], lng:it[0]]}], role:"authenticated" ],[ - id:"p3", name:"Project 3", protocols:[ + id:"p3", name:"Project 3", grantID:"g3", protocols:[ [id:1, identifier: "guid-1", name: "Protocol 1", version: 1, module: 'module-1'] ], project_area:null, plot_selections:[], role:'authenticated' ] @@ -70,7 +70,7 @@ class ParatooJsonViewSpec extends Specification implements JsonViewTest { Site tmp = buildSite(numberOfPlots+2) projectArea = [type:tmp.extent.geometry.type, coordinates:tmp.extent.geometry.coordinates] } - new ParatooProject(id:"p$projectIndex", name:"Project $projectIndex", protocols: protocols, projectArea: projectArea, plots:plots, accessLevel: AccessLevel.admin) + new ParatooProject(id:"p$projectIndex", name:"Project $projectIndex", grantID:"g$projectIndex", protocols: protocols, projectArea: projectArea, plots:plots, accessLevel: AccessLevel.admin) } private ActivityForm buildActivityForm(int i) { diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 0f661525a..40e5166fb 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -23,6 +23,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> {data, pId, updateCollectory -> Map dataSet = data.custom.dataSets[1] // The stubbed project already has a dataSet, so the new one will be index=1 - assert dataSet.surveyId.time == surveyId.timeAsISOString() - assert dataSet.surveyId.uuid == surveyId.uuid - assert dataSet.surveyId.surveyType == surveyId.surveyType - assert dataSet.protocol == surveyId.protocol.id + assert dataSet.surveyId != null + assert dataSet.surveyId.eventTime != null + assert dataSet.surveyId.userId == 'org1' + assert dataSet.surveyId.survey_metadata.orgMintedUUID == dataSet.dataSetId + assert dataSet.protocol == collectionId.protocolId assert dataSet.grantId == "g1" assert dataSet.progress == 'planned' - assert dataSet.name == "aParatooForm 1 - ${DateUtil.formatAsDisplayDate(surveyId.time)} (Project 1)" + assert dataSet.name == "aParatooForm 1 - ${DateUtil.formatAsDisplayDate(collectionId.eventTime)} (Project 1)" [status:'ok'] } @@ -151,16 +158,24 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest", + "version_core": "" + ] + ) + ParatooCollectionId paratooCollectionId = buildCollectionId() + Map dataSet = [dataSetId:'d1', grantId:'g1', surveyId:paratooCollectionId.toMap()] + dataSet.surveyId.survey_metadata.orgMintedUUID = orgMintedId Map expectedDataSet = dataSet+[progress:Activity.STARTED] ParatooProject project = new ParatooProject(id:projectId, project:new Project(projectId:projectId, custom:[dataSets:[dataSet]])) when: Map result = service.submitCollection(collection, project) then: - 1 * webService.getJson({it.indexOf('/s1s') >= 0}, null, _, false) >> [data:[], meta:[pagination:[total:0]]] + 1 * webService.getJson({it.indexOf('/coarse-woody-debris-surveys') >= 0}, null, _, false) >> [data:[], meta:[pagination:[total:0]]] 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) 1 * projectService.update([custom:[dataSets:[expectedDataSet]]], 'p1', false) >> [status:'ok'] @@ -232,9 +247,16 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest", + "version_core": "" + ] + ) + ParatooCollectionId paratooCollectionId = buildCollectionId("mintCollectionIdBasalAreaPayload") + Map dataSet = [dataSetId:'d1', grantId:'g1', surveyId:paratooCollectionId.toMap()] ParatooProject project = new ParatooProject(id:projectId, project:new Project(projectId:projectId, custom:[dataSets:[dataSet]])) Map surveyData = readSurveyData('basalAreaDbh') Map site @@ -243,7 +265,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest= 0}, null, _, false) >> [data:[surveyData], meta:[pagination:[total:0]]] + 1 * webService.getJson({it.indexOf('/basal-area-dbh-measure-surveys') >= 0}, null, _, false) >> [data:[surveyData], meta:[pagination:[total:0]]] 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) 1 * projectService.update(_, projectId, false) >> [status:'ok'] 1 * siteService.create(_) >> {site = it[0]; [siteId:'s1']} @@ -264,7 +286,13 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Wed, 20 Mar 2024 09:04:09 +1100 Subject: [PATCH 05/52] Removed logging, updated data set search #823 --- .../controllers/au/org/ala/ecodata/ParatooController.groovy | 1 - grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy index 3f5b100d2..0d44d3f7e 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy @@ -283,7 +283,6 @@ class ParatooController { if (log.isDebugEnabled()) { log.debug("ParatooController::submitCollection") - log.debug(request.JSON.toString()) } if (collection.hasErrors()) { error(collection.errors) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 0917c3a1c..191fd7cdd 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -263,7 +263,7 @@ class ParatooService { Map dataSet = null ParatooProject project = projects?.find { - dataSet = it.dataSets?.find { it.orgMintedUUID == orgMintedUUID } + dataSet = it.dataSets?.find { it.dataSetId == orgMintedUUID } dataSet } [dataSet:dataSet, project:project] From 3995dae41ecdd1d6e248930b83dc94e42b3643f9 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 20 Mar 2024 09:24:23 +1100 Subject: [PATCH 06/52] updated data set search #823 --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 191fd7cdd..9023d9ddb 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -173,7 +173,7 @@ class ParatooService { Map submitCollection(ParatooCollection collection, ParatooProject project) { - Map dataSet = project.project.custom?.dataSets?.find{it.surveyId.survey_metadata?.orgMintedUUID == collection.orgMintedUUID} + Map dataSet = project.project.custom?.dataSets?.find{it.dataSetId == collection.orgMintedUUID} if (!dataSet) { throw new RuntimeException("Unable to find data set with orgMintedUUID: "+collection.orgMintedUUID) From 03b0d10e43e644ee901c7dde58d10ec23f3328bb Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 20 Mar 2024 09:42:11 +1100 Subject: [PATCH 07/52] Adding debug logging #823 --- .../ala/ecodata/paratoo/ParatooProtocolConfig.groovy | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 55ca07e2c..b8734c4b2 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -98,9 +98,14 @@ class ParatooProtocolConfig { boolean matches(Map surveyData, ParatooCollectionId surveyId) { Map tmpSurveyId = getSurveyId(surveyData) - tmpSurveyId.surveyType == surveyId.survey_metadata.survey_details.survey_model && - tmpSurveyId.time == surveyId.survey_metadata.survey_details.time && - tmpSurveyId.uuid == surveyId.survey_metadata.survey_details.uuid + if (!tmpSurveyId) { + log.error("Cannot find surveyId:") + log.debug(surveyData) + return false + } + tmpSurveyId?.surveyType == surveyId.survey_metadata?.survey_details.survey_model && + tmpSurveyId?.time == surveyId.survey_metadata?.survey_details.time && + tmpSurveyId?.uuid == surveyId.survey_metadata?.survey_details.uuid } private Map extractSiteDataFromPlotVisit(Map survey) { From 31759977cba0ec9030cca296ca015c53ea31bfe4 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 20 Mar 2024 10:03:10 +1100 Subject: [PATCH 08/52] Adding debug logging #823 --- .../au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index b8734c4b2..97eb9354f 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -100,7 +100,7 @@ class ParatooProtocolConfig { Map tmpSurveyId = getSurveyId(surveyData) if (!tmpSurveyId) { log.error("Cannot find surveyId:") - log.debug(surveyData) + log.debug(surveyData.toString()) return false } tmpSurveyId?.surveyType == surveyId.survey_metadata?.survey_details.survey_model && From 31eca2e9b5a507b52ff512db949ce55878535bec Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 20 Mar 2024 10:46:54 +1100 Subject: [PATCH 09/52] API callback updates #823 --- .../au/org/ala/ecodata/graphql-test.http | 43 +++++++++++++++++++ .../paratoo/ParatooProtocolConfig.groovy | 6 +-- src/test/resources/paratoo/basalAreaDbh.json | 26 +++++++---- 3 files changed, 63 insertions(+), 12 deletions(-) create mode 100644 grails-app/controllers/au/org/ala/ecodata/graphql-test.http diff --git a/grails-app/controllers/au/org/ala/ecodata/graphql-test.http b/grails-app/controllers/au/org/ala/ecodata/graphql-test.http new file mode 100644 index 000000000..6ba0e69e8 --- /dev/null +++ b/grails-app/controllers/au/org/ala/ecodata/graphql-test.http @@ -0,0 +1,43 @@ +### +POST https://ecodata-test.ala.org.au/ws/graphql +Authorization: Bearer eyJraWQiOiI2UEpOaFwvdU5EYlBIWlk4Y2xmTHJvMnBKUnJhTFRXTnpaU0tOcVdka3Y0az0iLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJiOTMyNTY0Zi1hNjQxLTQ3MTYtYTAwMS0wMTRkODA2NzU0MmQiLCJjb2duaXRvOmdyb3VwcyI6WyJ1c2VyIiwiZWNvZGF0YV9hcGkiXSwiaXNzIjoiaHR0cHM6XC9cL2NvZ25pdG8taWRwLmFwLXNvdXRoZWFzdC0yLmFtYXpvbmF3cy5jb21cL2FwLXNvdXRoZWFzdC0yX09PWFU5R1czOSIsInZlcnNpb24iOjIsImNsaWVudF9pZCI6IjRyaGtvcDl0bDMwbHJ0OThjcWpwY3UzZTF0Iiwib3JpZ2luX2p0aSI6IjMxYmYzYTg0LThiMDktNDVhNi05OTE3LTY5ODE5N2Y3N2EzYiIsInRva2VuX3VzZSI6ImFjY2VzcyIsInNjb3BlIjoiYWxhXC9hdHRycyBhbGFcL3JvbGVzIG9wZW5pZCBwcm9maWxlIGVtYWlsIiwiYXV0aF90aW1lIjoxNzA3MDgwMTM5LCJleHAiOjE3MDcwODczNDgsImlhdCI6MTcwNzA4Mzc0OCwianRpIjoiNjNhZTJjNTItYzUxZC00ZmJjLWFkMDAtODQ4NGIyOTQ1ZDI0IiwidXNlcm5hbWUiOiI2NDUwIn0.crLqFuvY9JLmxM88GI3ukYTZGlcYHmXTlJ8AFJHSkB2bkPoSJ8F7OnGnJuZlpzlBHLcyMM91JpVoNNdLlVFYptnafX279Q2oUXzAe_iiwPI2-JPPVEp4bDSYd7x9Yl488EYFBNGJ5cbEaSGyMh9WHggWAFG8BLXXUdWM8GfS2-PdqUwcFkPQAu7DDgSh0k6BDviZaOuA0-2AtLdx4c6yWA9dzfRlzMinbTnR3at0QAJ4Dv7xJhEsgvXaR2z4Pe9JrcIuoTZlJx360UQtuDgNLWA-fzDHfmsSOIuBBYB8E0gFkdKt7f7TE-M6KvIUKhp6z30xI4_OwGC52U0D11LISg +Content-Type: application/json + +{ + "query": "query {\n project(projectId:\"30b6e207-51fb-4286-881c-9ca31959010c\") {\n projectId\n name\n description\n \n program {\n name\n programId\n acronym\n }\n organisation {\n \torganisationId\n name\n abn\n }\n managementUnit {\n \tmanagementUnitId\n name\n shortName\n }\n }\n}" +} + +### +POST http://localhost:8080/ws/graphql +Authorization: Bearer eyJraWQiOiI2UEpOaFwvdU5EYlBIWlk4Y2xmTHJvMnBKUnJhTFRXTnpaU0tOcVdka3Y0az0iLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJiOTMyNTY0Zi1hNjQxLTQ3MTYtYTAwMS0wMTRkODA2NzU0MmQiLCJjb2duaXRvOmdyb3VwcyI6WyJ1c2VyIiwiZWNvZGF0YV9hcGkiXSwiaXNzIjoiaHR0cHM6XC9cL2NvZ25pdG8taWRwLmFwLXNvdXRoZWFzdC0yLmFtYXpvbmF3cy5jb21cL2FwLXNvdXRoZWFzdC0yX09PWFU5R1czOSIsInZlcnNpb24iOjIsImNsaWVudF9pZCI6IjRyaGtvcDl0bDMwbHJ0OThjcWpwY3UzZTF0Iiwib3JpZ2luX2p0aSI6IjMxYmYzYTg0LThiMDktNDVhNi05OTE3LTY5ODE5N2Y3N2EzYiIsInRva2VuX3VzZSI6ImFjY2VzcyIsInNjb3BlIjoiYWxhXC9hdHRycyBhbGFcL3JvbGVzIG9wZW5pZCBwcm9maWxlIGVtYWlsIiwiYXV0aF90aW1lIjoxNzA3MDgwMTM5LCJleHAiOjE3MDcwODczNDgsImlhdCI6MTcwNzA4Mzc0OCwianRpIjoiNjNhZTJjNTItYzUxZC00ZmJjLWFkMDAtODQ4NGIyOTQ1ZDI0IiwidXNlcm5hbWUiOiI2NDUwIn0.crLqFuvY9JLmxM88GI3ukYTZGlcYHmXTlJ8AFJHSkB2bkPoSJ8F7OnGnJuZlpzlBHLcyMM91JpVoNNdLlVFYptnafX279Q2oUXzAe_iiwPI2-JPPVEp4bDSYd7x9Yl488EYFBNGJ5cbEaSGyMh9WHggWAFG8BLXXUdWM8GfS2-PdqUwcFkPQAu7DDgSh0k6BDviZaOuA0-2AtLdx4c6yWA9dzfRlzMinbTnR3at0QAJ4Dv7xJhEsgvXaR2z4Pe9JrcIuoTZlJx360UQtuDgNLWA-fzDHfmsSOIuBBYB8E0gFkdKt7f7TE-M6KvIUKhp6z30xI4_OwGC52U0D11LISg +Content-Type: application/json + +{ + "query": "query {\n project(projectId:\"5e416e05-cb4d-4f7e-84ec-e29291ce492a\") {\n projectId\n name\n description\n \n program {\n name\n programId\n acronym\n }\n organisation {\n \torganisationId\n name\n abn\n }\n managementUnit {\n \tmanagementUnitId\n name\n shortName\n }\n }\n}" +} + +### +GET http://localhost:8080/ws/paratoo/user-projects +Authorization: Bearer eyJraWQiOiI2UEpOaFwvdU5EYlBIWlk4Y2xmTHJvMnBKUnJhTFRXTnpaU0tOcVdka3Y0az0iLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI2ODMyZDkyNy03ZTE0LTRkMTYtODk0ZS00N2I4NGM3YjdkNWYiLCJjb2duaXRvOmdyb3VwcyI6WyJ1c2VyIiwiZWNvZGF0YV9hcGkiLCJhZG1pbiJdLCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAuYXAtc291dGhlYXN0LTIuYW1hem9uYXdzLmNvbVwvYXAtc291dGhlYXN0LTJfT09YVTlHVzM5IiwidmVyc2lvbiI6MiwiY2xpZW50X2lkIjoiNHJoa29wOXRsMzBscnQ5OGNxanBjdTNlMXQiLCJvcmlnaW5fanRpIjoiOGMxNmM3NmItNTA3Ny00OWRmLWI2YTAtY2Y1Y2NjMmExMTRmIiwiZXZlbnRfaWQiOiIzYzk1NmYzYS0yMGFhLTQxMjAtYjM1Zi0yMzhhMTkzMWE1NDUiLCJ0b2tlbl91c2UiOiJhY2Nlc3MiLCJzY29wZSI6ImFsYVwvYXR0cnMgYWxhXC9yb2xlcyBvcGVuaWQgcHJvZmlsZSBlbWFpbCIsImF1dGhfdGltZSI6MTcwNjU2NTg5NSwiZXhwIjoxNzA2NTc2NzY3LCJpYXQiOjE3MDY1NzMxNjgsImp0aSI6IjE5NWQ0ODkwLWM1YWUtNDJmNy05NjA2LTVmNjVlZGI5NmQ4YyIsInVzZXJuYW1lIjoiNTY1MDEifQ.yoePlhuO22-CqSiMh9rM41El_bI-gohdRk7KQWphxo9GUAMba4Kf8efT8bnxzLzuuQ5bM4egie0iRdmIcHPNzuZfoLidvn9Q_ItjSYkBeIaxozFKyy3uGl5n6_BOuzQdXN2J831Qj_me_38mGKbRRp1W-cjOJu7xH-lYudXiyzSjdv02MpRdAQEfKG47BW_r2FFXBdDCKrhZ_pSQjjFQng4NLFefeMPxmErR6CYDEwBG9_u5m_WQea9P_gy6mjsuQ-bXZbZq4BSNswJgd1pmayNz-65gX-hewA2eE6VVOWwzgmwakJNVEF1FX9tMcYWd0wRKJBvBOzz2Dii-5Xd3gg +Content-Type: */* + +### +GET https://ecodata-staging.ala.org.au/ws/paratoo/user-projects +Authorization: Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsIm9yZy5hcGVyZW8uY2FzLnNlcnZpY2VzLlJlZ2lzdGVyZWRTZXJ2aWNlIjoiMTY3MTE1MjI5MTY4OCIsImtpZCI6InNpZy0xNjU1OTU2NDkzIn0.eyJzdWIiOiIxMDkxNDYwNjc2NzgyNDkyNzg1ODQiLCJyb2xlIjpbIlJPTEVfQURNSU4iLCJST0xFX0FQSV9FRElUT1IiLCJST0xFX0NPTExFQ1RJT05fQURNSU4iLCJST0xFX0VDT0RBVEFfQVBJIiwiUk9MRV9FRElUT1IiLCJST0xFX1VTRVIiLCJST0xFX1ZQX0FETUlOIiwiUk9MRV9WUF9URVNUX0FETUlOIl0sIm9hdXRoQ2xpZW50SWQiOiIyWHZsZ0pYWHVzc3R4aFZkTlJaY0J0aUpIeFdiRlNUZVJDNWYiLCJjbGllbnROYW1lIjoiR29vZ2xlIiwiaXNzIjoiaHR0cHM6XC9cL2F1dGguYWxhLm9yZy5hdVwvY2FzXC9vaWRjIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2hyaXMuZ29kd2luLmFsYUBnbWFpbC5jb20iLCJjbGllbnRfaWQiOiIyWHZsZ0pYWHVzc3R4aFZkTlJaY0J0aUpIeFdiRlNUZVJDNWYiLCJ1cGRhdGVkX2F0IjoiMjAyMi0wNy0xNCAxMDoyMzoyMyIsImdyYW50X3R5cGUiOiJBVVRIT1JJWkFUSU9OX0NPREUiLCJzY29wZSI6WyJvcGVuaWQiLCJwcm9maWxlIl0sInNlcnZlcklwQWRkcmVzcyI6IjEyNy4wLjAuMSIsImxvbmdUZXJtQXV0aGVudGljYXRpb25SZXF1ZXN0VG9rZW5Vc2VkIjpmYWxzZSwic3RhdGUiOiI1NDg4YTdiZTkxOTk0ZDQ5OWJiMzEyMGUxNjhkY2MzZiIsImV4cCI6MTcxMDUzNTg2MywiaWF0IjoxNzEwNDQ5NDYzLCJqdGkiOiJBVC0zMjM2Ny1NOTFnRkFYdXRHaFlESkp6LTRlMm1nQVNKQXRIN1N3TCIsImVtYWlsIjoiY2hyaXMuZ29kd2luLmFsYUBnbWFpbC5jb20iLCJjbGllbnRJcEFkZHJlc3MiOiIxNDAuNzkuNzkuOTYiLCJpc0Zyb21OZXdMb2dpbiI6dHJ1ZSwiZW1haWxfdmVyaWZpZWQiOiIxIiwiYXV0aGVudGljYXRpb25EYXRlIjoiMjAyNC0wMy0xNFQyMDo1MTowMi45MzE0OTFaIiwic3VjY2Vzc2Z1bEF1dGhlbnRpY2F0aW9uSGFuZGxlcnMiOiJEZWxlZ2F0ZWRDbGllbnRBdXRoZW50aWNhdGlvbkhhbmRsZXIiLCJ1c2VyQWdlbnQiOiJNb3ppbGxhXC81LjAgKE1hY2ludG9zaDsgSW50ZWwgTWFjIE9TIFggMTBfMTVfNykgQXBwbGVXZWJLaXRcLzUzNy4zNiAoS0hUTUwsIGxpa2UgR2Vja28pIENocm9tZVwvMTIyLjAuMC4wIFNhZmFyaVwvNTM3LjM2IiwiZ2l2ZW5fbmFtZSI6IkNocmlzIiwibm9uY2UiOiIiLCJjcmVkZW50aWFsVHlwZSI6IkNsaWVudENyZWRlbnRpYWwiLCJzYW1sQXV0aGVudGljYXRpb25TdGF0ZW1lbnRBdXRoTWV0aG9kIjoidXJuOm9hc2lzOm5hbWVzOnRjOlNBTUw6MS4wOmFtOnVuc3BlY2lmaWVkIiwiYXVkIjoiaHR0cHM6XC9cL3Rva2Vucy5hbGEub3JnLmF1XC9sb2dpbiIsImF1dGhlbnRpY2F0aW9uTWV0aG9kIjoiRGVsZWdhdGVkQ2xpZW50QXV0aGVudGljYXRpb25IYW5kbGVyIiwiZ2VvTG9jYXRpb24iOiJ1bmtub3duIiwibmFtZSI6IkNocmlzIEdvZHdpbiIsInNjb3BlcyI6WyJvcGVuaWQiLCJwcm9maWxlIl0sImZhbWlseV9uYW1lIjoiR29kd2luIn0.jazpxAZPTWJ3vzEPnrHAZkmxllLK64Pk-mRmddv5k_IhSTHiEvvrA6IP9GTd7BDWSnBtEfmUFKqv5wk3Igz3m7oQR97aAvqA8t1ORqGhrdNUJu4dU5V5l7EG8wLuwcT7k668sAa_kpxH1YFpDzXp_tG4xIKtk3tpAZOSnjYEzCPy3MVCiw5f4Tm-7CGS125T_JOIyST6_nh4HHygYHoXDdjy-MvRJqrvWFR7tKMN99zscvCBuOBjHy5fyzj16dSPY67q9-J0DuGiHR0-okwtev4rajr8LNXGsX40N-CwtAlzXYJK6ThJcYtRM7KNlImtW6Jyyx7TqRKC-IRWCE8L7g +Content-Type: application/json + +### +GET http://localhost:8080/ws/paratoo/user-projects +Authorization: Bearer eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsIm9yZy5hcGVyZW8uY2FzLnNlcnZpY2VzLlJlZ2lzdGVyZWRTZXJ2aWNlIjoiMTY5NTg3MjAzMTM4MyIsImtpZCI6InNpZy0xNjU1OTU2NDkzIn0.eyJzdWIiOiIxMDkxNDYwNjc2NzgyNDkyNzg1ODQiLCJyb2xlIjpbIlJPTEVfQURNSU4iLCJST0xFX0FQSV9FRElUT1IiLCJST0xFX0NPTExFQ1RJT05fQURNSU4iLCJST0xFX0VDT0RBVEFfQVBJIiwiUk9MRV9FRElUT1IiLCJST0xFX1VTRVIiLCJST0xFX1ZQX0FETUlOIiwiUk9MRV9WUF9URVNUX0FETUlOIl0sIm9hdXRoQ2xpZW50SWQiOiJvUEJFZEYwNW5WWGNWamJjZ3JlY3B3cVd4cFdzeEtUQnRmRVYiLCJjbGllbnROYW1lIjoiR29vZ2xlIiwiaXNzIjoiaHR0cHM6XC9cL2F1dGguYWxhLm9yZy5hdVwvY2FzXC9vaWRjIiwicHJlZmVycmVkX3VzZXJuYW1lIjoiY2hyaXMuZ29kd2luLmFsYUBnbWFpbC5jb20iLCJ1c2VyaWQiOiIxNDkzIiwiY2xpZW50X2lkIjoib1BCRWRGMDVuVlhjVmpiY2dyZWNwd3FXeHBXc3hLVEJ0ZkVWIiwidXBkYXRlZF9hdCI6IjIwMjItMDctMTQgMTA6MjM6MjMiLCJncmFudF90eXBlIjoiQVVUSE9SSVpBVElPTl9DT0RFIiwic2NvcGUiOlsiYWxhIiwib3BlbmlkIiwicHJvZmlsZSIsInJvbGVzIl0sInNlcnZlcklwQWRkcmVzcyI6IjEyNy4wLjAuMSIsImxvbmdUZXJtQXV0aGVudGljYXRpb25SZXF1ZXN0VG9rZW5Vc2VkIjpmYWxzZSwic3RhdGUiOiI3Njc0OWRjMmI2Y2U0N2Y2YmJiMGI1ZDdhOTk5Y2QwMSIsImV4cCI6MTcwODEzMDE1MCwiaWF0IjoxNzA4MDQzNzUwLCJqdGkiOiJBVC00MDU4MC1lOHczT1YwZUgtbkF1VlRwbk1vc2NkUHNBOXlsdXFyQiIsImVtYWlsIjoiY2hyaXMuZ29kd2luLmFsYUBnbWFpbC5jb20iLCJjbGllbnRJcEFkZHJlc3MiOiIxNDAuMjUzLjIyNC4yNDciLCJpc0Zyb21OZXdMb2dpbiI6ZmFsc2UsImVtYWlsX3ZlcmlmaWVkIjoiMSIsImF1dGhlbnRpY2F0aW9uRGF0ZSI6IjIwMjQtMDItMTVUMjI6MTc6MzEuNjkwMTU3WiIsInN1Y2Nlc3NmdWxBdXRoZW50aWNhdGlvbkhhbmRsZXJzIjoiRGVsZWdhdGVkQ2xpZW50QXV0aGVudGljYXRpb25IYW5kbGVyIiwidXNlckFnZW50IjoiTW96aWxsYVwvNS4wIChNYWNpbnRvc2g7IEludGVsIE1hYyBPUyBYIDEwXzE1XzcpIEFwcGxlV2ViS2l0XC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWVcLzEyMS4wLjAuMCBTYWZhcmlcLzUzNy4zNiIsImdpdmVuX25hbWUiOiJDaHJpcyIsIm5vbmNlIjoiIiwiY3JlZGVudGlhbFR5cGUiOiJDbGllbnRDcmVkZW50aWFsIiwic2FtbEF1dGhlbnRpY2F0aW9uU3RhdGVtZW50QXV0aE1ldGhvZCI6InVybjpvYXNpczpuYW1lczp0YzpTQU1MOjEuMDphbTp1bnNwZWNpZmllZCIsImF1ZCI6Imh0dHBzOlwvXC90b2tlbnMuYWxhLm9yZy5hdVwvbG9naW4iLCJhdXRoZW50aWNhdGlvbk1ldGhvZCI6IkRlbGVnYXRlZENsaWVudEF1dGhlbnRpY2F0aW9uSGFuZGxlciIsImdlb0xvY2F0aW9uIjoidW5rbm93biIsIm5hbWUiOiJDaHJpcyBHb2R3aW4iLCJzY29wZXMiOlsib3BlbmlkIiwicHJvZmlsZSIsImFsYSIsInJvbGVzIl0sImZhbWlseV9uYW1lIjoiR29kd2luIn0.M74V6ibistW_VGeTQhUdDpn_h7rk8G5QoCm56EPOgpmnyxAeZK3Kz_omMELPI5p2cdvaAttJ6ZMvd268jtHju9VfDmm7IP668ikpDQsWoaFVaXiFi1X0FPvucilDm3Aa1Sd3rqeEH9fNDO_OEJe4NvE4JaKUAXGp4ZjboV5xZZniuczlZyPXpQ8p-YqIIZRtEfYnbBrSJt5xA8GzVXIhXCbuUfb6AjH1qkG5qebHGh3LYZ2BEx_X9nfew8WD3wP6k8OebLJecvxm-u3ELKV48c5LAf4RFHjNSEcR9-NM3Yg7r5y4PP-wGuikh1cFjWMEPgwtB6T3Bq2xmPnLtkuZAQ +Content-Type: application/json + +### +GET https://dev.core-api.monitor.tern.org.au/api/cover-point-intercept-species-intercepts?populate=deep +Authorization: Bearer eyJraWQiOiI2UEpOaFwvdU5EYlBIWlk4Y2xmTHJvMnBKUnJhTFRXTnpaU0tOcVdka3Y0az0iLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI2ODMyZDkyNy03ZTE0LTRkMTYtODk0ZS00N2I4NGM3YjdkNWYiLCJjb2duaXRvOmdyb3VwcyI6WyJ1c2VyIiwiZWNvZGF0YV9hcGkiLCJhZG1pbiJdLCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAuYXAtc291dGhlYXN0LTIuYW1hem9uYXdzLmNvbVwvYXAtc291dGhlYXN0LTJfT09YVTlHVzM5IiwidmVyc2lvbiI6MiwiY2xpZW50X2lkIjoiN2E5MGMzNjB2MXEwazY4cjFwYXM5MmRka3AiLCJvcmlnaW5fanRpIjoiMWY0MDIzMmEtNzAyMy00NDMyLWE1NTAtMTNkMjA5MTEzYTYzIiwidG9rZW5fdXNlIjoiYWNjZXNzIiwic2NvcGUiOiJhbGFcL2F0dHJzIGFsYVwvcm9sZXMgb3BlbmlkIHByb2ZpbGUgZW1haWwiLCJhdXRoX3RpbWUiOjE3MDgzODI4MzAsImV4cCI6MTcwODM4NjQzMCwiaWF0IjoxNzA4MzgyODMxLCJqdGkiOiI3YTVhN2I4MS1kOWQ3LTRlMjUtOWVmMy03M2M0OTk5Zjg2YzUiLCJ1c2VybmFtZSI6IjU2NTAxIn0.hHOysWUwZer0JzpDWJZtWWAXebEbzAkLz3Q74wg0LuGCyWMFonredGZ6fA2NxwCwX3FB-4ppTA6QdyUiNAnz846dK7pIN6Tfv1shG6whqYvBDvpmHblgRl6MFnIHiFJVoCUoRxWM6A9nArwRAvPqv9fOPQzxeIWDGn0f2sQkfPdCCvYsTFYqC2qhF_GqjaXxb2Wsf2dDq0RhldNRmBGMuuu-BiloPGzVl9tZAsH7kyiNsZD8UdCzUkIBQhYmgWlhYBRUzbcAeJ7vrvQMdHQ-vO7xGT-s6xxudDwt24zRsjXt83YDTHdMxY8roc_3ZEn0cGo3gBaoMq4HBpXUo_8w-Q +Content-Type: application/json + +### +GET https://dev.core-api.monitor.tern.org.au/api/protocols/reverse-lookup +Authorization: Bearer eyJraWQiOiI2UEpOaFwvdU5EYlBIWlk4Y2xmTHJvMnBKUnJhTFRXTnpaU0tOcVdka3Y0az0iLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiI2ODMyZDkyNy03ZTE0LTRkMTYtODk0ZS00N2I4NGM3YjdkNWYiLCJjb2duaXRvOmdyb3VwcyI6WyJ1c2VyIiwiZWNvZGF0YV9hcGkiLCJhZG1pbiJdLCJpc3MiOiJodHRwczpcL1wvY29nbml0by1pZHAuYXAtc291dGhlYXN0LTIuYW1hem9uYXdzLmNvbVwvYXAtc291dGhlYXN0LTJfT09YVTlHVzM5IiwidmVyc2lvbiI6MiwiY2xpZW50X2lkIjoiN2E5MGMzNjB2MXEwazY4cjFwYXM5MmRka3AiLCJvcmlnaW5fanRpIjoiMWY0ZDljOTYtNDBkOC00YzNmLTkxMDctNDQ1YjdiOTEzOGIzIiwiZXZlbnRfaWQiOiJhZWVmYzEwNi1lYjc1LTQ1OTEtYjE3OC02MjM2NWU3NWU4NDAiLCJ0b2tlbl91c2UiOiJhY2Nlc3MiLCJzY29wZSI6ImFsYVwvYXR0cnMgYWxhXC9yb2xlcyBvcGVuaWQgcHJvZmlsZSBlbWFpbCIsImF1dGhfdGltZSI6MTcwOTA2Njg1MSwiZXhwIjoxNzA5MDcwNDUxLCJpYXQiOjE3MDkwNjY4NTIsImp0aSI6IjFlZGYyNzQ2LTJlY2ItNDRhOC05NjNlLTAwZTM1ODNkYjllMyIsInVzZXJuYW1lIjoiNTY1MDEifQ.x5h1Hw1fbomORk_jVxL8-fdjeO4uqbG6UoCKfeGQo_CJV02ZS5OhYmEolZkVSL5MkdytHBqY6Vojh5MrP9738He8exlbBRh8OnaA6QCb4Iasio0xWJcbysxt5Y98TGOC1cfcUAbjaZpwMH7r6ifQVoiK0RcztZJWlcSBp5IE2rZeSZ2tawg-Xl_JqR0d69wukn6gcNgrg5gPGPRqGbfrFwqciIKaj2KbZ8axZZi1kakPRGPH8FB2nxvPp4XGsjs-hV__8KxeNMRD2G0-paI0LyqWbvNOJnUfKYYNY2G8AgH-CJWLEA0Llo3kHORqnm3PyzcXvr82MhAn1ujdGlxDYQ + +userId=1493 \ No newline at end of file diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 97eb9354f..71c32d4ef 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -21,7 +21,7 @@ class ParatooProtocolConfig { String geometryPath String startDatePath = 'attributes.start_date_time' String endDatePath = 'attributes.end_date_time' - String surveyIdPath = 'attributes.surveyId' + String surveyIdPath = 'attributes.survey_metadata' String plotLayoutIdPath = 'attributes.plot_visit.data.attributes.plot_layout.data.id' String plotLayoutPointsPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_points' String plotSelectionPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_selection.data.attributes' @@ -103,9 +103,7 @@ class ParatooProtocolConfig { log.debug(surveyData.toString()) return false } - tmpSurveyId?.surveyType == surveyId.survey_metadata?.survey_details.survey_model && - tmpSurveyId?.time == surveyId.survey_metadata?.survey_details.time && - tmpSurveyId?.uuid == surveyId.survey_metadata?.survey_details.uuid + tmpSurveyId?.orgMintedUUID == surveyId.survey_metadata?.orgMintedUUID } private Map extractSiteDataFromPlotVisit(Map survey) { diff --git a/src/test/resources/paratoo/basalAreaDbh.json b/src/test/resources/paratoo/basalAreaDbh.json index 39c2c3fa4..2d8a01a07 100644 --- a/src/test/resources/paratoo/basalAreaDbh.json +++ b/src/test/resources/paratoo/basalAreaDbh.json @@ -1,15 +1,25 @@ { "id": 2, "attributes": { - "surveyId": { - "time": "2023-09-22T01:03:15.556Z", - "uuid": "43389075", - "protocol": { - "id": "1", - "version": 1 + "survey_metadata": { + "orgMintedUUID": "d1", + "survey_details": { + "survey_model": "basal-area-dbh-measure-survey", + "time": "2023-09-22T01:03:15.556Z", + "uuid": "43389075", + "project_id": "p1", + "protocol_id": "guid-2", + "protocol_version": "1" }, - "projectId": "p1", - "surveyType": "basal-area-dbh-measure-survey" + "provenance":{ + "version_app": "0.0.1-xxxxx", + "version_core": "0.1.0-1fb53f81", + "version_core_documentation": "0.0.1-xxxxx", + "version_org": "4.4-SNAPSHOT", + "system_app": "monitor", + "system_core": "Monitor-dummy-data-production", + "system_org": "MERIT" + } }, "start_date": "2023-09-22T00:59:47.807Z", "createdAt": "2023-09-22T01:05:19.981Z", From 0bef0b619a67f570e29292abd491b3bfac710755 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 20 Mar 2024 11:25:09 +1100 Subject: [PATCH 10/52] More logging #823 --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 9023d9ddb..4b8d58a2c 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -460,14 +460,19 @@ class ParatooService { String url = paratooBaseUrl+'/'+apiEndpoint String query = buildSurveyQueryString(start, limit) + log.debug("Retrieving survey data from: "+url+query) Map response = webService.getJson(url+query, null, authHeader, false) + log.debug(response) Map survey = findMatchingSurvey(surveyId, response.data, config) + int total = response.meta?.pagination?.total ?: 0 while (!survey && start+limit < total) { start += limit query = buildSurveyQueryString(start, limit) + log.debug("Retrieving survey data from: "+url+query) response = webService.getJson(url+query, null, authHeader, false) + log.debug(response) survey = findMatchingSurvey(surveyId, response.data, config) } From 62104a923c5c3e12b09d2d621c5616ff41f24aab Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 20 Mar 2024 11:47:39 +1100 Subject: [PATCH 11/52] More logging #823 --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 4b8d58a2c..a60b070b7 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -462,7 +462,7 @@ class ParatooService { String query = buildSurveyQueryString(start, limit) log.debug("Retrieving survey data from: "+url+query) Map response = webService.getJson(url+query, null, authHeader, false) - log.debug(response) + log.debug((response as JSON).toString()) Map survey = findMatchingSurvey(surveyId, response.data, config) int total = response.meta?.pagination?.total ?: 0 @@ -472,7 +472,7 @@ class ParatooService { query = buildSurveyQueryString(start, limit) log.debug("Retrieving survey data from: "+url+query) response = webService.getJson(url+query, null, authHeader, false) - log.debug(response) + log.debug((response as JSON).toString()) survey = findMatchingSurvey(surveyId, response.data, config) } From c01835f355e6a0e5c5859fd53d0893aa11e58cd1 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 20 Mar 2024 12:09:35 +1100 Subject: [PATCH 12/52] Trying different matching approach #823 --- .../au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 71c32d4ef..558bf6c80 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -103,7 +103,10 @@ class ParatooProtocolConfig { log.debug(surveyData.toString()) return false } - tmpSurveyId?.orgMintedUUID == surveyId.survey_metadata?.orgMintedUUID + + tmpSurveyId?.survey_details?.survey_model == surveyId.survey_metadata?.survey_details.survey_model && + tmpSurveyId?.survey_details?.time == surveyId.survey_metadata?.survey_details.time && + tmpSurveyId?.survey_details?.uuid == surveyId.survey_metadata?.survey_details.uuid } private Map extractSiteDataFromPlotVisit(Map survey) { From 978a4c891e010bc6668c329c2d8c9b9f096fa6fd Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 20 Mar 2024 13:35:25 +1100 Subject: [PATCH 13/52] Changed name of site created as project extent #823 --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index a60b070b7..61c2dfdfe 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -575,7 +575,7 @@ class ParatooService { else { Map site = [ - name:'Monitor project area', + name:'Monitor Project Extent', type:Site.TYPE_PROJECT_AREA, extent: [ source:'drawn', From 5bb929ef35e33a10e88c74aa6e72ded55e91be22 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 20 Mar 2024 13:57:27 +1100 Subject: [PATCH 14/52] Don't return plot layouts from /user-projects #823 --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 5 ++++- src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 61c2dfdfe..3b1cebc1a 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -404,7 +404,10 @@ class ParatooService { projectAreaGeoJson = siteService.geometryAsGeoJson(projectArea) } - List plotSelections = sites.findAll{it.type == Site.TYPE_SURVEY_AREA} + // Monitor has users selecting a point as an approximate survey location then + // laying out the plot using GPS when at the site. We only want to return the approximate planning + // sites from this call + List plotSelections = sites.findAll{it.type == Site.TYPE_SURVEY_AREA && it.extent?.geometry?.type == 'Point'} Map attributes = [ id:project.projectId, diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 40e5166fb..2b346f75e 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -26,6 +26,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [projectArea, plot] From 05b115c1a64ffa2ab54a7b60845d94147c6de5c9 Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 22 Mar 2024 09:56:39 +1100 Subject: [PATCH 15/52] AtlasOfLivingAustralia/fieldcapture#3049 - synchronises protocols - creates records for submitted data --- grails-app/conf/application.groovy | 103 ++ .../org/ala/ecodata/ParatooController.groovy | 7 +- .../au/org/ala/ecodata/OutputService.groovy | 16 +- .../au/org/ala/ecodata/ParatooService.groovy | 1636 ++++++++++++++--- .../au/org/ala/ecodata/WebService.groovy | 19 +- .../au/org/ala/ecodata/GeometryUtils.groovy | 25 +- .../ecodata/converter/FeatureConverter.groovy | 47 + .../ecodata/converter/ListConverter.groovy | 31 +- .../ecodata/metadata/OutputMetadata.groovy | 8 +- .../ecodata/paratoo/ParatooCollection.groovy | 5 + .../paratoo/ParatooProtocolConfig.groovy | 113 +- .../org/ala/ecodata/ParatooServiceSpec.groovy | 966 +++++++++- .../paratoo/ParatooProtocolConfigSpec.groovy | 24 +- 13 files changed, 2651 insertions(+), 349 deletions(-) create mode 100644 src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 324357ad7..d410ed7ca 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -531,6 +531,7 @@ ecodata.documentation.exampleProjectUrl = 'http://ecodata-test.ala.org.au/ws/act // Used by ParatooService to sync available protocols paratoo.core.baseUrl = 'https://dev.core-api.monitor.tern.org.au/api' paratoo.excludeInterventionProtocols = true +paratoo.core.documentationUrl = '/documentation/swagger.json' auth.baseUrl = 'https://auth-test.ala.org.au' userDetails.web.url = "${auth.baseUrl}/userdetails/" @@ -1391,3 +1392,105 @@ elasticsearch { username = 'elastic' password = 'password' } + +// paratoo / monitor + +paratoo.defaultPlotLayoutDataModels = [ + [ + dataType: "geoMap", + name: "plot_layout", + dwcAttribute: "verbatimCoordinates", + validate: "required", + columns: [ + [ + dwcAttribute: "verbatimLatitude", + source: "plot_layoutLatitude" + ], + [ + dwcAttribute: "verbatimLongitude", + source: "plot_layoutLongitude" + ] + ] + ], + [ + dataType: "list", + name: "plot_visit", + validate: "required", + columns: [ + [ + dataType: "date", + name: "end_date", + dwcAttribute: "eventDate" + ], + [ + dataType: "text", + name: "visit_field_name" + ], + [ + dataType: "date", + name: "start_date", + dwcAttribute: "eventDate" + ] + ] + ] + ] + +paratoo.defaultPlotLayoutViewModels = [ + [ + type: "row", + items: [ + [ + type: "col", + items: [ + [ + type: "section", + title: "Plot Visit", + preLabel: "Plot Visit", + boxed: true, + items: [ + [ + type: "repeat", + source: "plot_visit", + userAddedRows: false, + items: [ + [ + type: "row", + class: "output-section", + items: [ + [ + type: "col", + items: [ + [ + type: "date", + source: "end_date", + preLabel: "End Date" + ], + [ + type: "text", + source: "visit_field_name", + preLabel: "Visit Field Name" + ], + [ + type: "date", + source: "start_date", + preLabel: "Start Date" + ] + ] + ] + ] + ] + ] + ] + ] + ], + [ + type: "geoMap", + source: "plot_layout", + orientation: "vertical" + ] + ] + ] + ] + ] + ] + diff --git a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy index 0042a736f..4ec8e3fec 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy @@ -1,12 +1,7 @@ package au.org.ala.ecodata import au.ala.org.ws.security.SkipApiKeyCheck -import au.org.ala.ecodata.paratoo.ParatooCollection -import au.org.ala.ecodata.paratoo.ParatooCollectionId -import au.org.ala.ecodata.paratoo.ParatooPlotSelection -import au.org.ala.ecodata.paratoo.ParatooPlotSelectionData -import au.org.ala.ecodata.paratoo.ParatooProject -import au.org.ala.ecodata.paratoo.ParatooToken +import au.org.ala.ecodata.paratoo.* import groovy.util.logging.Slf4j import io.swagger.v3.oas.annotations.OpenAPIDefinition import io.swagger.v3.oas.annotations.Operation diff --git a/grails-app/services/au/org/ala/ecodata/OutputService.groovy b/grails-app/services/au/org/ala/ecodata/OutputService.groovy index 6f049cec1..5e3597b67 100644 --- a/grails-app/services/au/org/ala/ecodata/OutputService.groovy +++ b/grails-app/services/au/org/ala/ecodata/OutputService.groovy @@ -3,7 +3,6 @@ package au.org.ala.ecodata import au.org.ala.ecodata.converter.RecordConverter import au.org.ala.ecodata.metadata.OutputMetadata -import static au.org.ala.ecodata.Status.ACTIVE import static au.org.ala.ecodata.Status.DELETED class OutputService { @@ -155,8 +154,12 @@ class OutputService { props.data = saveImages(props.data, props.name, output.outputId, props.activityId); props.data = saveAudio(props.data, props.name, output.outputId, props.activityId); - - createOrUpdateRecordsForOutput(activity, output, props) + try { + createOrUpdateRecordsForOutput(activity, output, props) + } + catch (Exception ex) { + log.error("Error creating records for activity - ${activity.activityId}", ex) + } commonService.updateProperties(output, props) return [status: 'ok', outputId: output.outputId] @@ -241,7 +244,12 @@ class OutputService { List statusUpdate = recordService.updateRecordStatusByOutput(outputId, Status.DELETED) if (!statusUpdate) { - createOrUpdateRecordsForOutput(activity, output, props) + try { + createOrUpdateRecordsForOutput(activity, output, props) + } + catch (Exception ex) { + log.error("Error creating records for activity - ${activity.activityId}", ex) + } commonService.updateProperties(output, props) result = [status: 'ok'] } else { diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index d61a1beb1..079dcfd83 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -1,17 +1,14 @@ package au.org.ala.ecodata -import au.org.ala.ecodata.paratoo.ParatooCollection -import au.org.ala.ecodata.paratoo.ParatooCollectionId -import au.org.ala.ecodata.paratoo.ParatooMintedIdentifier -import au.org.ala.ecodata.paratoo.ParatooPlotSelectionData -import au.org.ala.ecodata.paratoo.ParatooProject -import au.org.ala.ecodata.paratoo.ParatooProtocolConfig -import au.org.ala.ecodata.paratoo.ParatooProtocolId -import au.org.ala.ecodata.paratoo.ParatooSurveyId +import au.org.ala.ecodata.metadata.PropertyAccessor +import au.org.ala.ecodata.paratoo.* import au.org.ala.ws.tokens.TokenService import grails.converters.JSON import grails.core.GrailsApplication import groovy.util.logging.Slf4j + +import java.util.regex.Matcher +import java.util.regex.Pattern /** * Supports the implementation of the paratoo "org" interface */ @@ -19,6 +16,7 @@ import groovy.util.logging.Slf4j class ParatooService { static final String PARATOO_PROTOCOL_PATH = '/protocols' + static final String PARATOO_DATA_PATH = '/protocols/reverse-lookup' static final String PARATOO_PROTOCOL_FORM_TYPE = 'EMSA' static final String PARATOO_PROTOCOLS_KEY = 'paratoo.protocols' static final String PARATOO_PROTOCOL_DATA_MAPPING_KEY = 'paratoo.surveyData.mapping' @@ -29,6 +27,34 @@ class ParatooService { ['Plot Selection and Layout', 'Plot Description', 'Opportune'] static final List ADMIN_ONLY_PROTOCOLS = ['Plot Selection'] static final String INTERVENTION_PROTOCOL_TAG = 'intervention' + static final String PARATOO_UNIT_FIELD_NAME = "x-paratoo-unit" + static final String PARATOO_HINT = "x-paratoo-hint" + static final String PARATOO_MODEL_REF = "x-model-ref" + static final int PARATOO_FLOAT_DECIMAL_PLACES = 6 + static final String PARATOO_SCHEMA_POSTFIX = "Request" + static final String PARATOO_FILE_SCHEMA_NAME = "UploadFile" + static final String PARATOO_FILE_MODEL_NAME = "file" + static final String PARATOO_LUT_REF = "x-lut-ref" + static final String PARATOO_FILE_TYPE = "x-paratoo-file-type" + static final String PARATOO_IMAGE_FILE_TYPE = "images" + static final String PARATOO_SPECIES_TYPE = "x-paratoo-csv-list-taxa" + static final String PARATOO_FIELD_LABEL = "x-paratoo-rename" + static final List PARATOO_IGNORE_MODEL_LIST = [ + 'created_at', 'createdAt', 'updated_at', 'updatedAt', 'created_by', 'createdBy', 'updated_by', 'updatedBy', + 'published_at', 'publishedAt', 'x-paratoo-file-type' + ] + static final List PARATOO_IGNORE_X_MODEL_REF_LIST_MINIMUM = [ + 'file', 'admin::user' + ] + static final List PARATOO_IGNORE_X_MODEL_REF_LIST = [ + 'plot-visit', 'plot-selection', 'plot-layout' + ] + PARATOO_IGNORE_X_MODEL_REF_LIST_MINIMUM + static final String PARATOO_WORKFLOW_PLOT_LAYOUT = "plot-layout" + static final String PARATOO_COMPONENT = "x-paratoo-component" + static final String PARATOO_LOCATION_COMPONENT = "location.location" + static final String PARATOO_DATAMODEL_PLOT_LAYOUT = "plot_layout" + static final String PARATOO_DATAMODEL_PLOT_SELECTION = "plot_selection" + static final String PARATOO_DATAMODEL_PLOT_VISIT = "plot_visit" GrailsApplication grailsApplication SettingService settingService @@ -37,8 +63,10 @@ class ParatooService { SiteService siteService PermissionService permissionService TokenService tokenService - MetadataService metadataService + CacheService cacheService ActivityService activityService + RecordService recordService + MetadataService metadataService /** * The rules we use to find projects eligible for use by paratoo are: @@ -60,7 +88,7 @@ class ParatooService { project.protocols = findProjectProtocols(project) } - projects.findAll{it.protocols} + projects.findAll { it.protocols } } private List findProjectProtocols(ParatooProject project) { @@ -72,11 +100,11 @@ class ParatooService { List categoriesWithDefaults = monitoringProtocolCategories + DEFAULT_MODULES protocols += findProtocolsByCategories(categoriesWithDefaults.unique()) if (!project.isParaooAdmin()) { - protocols = protocols.findAll{!(it.name in ADMIN_ONLY_PROTOCOLS)} + protocols = protocols.findAll { !(it.name in ADMIN_ONLY_PROTOCOLS) } } // Temporarily exclude intervention protocols until they are ready if (grailsApplication.config.getProperty('paratoo.excludeInterventionProtocols', Boolean.class, true)) { - protocols = protocols.findAll{!(INTERVENTION_PROTOCOL_TAG in it.tags)} + protocols = protocols.findAll { !(INTERVENTION_PROTOCOL_TAG in it.tags) } } } @@ -102,8 +130,7 @@ class ParatooService { permission = permissionService.findParentPermission(permission) projectAccessLevels[projectId] = permission?.accessLevel } - } - else { + } else { // Update the map of projectId to accessLevel projectAccessLevels[projectId] = permission.accessLevel } @@ -159,7 +186,7 @@ class ParatooService { dataSet.orgMintedIdentifier = orgMintedIdentifier.encodeAsMintedCollectionId() log.info "Minting identifier for Monitor collection: ${paratooCollectionId}: ${dataSet.orgMintedIdentifier}" project.custom.dataSets << dataSet - Map result = projectService.update([custom:project.custom], projectId, false) + Map result = projectService.update([custom: project.custom], projectId, false) if (!result.error) { result.orgMintedIdentifier = dataSet.orgMintedIdentifier @@ -173,70 +200,131 @@ class ParatooService { dataSetName } + /** + * Stores a collection created by monitor. This method will create a site, activity and output. + * If species are recorded, record of it are automatically generated. + * @param collection + * @param project + * @return + */ Map submitCollection(ParatooCollection collection, ParatooProject project) { - - Map dataSet = project.project.custom?.dataSets?.find{it.orgMintedIdentifier == collection.orgMintedIdentifier} + boolean forceActivityCreation = true + Map dataSet = project.project.custom?.dataSets?.find { it.orgMintedIdentifier == collection.orgMintedIdentifier } if (!dataSet) { - throw new RuntimeException("Unable to find data set with orgMintedIdentifier: "+collection.orgMintedIdentifier) + throw new RuntimeException("Unable to find data set with orgMintedIdentifier: " + collection.orgMintedIdentifier) } dataSet.progress = Activity.STARTED ParatooSurveyId surveyId = ParatooSurveyId.fromMap(dataSet.surveyId) ParatooProtocolConfig config = getProtocolConfig(surveyId.protocol.id) - Map surveyData = retrieveSurveyData(surveyId, config) - List surveyObservations = retrieveSurveyObservations(surveyId, config) - - if (surveyData) { + ActivityForm form = ActivityForm.findByExternalId(surveyId.protocol.id) + // reverse lookup gets survey and observations but not plot based data + Map surveyDataAndObservations = retrieveSurveyAndObservations(surveyId, config, collection) + // get survey data from reverse lookup response + Map surveyData = config.getSurveyDataFromObservation(surveyDataAndObservations) + // get plot data by querying protocol api endpoint + Map surveyDataWithPlotInfo = retrieveSurveyData(surveyId, config, surveyData.createdAt) + // add plot data to survey observations + addPlotDataToObservations(surveyDataWithPlotInfo, surveyDataAndObservations, config) + if (surveyDataAndObservations) { + // transform data to make it compatible with data model + surveyDataAndObservations = recursivelyTransformData(form.sections[0].template.dataModel, surveyDataAndObservations) + rearrangeSurveyData(surveyDataAndObservations, surveyDataAndObservations, form.sections[0].template.relationships.ecodata) // If we are unable to create a site, null will be returned - assigning a null siteId is valid. - dataSet.siteId = createSiteFromSurveyData(surveyData, collection, surveyId, project.project, config) - List species = createSpeciesFromSurveyData(surveyObservations, collection, config, dataSet ) - dataSet.areSpeciesRecorded = species?.size() > 0 - dataSet.startDate = config.getStartDate(surveyData) - dataSet.endDate = config.getEndDate(surveyData) + if (!dataSet.siteId) + dataSet.siteId = createSiteFromSurveyData(surveyDataWithPlotInfo, surveyDataAndObservations, collection, surveyId, project.project, config, form) + + // make sure activity has not been created for this data set + if (!dataSet.activityId || forceActivityCreation) { + String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, collection, dataSet.siteId) + List records = recordService.getAllByActivity(activityId) + dataSet.areSpeciesRecorded = records?.size() > 0 + dataSet.activityId = activityId + } - createActivityFromSurveyData(surveyId, collection.orgMintedIdentifier, surveyData, config, project) + dataSet.startDate = config.getStartDate(surveyDataAndObservations) + dataSet.endDate = config.getEndDate(surveyDataAndObservations) + } else { + log.warn("Unable to retrieve survey data for: " + collection.orgMintedIdentifier) } - else { - log.warn("Unable to retrieve survey data for: "+collection.orgMintedIdentifier) - } - - Map result = projectService.update([custom:project.project.custom], project.id, false) - result + projectService.update([custom: project.project.custom], project.id, false) } - private void createActivityFromSurveyData(ParatooSurveyId paratooSurveyId, String mintedCollectionId, Map surveyData, ParatooProtocolConfig config, ParatooProject project) { - ActivityForm form = ActivityForm.findByExternalId(paratooSurveyId.protocol.id) - if (!form) { - log.error("No activity form found for protocol: "+paratooSurveyId.protocol.id) - } - else { - Map activity = mapActivity(mintedCollectionId, paratooSurveyId, surveyData, form, config, project) - activityService.create(activity) + /** + * Rearrange survey data to match the data model. + * e.g. [a: [b: [c: 1, d: 2], d: 1], b: [c: 1, d: 2]] => [b: [c: 1, d: 2, a: [d: 1]]] + * where relationship [b: [a: [:]]] + * @param properties + * @param rootProperties + * @param relationship + * @param nodesToRemove + * @param ancestors + * @param isChild + * @return + */ + Map rearrangeSurveyData (Map properties, Map rootProperties, Map relationship, List nodesToRemove = [], List ancestors = [] , Boolean isChild = false) { + if (relationship instanceof Map) { + relationship.each { String nodeName, Map children -> + ancestors.add(nodeName) + def nodeObject = rootProperties[nodeName] ?: [:] + if (isChild) { + properties[nodeName] = nodeObject + nodesToRemove.add(nodeName) + } + + if (children) { + if (nodeObject instanceof Map) { + rearrangeSurveyData(nodeObject, rootProperties, children, nodesToRemove, ancestors, true ) + } + else if (nodeObject instanceof List) { + nodeObject.each { Map node -> + rearrangeSurveyData(node, rootProperties, children, nodesToRemove, ancestors, true ) + } + } + } + } } - } + // remove nodes that have been rearranged. removing during iteration will cause exception. + if (!isChild) { + nodesToRemove.each { String node -> + properties.remove(node) + } - private static Map mapActivity(String mintedCollectionId, ParatooSurveyId surveyId, Map surveyData, ActivityForm activityForm, ParatooProtocolConfig config, ParatooProject project) { - Map activity = [:] - activity.projectId = project.id - activity.startDate = config.getStartDate(surveyData) - activity.endDate = config.getEndDate(surveyData) - activity.type = activityForm.name - activity.description = activityForm.name + " - " + surveyId.timeAsDisplayDate() + nodesToRemove.clear() + } - Map output = [ - name: 'Unstructured', - data: surveyData - ] - activity.outputs = [output] - activity.externalIds = [new ExternalId(idType:ExternalId.IdType.MONITOR_MINTED_COLLECTION_ID, externalId: mintedCollectionId)] + properties + } - activity + /** + * Extract plot selection, layout and visit data from survey and copy it to the observations + * @param surveyData + * @param surveyDataAndObservations + * @param config + */ + static void addPlotDataToObservations(Map surveyData, Map surveyDataAndObservations, ParatooProtocolConfig config) { + if (surveyDataAndObservations && surveyData) { + Map plotSelection = config.getPlotSelection(surveyData) + Map plotLayout = config.getPlotLayout(surveyData) + Map plotVisit = config.getPlotVisit(surveyData) + + if (plotSelection) { + surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_SELECTION] = plotSelection + surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_LAYOUT] = plotLayout + surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_VISIT] = plotVisit + } + } } + /** + * Get the protocol config for the given protocol id. + * @param protocolId + * @return + */ private ParatooProtocolConfig getProtocolConfig(String protocolId) { String result = settingService.getSetting(PARATOO_PROTOCOL_DATA_MAPPING_KEY) Map protocolDataConfig = JSON.parse(result ?: '{}') @@ -254,8 +342,8 @@ class ParatooService { private boolean protocolCheck(String userId, String projectId, String protocolId, boolean read) { List projects = userProjects(userId) - ParatooProject project = projects.find{it.id == projectId} - boolean protocol = project?.protocols?.find{it.externalIds.find{it.externalId == protocolId}} + ParatooProject project = projects.find { it.id == projectId } + boolean protocol = project?.protocols?.find { it.externalIds.find { it.externalId == protocolId } } int minimumAccess = read ? AccessLevel.projectParticipant.code : AccessLevel.editor.code protocol && project.accessLevel.code >= minimumAccess } @@ -268,74 +356,109 @@ class ParatooService { dataSet = it.dataSets?.find { it.orgMintedIdentifier == collectionId } dataSet } - [dataSet:dataSet, project:project] + [dataSet: dataSet, project: project] } - private List createSpeciesFromSurveyData(List surveyObservations, ParatooCollection collection, ParatooProtocolConfig config, Map dataSet) { - // delete records - Record.where { - dataSetId == dataSet.dataSetId - }.deleteAll() + /** + * Create an activity from survey data. + * @param activityForm + * @param surveyObservations + * @param collection + * @param siteId + * @return + */ + private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollection collection, String siteId) { + Map activityProps = [ + type : activityForm.name, + formVersion : activityForm.formVersion, + description : "Activity submitted by monitor", + projectId : collection.projectId, + publicationStatus: "published", + siteId : siteId, + userId : collection.parseOrgMintedIdentifier()?.userId, + outputs : [[ + data: surveyObservations, + name: activityForm.name + ]] + ] - createRecords(surveyObservations, config, collection, dataSet) + Map result = activityService.create(activityProps) + result.activityId } - private static List createRecords (List surveyObservations, ParatooProtocolConfig config, ParatooCollection collection, Map dataSet) { - List result = [] - surveyObservations?.each { observation -> - def obs = transformSpeciesObservation(observation, config, collection, dataSet) - def record = new Record(obs) - try { - record.save(flush: true, failOnError: true) - result.add(record) - } catch (Exception e) { - log.error("Error saving record: ${record.name} ${record.projectId}", e) + /** + * Converts species, feature, document, image and list to appropriate formats. + * @param dataModel + * @param output + * @param path + * @return + */ + def recursivelyTransformData(List dataModel, Map output) { + dataModel?.each { Map model -> + switch (model.dataType) { + case "list": + String updatedPath = model.name + def rows = getProperty(output, updatedPath) + if (rows instanceof Map) { + output[updatedPath] = rows = [rows] + } + + rows?.each { row -> + recursivelyTransformData(model.columns, row) + } + break + case "species": + String speciesName = getProperty(output, model.name) + output[model.name] = transformSpeciesName(speciesName) + break + case "feature": + // used by protocols like bird survey where a point represents a sight a bird has been observed in a + // bird survey plot + def point = output[model.name] + output[model.name] = [ + type : 'Point', + coordinates: [point.lng, point.lat] + ] + break + case "image": + case "document": + // backup a copy of multimedia to another attribute and remove it from existing attribute since it interferes with existing logic + String backupAttributeName = "${model.name}_backup" + output[backupAttributeName] = output[model.name] + output.remove(model.name) + break } } - result + output } - private static Map transformSpeciesObservation (Map observation, ParatooProtocolConfig config, ParatooSurveyId surveyId, ParatooCollection collection, Map dataSet) { - def lat = config.getDecimalLatitude(observation), lng = config.getDecimalLongitude(observation) - Map result = [ - dataSetId: dataSet.dataSetId, - projectId: surveyId.projectId, - eventDate: config.getEventDate(observation), - decimalLatitude: lat, - decimalLongitude: lng, - individualCount: config.getIndividualCount(observation), - individualsOrGroups: config.getIndividualOrGroup(observation), - recordedBy: config.getRecordedBy(observation) - ] - - result << config.parseSpecies(config.getSpecies(observation)) - result - } - - private String createSiteFromSurveyData(Map surveyData, ParatooCollection collection, ParatooSurveyId surveyId, Project project, ParatooProtocolConfig config) { + private String createSiteFromSurveyData(Map surveyData, Map observation, ParatooCollection collection, ParatooSurveyId surveyId, Project project, ParatooProtocolConfig config, ActivityForm form) { String siteId = null // Create a site representing the location of the collection - Map geoJson = config.getGeoJson(surveyData) + Map geoJson = config.getGeoJson(surveyData, observation, form) if (geoJson) { + List features = geoJson?.features ?: [] + geoJson.remove('features') Map siteProps = siteService.propertiesFromGeoJson(geoJson, 'upload') siteProps.type = Site.TYPE_SURVEY_AREA siteProps.publicationStatus = PublicationStatus.PUBLISHED siteProps.projects = [project.projectId] + siteProps.features = features String externalId = geoJson.properties?.externalId if (externalId) { - siteProps.externalIds = [new ExternalId(idType:ExternalId.IdType.MONITOR_PLOT_GUID, externalId: externalId)] + siteProps.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: externalId)] } Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, externalId) Map result if (!site) { result = siteService.create(siteProps) + } else { + result = [siteId: site.siteId] } - else { - result = [siteId:site.siteId] - } - if (result.error) { // Don't treat this as a fatal error for the purposes of responding to the paratoo request - log.error("Error creating a site for survey "+collection.orgMintedIdentifier+", project "+project.projectId+": "+result.error) + if (result.error) { + // Don't treat this as a fatal error for the purposes of responding to the paratoo request + log.error("Error creating a site for survey " + collection.orgMintedIdentifier + ", project " + project.projectId + ": " + result.error) } siteId = result.siteId } @@ -343,52 +466,55 @@ class ParatooService { } private Map syncParatooProtocols(List protocols) { - - Map result = [errors:[], messages:[]] + Map result = [errors: [], messages: []] List guids = [] protocols.each { Map protocol -> + String message String id = protocol.id String guid = protocol.attributes.identifier guids << guid String name = protocol.attributes.name - ParatooProtocolConfig protocolConfig = getProtocolConfig(guid) - ActivityForm form = ActivityForm.findByExternalId(guid) - if (!form) { - form = new ActivityForm() - form.externalIds = [] - form.externalIds << new ExternalId(idType: ExternalId.IdType.MONITOR_PROTOCOL_INTERNAL_ID, externalId: id) - form.externalIds << new ExternalId(idType: ExternalId.IdType.MONITOR_PROTOCOL_GUID, externalId: guid) - - String message = "Creating form with id: "+id+", name: "+name - result.messages << message - log.info message - } - else { - ExternalId paratooInternalId = form.externalIds.find{it.idType == ExternalId.IdType.MONITOR_PROTOCOL_INTERNAL_ID} - - // Paratoo internal protocol ids are not stable so if we match the guid, we may need to update - // the id as that is used in other API methods. - if (paratooInternalId) { - String message = "Updating form with id: "+paratooInternalId.externalId+", guid: "+guid+", name: "+name+", new id: "+id - paratooInternalId.externalId = id + if (guid) { + ParatooProtocolConfig protocolConfig = getProtocolConfig(guid) + ActivityForm form = ActivityForm.findByExternalId(guid) + if (!form) { + form = new ActivityForm() + form.externalIds = [] + form.externalIds << new ExternalId(idType: ExternalId.IdType.MONITOR_PROTOCOL_INTERNAL_ID, externalId: id) + form.externalIds << new ExternalId(idType: ExternalId.IdType.MONITOR_PROTOCOL_GUID, externalId: guid) + + message = "Creating form with id: " + id + ", name: " + name result.messages << message log.info message - } - else { - String error = "Error: Missing internal id for form with id: "+id+", name: "+name - result.errors << error - log.error error - } + } else { + ExternalId paratooInternalId = form.externalIds.find { it.idType == ExternalId.IdType.MONITOR_PROTOCOL_INTERNAL_ID } + + // Paratoo internal protocol ids are not stable so if we match the guid, we may need to update + // the id as that is used in other API methods. + if (paratooInternalId) { + message = "Updating form with id: " + paratooInternalId.externalId + ", guid: " + guid + ", name: " + name + ", new id: " + id + paratooInternalId.externalId = id + result.messages << message + log.info message + } else { + String error = "Error: Missing internal id for form with id: " + id + ", name: " + name + result.errors << error + log.error error + } - } + } - List tags = protocolConfig?.tags ?: [ActivityForm.SURVEY_TAG] - mapProtocolToActivityForm(protocol, form, tags) - form.save() + mapProtocolToActivityForm(protocol, form, protocolConfig) + form.save() - if (form.hasErrors()) { - result.errors << form.errors - log.warn "Error saving form with id: "+id+", name: "+name + if (form.hasErrors()) { + result.errors << form.errors + log.warn "Error saving form with id: " + id + ", name: " + name + } + } else { + String error = "Error: No valid guid found for protocol: " + name + result.errors << error + log.error error } } @@ -399,10 +525,12 @@ class ParatooService { status != Status.DELETED } - List deletions = allProtocolForms.findAll{it.externalIds.find{it.idType == ExternalId.IdType.MONITOR_PROTOCOL_GUID && !(it.externalId in guids)}} + List deletions = allProtocolForms.findAll { it.externalIds.find { it.idType == ExternalId.IdType.MONITOR_PROTOCOL_GUID && !(it.externalId in guids) } } deletions.each { ActivityForm activityForm -> - result.messages << "Form ${activityForm.name} with guid: ${activityForm.externalIds.find{it.idType == ExternalId.IdType.MONITOR_PROTOCOL_GUID}.externalId} has been deleted" + result.messages << "Form ${activityForm.name} with guid: ${activityForm.externalIds.find { it.idType == ExternalId.IdType.MONITOR_PROTOCOL_GUID }.externalId} has been deleted" } + + log.debug("Completed syncing paratoo protocols") result } @@ -417,19 +545,37 @@ class ParatooService { grailsApplication.config.getProperty('paratoo.core.baseUrl') } + private String getDocumentationEndpoint() { + grailsApplication.config.getProperty('paratoo.core.documentationUrl') + } - Map syncProtocolsFromParatoo() { - String url = paratooBaseUrl+PARATOO_PROTOCOL_PATH + List getProtocolsFromParatoo() { + (List) cacheService.get("paratoo-protocols", { -> + String url = paratooBaseUrl + PARATOO_PROTOCOL_PATH + Map authHeader = getAuthHeader() + webService.getJson(url, null, authHeader, false)?.data + }) + } + + Map getAuthHeader() { String accessToken = tokenService.getAuthToken(true) if (!accessToken?.startsWith('Bearer')) { - accessToken = 'Bearer '+accessToken + accessToken = 'Bearer ' + accessToken } - Map authHeader = [(MONITOR_AUTH_HEADER):accessToken] - Map response = webService.getJson(url, null, authHeader, false) - syncParatooProtocols(response?.data) + + if (!accessToken) { + throw new RuntimeException("Unable to get access token") + } + + [(MONITOR_AUTH_HEADER): accessToken] } - private static void mapProtocolToActivityForm(Map protocol, ActivityForm form, List tags) { + Map syncProtocolsFromParatoo() { + List protocols = getProtocolsFromParatoo() + syncParatooProtocols(protocols) + } + + private void mapProtocolToActivityForm(Map protocol, ActivityForm form, ParatooProtocolConfig config) { form.name = protocol.attributes.name form.formVersion = protocol.attributes.version form.type = PARATOO_PROTOCOL_FORM_TYPE @@ -437,27 +583,28 @@ class ParatooService { form.external = true form.publicationStatus = PublicationStatus.PUBLISHED form.description = protocol.attributes.description - form.tags = tags + form.tags = config.tags form.externalIds + form.sections = [getFormSectionForProtocol(protocol, config)] } - private ParatooProject mapProject(Project project, AccessLevel accessLevel, List sites) { - Site projectArea = sites.find{it.type == Site.TYPE_PROJECT_AREA} + ParatooProject mapProject(Project project, AccessLevel accessLevel, List sites) { + Site projectArea = sites.find { it.type == Site.TYPE_PROJECT_AREA } Map projectAreaGeoJson = null if (projectArea) { projectAreaGeoJson = siteService.geometryAsGeoJson(projectArea) } - List plotSelections = sites.findAll{it.type == Site.TYPE_SURVEY_AREA} + List plotSelections = sites.findAll { it.type == Site.TYPE_SURVEY_AREA } Map attributes = [ - id:project.projectId, - name:project.name, - accessLevel: accessLevel, - project:project, - projectArea: projectAreaGeoJson, + id : project.projectId, + name : project.name, + accessLevel : accessLevel, + project : project, + projectArea : projectAreaGeoJson, projectAreaSite: projectArea, - plots: plotSelections] + plots : plotSelections] new ParatooProject(attributes) } @@ -474,73 +621,44 @@ class ParatooService { dataSet } - private static String buildSurveyQueryString(int start, int limit) { - "?populate=deep&sort=updatedAt&pagination[start]=$start&pagination[limit]=$limit" + private static String buildSurveyQueryString(int start, int limit, String createdAt) { + "?populate=deep&sort=updatedAt&pagination[start]=$start&pagination[limit]=$limit&filters[createdAt][\$eq]=$createdAt" } - Map retrieveSurveyData(ParatooSurveyId surveyId, ParatooProtocolConfig config) { + Map retrieveSurveyData(ParatooSurveyId surveyId, ParatooProtocolConfig config, String createdAt) { String apiEndpoint = config.getApiEndpoint(surveyId) + Map authHeader = getAuthHeader() - String accessToken = tokenService.getAuthToken(true) - if (!accessToken?.startsWith('Bearer')) { - accessToken = 'Bearer '+accessToken - } - Map authHeader = [(MONITOR_AUTH_HEADER):accessToken] - - if (!accessToken) { - throw new RuntimeException("Unable to get access token") - } int start = 0 int limit = 10 - - String url = paratooBaseUrl+'/'+apiEndpoint - String query = buildSurveyQueryString(start, limit) - Map response = webService.getJson(url+query, null, authHeader, false) + String url = paratooBaseUrl + '/' + apiEndpoint + String query = buildSurveyQueryString(start, limit, createdAt) + Map response = webService.getJson(url + query, null, authHeader, false) Map survey = findMatchingSurvey(surveyId, response.data, config) int total = response.meta?.pagination?.total ?: 0 - while (!survey && start+limit < total) { + while (!survey && start + limit < total) { start += limit - - query = buildSurveyQueryString(start, limit) - response = webService.getJson(url+query, null, authHeader, false) + query = buildSurveyQueryString(start, limit, createdAt) + response = webService.getJson(url + query, null, authHeader, false) survey = findMatchingSurvey(surveyId, response.data, config) } survey } - List retrieveSurveyObservations(ParatooSurveyId surveyId, ParatooProtocolConfig config) { - - String apiEndpoint = config.observationEndpoint - String accessToken = tokenService.getAuthToken(true) - if (!accessToken?.startsWith('Bearer')) { - accessToken = 'Bearer '+accessToken - } - Map authHeader = [(MONITOR_AUTH_HEADER):accessToken] - - if (!accessToken) { - throw new RuntimeException("Unable to get access token") - } - int start = 0 - int limit = 10 - - - String url = paratooBaseUrl+'/'+apiEndpoint - String query = buildSurveyQueryString(start, limit) - Map response = webService.getJson(url+query, null, authHeader, false) - List data = config.findObservationsBelongingToSurvey(response.data, surveyId) ?: [] - int total = response.meta?.pagination?.total ?: 0 - while (!data && start+limit < total) { - start += limit + Map retrieveSurveyAndObservations(ParatooSurveyId surveyId, ParatooProtocolConfig config, ParatooCollection collection) { + String apiEndpoint = PARATOO_DATA_PATH + Map payload = [ + collection_id: collection.orgMintedIdentifier + ] - query = buildSurveyQueryString(start, limit) - response = webService.getJson(url+query, null, authHeader, false) - data.addAll(config.findObservationsBelongingToSurvey(response.data, surveyId)) - } + Map authHeader = getAuthHeader() - data + String url = paratooBaseUrl + apiEndpoint + Map response = webService.doPost(url, payload, false, authHeader)?.resp + response?.collections } @@ -552,7 +670,7 @@ class ParatooService { List projects = userProjects(userId) if (!projects) { - return [error:'User has no projects eligible for Monitor site data'] + return [error: 'User has no projects eligible for Monitor site data'] } Map siteData = mapPlotSelection(plotSelectionData) @@ -563,8 +681,7 @@ class ParatooService { Map result if (site) { result = siteService.update(siteData, site.siteId) - } - else { + } else { result = siteService.create(siteData) } @@ -574,10 +691,12 @@ class ParatooService { private static Map mapPlotSelection(ParatooPlotSelectionData plotSelectionData) { Map geoJson = ParatooProtocolConfig.plotSelectionToGeoJson(plotSelectionData) Map site = SiteService.propertiesFromGeoJson(geoJson, 'point') - site.projects = [] // get all projects for the user I suppose - not sure why this isn't in the payload as it's in the UI... + site.projects = [] + // get all projects for the user I suppose - not sure why this isn't in the payload as it's in the UI... site.type = Site.TYPE_SURVEY_AREA - site.externalIds = [new ExternalId(idType:ExternalId.IdType.MONITOR_PLOT_GUID, externalId:geoJson.properties.externalId)] - site.publicationStatus = PublicationStatus.PUBLISHED // Mark the plot as read only as it is managed by the Monitor app + site.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: geoJson.properties.externalId)] + site.publicationStatus = PublicationStatus.PUBLISHED + // Mark the plot as read only as it is managed by the Monitor app site } @@ -585,7 +704,7 @@ class ParatooService { Map updateProjectSites(ParatooProject project, Map siteData, List userProjects) { if (siteData.plot_selections) { List siteExternalIds = siteData.plot_selections - siteExternalIds = siteExternalIds.findAll{it} // Remove null / empty ids + siteExternalIds = siteExternalIds.findAll { it } // Remove null / empty ids if (siteExternalIds) { linkProjectToSites(project, siteExternalIds, userProjects) } @@ -609,7 +728,7 @@ class ParatooService { // Validate that the user has permission to link the site to the project by checking // if the user has permission on any other projects this site is linked to. if (site.projects) { - if (!userProjects.collect{it.id}.containsAll(site.projects)) { + if (!userProjects.collect { it.id }.containsAll(site.projects)) { errors << "User does not have permission to link site ${site.externalId} to project ${project.id}" return } @@ -620,12 +739,11 @@ class ParatooService { errors << site.errors } } - } - else { + } else { errors << "No site exists with externalId = ${siteExternalId}" } } - [success:!errors, error:errors] + [success: !errors, error: errors] } private Map updateProjectArea(ParatooProject project, String type, List coordinates) { @@ -635,19 +753,1093 @@ class ParatooService { projectArea.extent.geometry.type = geometry.type projectArea.extent.geometry.coordinates = geometry.coordinates siteService.update(projectArea.extent, projectArea.siteId) - } - else { + } else { Map site = [ - name:'Monitor project area', - type:Site.TYPE_PROJECT_AREA, - extent: [ - source:'drawn', - geometry:geometry + name : 'Monitor project area', + type : Site.TYPE_PROJECT_AREA, + extent : [ + source : 'drawn', + geometry: geometry ], projects: [project.id] ] siteService.create(site) } } + + Map getParatooSwaggerDocumentation() { + (Map) cacheService.get('paratoo-swagger-documentation', { + String url = "${getParatooBaseUrl()}${getDocumentationEndpoint()}" + webService.getJson(url, null, null, false) + }) + } + + FormSection getFormSectionForProtocol(Map protocol, ParatooProtocolConfig config) { + Map documentation = getParatooSwaggerDocumentation() + Map template = buildTemplateForProtocol(protocol, documentation, config) + + new FormSection(template: template, templateName: template.modelName, name: protocol.attributes.name) + } + + Map buildPathToModel(Map properties) { + Map relationship = buildChildParentRelationship(properties) + buildTreeFrom2DRelationship(relationship) + } + + def getModelStructureFromDefinition(def definition) { + if (definition instanceof Map) + definition?.properties?.data?.properties ?: definition?.items?.properties?.data?.properties ?: definition?.items?.properties ?: definition?.properties ?: definition?.items + } + + def getRequiredModels(Map properties) { + properties.properties?.data?.required ?: properties.items?.properties?.data?.required ?: properties.items?.required ?: properties.properties?.required ?: properties?.required ?: [] + } + + Map simplifyModelStructure(Map definition) { + Map simplifiedDefinition = [:] + Map properties = getModelStructureFromDefinition(definition) + List required = getRequiredModels(definition) + String componentName = definition[PARATOO_COMPONENT] + if (properties) { + simplifiedDefinition.type = definition.type ?: "object" + simplifiedDefinition.properties = properties + } else { + simplifiedDefinition << definition + } + + if (required) { + simplifiedDefinition.required = required + } + + if (componentName) { + simplifiedDefinition[PARATOO_COMPONENT] = componentName + } + + simplifiedDefinition + } + + def cleanSwaggerDefinition(def definition) { + def cleanedDefinition + if (definition instanceof Map) { + cleanedDefinition = simplifyModelStructure(definition) + if (cleanedDefinition.properties) { + cleanedDefinition.properties?.each { String name, def value -> + cleanedDefinition.properties[name] = cleanSwaggerDefinition(value) + } + } else { + cleanedDefinition?.each { String name, def value -> + def cleanedValue = value instanceof Map ? simplifyModelStructure(value) : value + cleanedDefinition[name] = cleanSwaggerDefinition(cleanedValue) + } + } + } else if (definition instanceof List) { + cleanedDefinition = [] + definition.each { def value -> + cleanedDefinition << cleanSwaggerDefinition(value) + } + } else { + try { + cleanedDefinition = definition?.clone() + } + catch (CloneNotSupportedException e) { + // if not cloneable, then it is a primitive type + cleanedDefinition = definition + } + } + + cleanedDefinition + } + + Map buildChildParentRelationship(def definition, ArrayDeque currentNodes = null, Map relationship = null) { + currentNodes = currentNodes ?: new ArrayDeque() + relationship = relationship != null ? relationship : [:].withDefault { [] } as Map + if (definition instanceof Map) { + // model references are usually has the following representation + // { + // "type": "integer", + // "x-model-ref": "bird-survey" + // } + if (definition[PARATOO_MODEL_REF] && (definition["type"] == "integer")) { + String modelName = definition[PARATOO_MODEL_REF] + String parentNode = currentNodes.first() + String grandParentNode = currentNodes.size() > 1 ? currentNodes.getAt(1) : null + if (!PARATOO_IGNORE_X_MODEL_REF_LIST_MINIMUM.contains(modelName)) { + // no circular references + List nodes = [grandParentNode, parentNode, modelName].unique() + if ((nodes.size() == 3) && + !currentNodes.contains(modelName) && + !(relationship.containsKey(grandParentNode) && relationship[grandParentNode].contains(modelName))) { + String nodeListToParent = parentNode ? currentNodes.toList().reverse().join(".") : null + // make sure there are no duplicate child entries + if (nodeListToParent && (!relationship.containsKey(nodeListToParent) + || !relationship[nodeListToParent].contains(modelName))) { + relationship[nodeListToParent].add(modelName) + } + } + } + } else if (definition.properties && definition.properties[PARATOO_MODEL_REF]) { + // For representation like + // { + // "quad" : { + // "type" : "array", + // "properties" : { + // "type" : "integer" + // "x-model-ref" : "quadrant" + // } + // } + // } + buildChildParentRelationship(definition.properties, currentNodes, relationship) + } else { + if (definition instanceof Map) { + definition?.each { String name, model -> + currentNodes.push(name) + buildChildParentRelationship(model, currentNodes, relationship) + currentNodes.pop() + } + } + } + } + + relationship + } + + Map buildTreeRelationshipOfModels(Map properties) { + def twoDRelationships = buildParentChildRelationship(properties) + buildTreeFrom2DRelationship(twoDRelationships) + } + + def buildParentChildRelationship(def definition, ArrayDeque ancestorNodes = null, Map relationships = null) { + ancestorNodes = ancestorNodes ?: new ArrayDeque() + relationships = relationships != null ? relationships : [:].withDefault { [] } + if (definition instanceof Map) { + // model references are usually has the following representation + // { + // "type": "integer", + // "x-model-ref": "bird-survey" + // } + if (definition[PARATOO_MODEL_REF] && (definition["type"] == "integer")) { + String modelName = definition[PARATOO_MODEL_REF] + String currentNode = ancestorNodes.size() ? ancestorNodes.last() : name + if (!PARATOO_IGNORE_X_MODEL_REF_LIST.contains(modelName)) { + if (!relationships.containsKey(modelName) || !relationships[modelName].contains(currentNode)) { + // prevent circular references + if (modelName != currentNode + && !ancestorNodes.contains(modelName) + && (!relationships.containsKey(currentNode) || !relationships[currentNode].contains(modelName))) { + relationships[modelName].add(currentNode) + } + } + } + } else if (definition.properties && definition.properties[PARATOO_MODEL_REF]) { + // For representation like + // { + // "quad" : { + // "type" : "array", + // "properties" : { + // "type" : "integer" + // "x-model-ref" : "quadrant" + // } + // } + // } + buildParentChildRelationship(definition.properties, ancestorNodes, relationships) + } else { + def iteratingObject = definition.properties ?: definition + if (iteratingObject instanceof Map) { + iteratingObject?.each { String modelName, model -> + ancestorNodes.push(modelName) + buildParentChildRelationship(model, ancestorNodes, relationships) + ancestorNodes.pop() + } + } + } + } + + relationships + } + + /** + * Builds a tree representation from a list of two dimensional relationships. + * @param twoDRelationships + * e.g. + * [ + * "bird-survey": ["bird-observation"], + * "fauna-survey": ["fauna-observation"], + * "plot-visit": ["bird-survey", "fauna-survey"] + * ] + * @return + */ + Map buildTreeFrom2DRelationship(Map twoDRelationships) { + Map treeRepresentation = [:] + + twoDRelationships.each { String name, List children -> + treeRepresentation[name] = addChildrenToNode(children) + } + + List nodesToRemove = iterateTreeAndAddChildren(treeRepresentation) + + nodesToRemove?.each { + treeRepresentation.remove(it) + } + + treeRepresentation + } + + List iterateTreeAndAddChildren(Map treeRepresentation, Map root = null, HashSet nodesToRemove = null, depth = 0, ArrayDeque visited = null) { + nodesToRemove = nodesToRemove ?: new HashSet() + root = root ?: treeRepresentation + visited = visited ?: new ArrayDeque() + treeRepresentation.each { String parent, Map children -> + visited.push(parent) + children?.each { String child, Map grandChildren -> + if (!visited.contains(child)) { + visited.push(child) + if (root.containsKey(child)) { + grandChildren << deepCopy(root[child]) + nodesToRemove.add(child) + } + + iterateTreeAndAddChildren(children, root, nodesToRemove, depth + 1, visited) + visited.pop() + } + } + visited.pop() + } + + nodesToRemove.toList() + } + + Map addChildrenToNode(List children) { + Map childrenNode = [:] + children?.each { String child -> + if (!childrenNode.containsKey(child)) + childrenNode.put(child, [:]) + } + + childrenNode + } + + public static def deepCopy(def original) { + def copy + + if (original instanceof Map) { + copy = [:] + original.each { String key, Object value -> + copy.put(key, deepCopy(value)) + } + } else if (original instanceof List) { + copy = original.collect { deepCopy(it) } + } else { + copy = original + } + + copy + } + + Map buildTemplateForProtocol(Map protocol, Map documentation, ParatooProtocolConfig config) { + ArrayDeque modelVisitStack = new ArrayDeque<>() + documentation = deepCopy(documentation) + Map components = deepCopy(getComponents(documentation)) + + Map template = [dataModel: [], viewModel: [], modelName: capitalizeModelName(protocol.attributes.name), record: true, relationships: [ecodata: [:], apiOutput: [:]]] + Map properties = deepCopy(findProtocolEndpointDefinition(protocol, documentation)) + resolveReferences(properties, components) + Map cleanedProperties = cleanSwaggerDefinition(properties) + cleanedProperties = deepCopy(cleanedProperties) + template.relationships.ecodata = buildTreeRelationshipOfModels(cleanedProperties) + template.relationships.apiOutput = buildPathToModel(cleanedProperties) + println((template.relationships.apiOutput as JSON).toString(true)) + println((template.relationships.ecodata as JSON).toString(true)) + resolveModelReferences(cleanedProperties, components) + cleanedProperties = rearrangePropertiesAccordingToModelRelationship(cleanedProperties, template.relationships.apiOutput, template.relationships.ecodata) + cleanedProperties = deepCopy(cleanedProperties) + log.debug((properties as JSON).toString()) + + if (isPlotLayoutNeededByProtocol(protocol)) { + template.dataModel.addAll(grailsApplication.config.getProperty("paratoo.defaultPlotLayoutDataModels", List)) + template.viewModel.addAll(grailsApplication.config.getProperty("paratoo.defaultPlotLayoutViewModels", List)) + } + cleanedProperties.properties.each { String name, def definition -> + if (definition instanceof Map) { + modelVisitStack.push(name) + Map result = convertToDataModelAndViewModel(definition, documentation, name, null, modelVisitStack, 0, name, config) + modelVisitStack.pop() + if (result) { + template.dataModel.addAll(result.dataModel) + template.viewModel.addAll(result.viewModel) + } + } + } + + template + } + + def resolveModelReferences(def model, Map components, ArrayDeque modelVisitStack = null) { + modelVisitStack = modelVisitStack ?: new ArrayDeque() + boolean modelNameStacked = false + if (model instanceof Map) { + if (model.containsKey(PARATOO_MODEL_REF)) { + String modelName = model[PARATOO_MODEL_REF] + String componentName = getSchemaNameFromModelName(modelName) + Map referencedComponent = components[componentName] + + if (referencedComponent) { + if (modelVisitStack.contains(modelName)) + log.error("Circular dependency - ignoring model resolution ${modelName}") + else { + modelNameStacked = true + modelVisitStack.push(modelName) + model << cleanSwaggerDefinition(components[componentName]) + } + } + } + + model.each { String modelName, def value -> + resolveModelReferences(value, components, modelVisitStack) + } + } else if (model instanceof List) { + model.each { def value -> + resolveModelReferences(value, components, modelVisitStack) + } + } + + if (modelNameStacked) { + modelVisitStack.pop() + } + + model + } + + List findPathFromRelationship(String nameToFind, def relationship, List ancestors = null) { + ancestors = ancestors ?: [] + List results = [] + relationship?.each { String path, def models -> + if (path == nameToFind) { + results << ancestors.join('.') + } else if (models instanceof Map) { + ancestors.add(path) + results.addAll(findPathFromRelationship(nameToFind, models, ancestors)) + ancestors.removeLast() + } + } + + results + } + + /** + * Check if protocol requires a plot + * @param protocol + * @return boolean + */ + boolean isPlotLayoutNeededByProtocol(Map protocol) { + List modelNames = protocol.attributes.workflow?.collect { it.modelName } + modelNames.contains(PARATOO_WORKFLOW_PLOT_LAYOUT) + } + + /** + * Find the models associated with a protocol. Endpoint is of the format Protocol's END_POINT_PREFIX/bulk. + * + * @param protocol + * @param documentation + * @return + */ + Map findProtocolEndpointDefinition (Map protocol, Map documentation) { + (Map) documentation.paths.findResult { String pathName, Map path -> + if (pathName == "${protocol.attributes.endpointPrefix}/bulk") { + return path.post.requestBody.content['application/json'].schema.properties + .data.properties.collections.items.properties + } + } + } + + Map getComponents(Map documentation) { + documentation.components.schemas + } + + Map resolveReferences(Map schema, Map components) { + String componentName + schema.each { String model, def value -> + if (!(value instanceof Map)) + return + + if (value['$ref']) { + componentName = getModelNameFromRef(value['$ref']) + if (components[componentName]) + value.putAll(components[componentName]) + else + log.debug("No component definition found for ${componentName}") + value.remove('$ref') + resolveReferences(value, components) + } else if (value.items) { + resolveReferences(value, components) + } else if (value.properties && value.properties.data && value.properties.data.properties) { + resolveReferences(value.properties.data, components) + } else if (value.anyOf) { + def definition = value.anyOf.find { + it['$ref'] !== null + } + + + value.putAll(definition ?: [:]) + value.remove('anyOf') + resolveReferences(value, components) + } else { + resolveReferences(value, components) + } + + } + + schema + } + + String getModelNameFromRef(String ref) { + ref.replace("#/components/schemas/", "") + } + + Map getActivityFormForComponent(String modelName) { + Map documentation = getParatooSwaggerDocumentation() + Map components = getComponents(documentation) + Map component = components[modelName] + + if (component) { + log.debug((component as JSON).toString(true)) + convertToDataModelAndViewModel(component, documentation, modelName) + } + } + + Map convertToDataModelAndViewModel(Map component, Map documentation, String name, List required = null, Deque modelVisitStack = null, int depth = 0, String path = "", ParatooProtocolConfig config = null) { + boolean modelNameStacked = false + Map model = [dataModel: [], viewModel: []], dataType, viewModel, template + modelVisitStack = modelVisitStack ?: new ArrayDeque() + String componentName, modelName = component[PARATOO_MODEL_REF] + + if (PARATOO_IGNORE_MODEL_LIST.contains(name)) + return + else if (component[PARATOO_SPECIES_TYPE]) { + component.type = "species" + } else if (modelName == PARATOO_FILE_MODEL_NAME) { + if (component[PARATOO_FILE_TYPE] == [PARATOO_IMAGE_FILE_TYPE]) { + component.type = "image" + } else { + component.type = "document" + } + } + + switch (component.type) { + case "object": + if (isLocationObject(component)) { + // complex object here represents a point with lat,lng attributes + dataType = getFeatureDataType(component, documentation, name) + viewModel = getFeatureViewModel(dataType, component, documentation, name) + } else { + template = getColumns(component, documentation, modelVisitStack, depth, path, config) + dataType = getListDataType(component, documentation, name, template.dataModel, true) + viewModel = getListViewModel(dataType, component, documentation, name, template.viewModel) + } + break + case "array": + template = getColumns(component, documentation, modelVisitStack, depth, path, config) + dataType = getListDataType(component, documentation, name, template.dataModel) + viewModel = getListViewModel(dataType, component, documentation, name, template.viewModel) + break + case "integer": + dataType = getIntegerDataType(component, documentation, name) + viewModel = getIntegerViewModel(dataType, component, documentation, name) + break + case "number": + dataType = getNumberDataType(component, documentation, name) + viewModel = getNumberViewModel(dataType, component, documentation, name) + break + case "string": + dataType = getStringDataType(component, documentation, name) + viewModel = getStringViewModel(dataType, component, documentation, name) + break + case "boolean": + dataType = getBooleanDataType(component, documentation, name) + viewModel = getBooleanViewModel(dataType, component, documentation, name) + break + case "document": + dataType = getDocumentDataType(component, documentation, name) + viewModel = getDocumentViewModel(dataType, component, documentation, name) + break + case "image": + dataType = getImageDataType(component, documentation, name) + viewModel = getImageViewModel(dataType, component, documentation, name) + break + case "species": + dataType = getSpeciesDataType(component, documentation, name) + viewModel = getSpeciesViewModel(dataType, component, documentation, name) + break + default: + log.error("Cannot convert Paratoo component to dataModel - ${component.type}") + break + } + + addOverrides(dataType, viewModel, path, config) + + if (dataType) { + addRequiredFlag(component, dataType, required) + model.dataModel.add(dataType) + } + + if (viewModel) { + model.viewModel.add(viewModel) + } + + if (modelNameStacked) { + modelVisitStack.pop() + } + + model + } + + Map addOverrides(Map dataModel, Map viewModel, String path, ParatooProtocolConfig config) { + Map overrides = config.overrides + if (overrides?.dataModel?.containsKey(path)) { + Map override = overrides?.dataModel[path] + if (override) { + dataModel.putAll(override) + } + } + + if (overrides?.viewModel?.containsKey(path)) { + Map override = overrides?.viewModel[path] + if (override) { + viewModel.putAll(override) + } + } + } + + static boolean isLocationObject(Map input) { + input[PARATOO_COMPONENT] == PARATOO_LOCATION_COMPONENT + } + + static Map addRequiredFlag(Map component, Map dataType, List required) { + if (required?.contains(component.name)) { + dataType["validate"] = "required" + } + + dataType + } + + Map getStringDataType(Map component, Map documentation, String name) { + Map dataType + switch (component.format) { + case "date-time": + dataType = addUnitsAndDescription([ + "dataType" : "date", + "name" : name, + "dwcAttribute": "eventDate" + ], component) + break + default: + dataType = addUnitsAndDescription([ + "dataType": "text", + "name" : name + ], component) + + if (component[PARATOO_LUT_REF]) { + List items = getLutValues(component[PARATOO_LUT_REF]) + if (items) + dataType << transFormLutValuesToDataModel(items) + else + dataType.constraints = component.enum + dataType[PARATOO_LUT_REF] = component[PARATOO_LUT_REF] + } + break + } + + dataType + } + + Map transFormLutValuesToDataModel(List items) { + Map dataModel = [ + constraints : [ + "textProperty" : "label", + "type" : "literal", + "valueProperty": "value" + ], + "displayOptions": [ + "placeholder": "Select an option", + "tags" : true + ] + ] + + dataModel.constraints.literal = items.collect { Map item -> + [label: item.attributes.label, value: item.attributes.symbol] + } + + dataModel + } + + /** + * Get display names for symbols. Swagger documentation does not have display names but only list of symbols. + * @param lutRef + * @return [ + * [ + * "id": 1, + * "attributes": [ + * "symbol": "E", + * "label": "Estuary", + * "description": "", + * "uri": "", + * "createdAt": "2024-03-05T07:21:46.070Z", + * "updatedAt": "2024-03-12T07:30:12.903Z" + * ] + * ] + * ] + */ + List getLutValues(String lutRef) { + cacheService.get("paratoo-$lutRef", { + String url = "${getParatooBaseUrl()}/${getPluralizedName(lutRef)}" + int start = 0 + int limit = 20 + String query = "?" + buildPaginationQuery(start, limit) + Map authHeader = getAuthHeader() + Map response = webService.getJson(url + query, null, authHeader, false) + List items = response.data ?: [] + if (!response.error) { + int total = response.meta?.pagination?.total ?: 0 + while (items && start + limit < total) { + start += limit + query = "?" + buildPaginationQuery(start, limit) + response = webService.getJson(url + query, null, authHeader, false) + if (!response.error) { + items.addAll(response.data) + } + } + } + + items + }) + } + + String buildPaginationQuery(int start, int limit) { + "pagination[start]=$start&pagination[limit]=$limit" + } + + /** + * Get pluralized name of an enumeration. This is used to get the correct endpoint to get display name of an enumeration. + * @param name + * @return + */ + String getPluralizedName(String name) { + Map documentation = getParatooSwaggerDocumentation() + // strapi uses plural names for endpoints + List suffixes = ["s", "es", "ies"] + String winner = name + suffixes.each { String suffix -> + String tempName + if (suffix.equals("ies") && name.endsWith("y")) { + // Apply special rule for words ending with 'y' + tempName = name.substring(0, name.length() - 1) + "ies" + } else { + tempName = "${name}${suffix}" + } + + // check if the endpoint exists in swagger documentation to decide if the plural value is correct. + if (documentation.paths["/${tempName}"] != null) + winner = tempName + } + + winner + } + + /** + * Get model definitions according to model relationship so that parent will come first and children will come under it. + * If relationship is [b:[a:[:]] then rearranged output will be + * [ type: "object", + * properties: [ + * b: [ + * type: "object", + * properties: [ + * a: [...] + * ] + * ] + * ] + * ] + * @param properties + * @param apiOutput + * @param relationship + * @param newOrder + * @return + */ + Map rearrangePropertiesAccordingToModelRelationship(properties, apiOutput, relationship, Map newOrder = null) { + newOrder = newOrder ?: [type: "object", properties: [:]] + if (relationship instanceof Map) { + relationship.each { String parent, Map children -> + List paths = findPathFromRelationship(parent, apiOutput) + String path = paths.size() ? paths.first() : null + if (path) { + newOrder["type"] = "object" + // get model definition for the parent + def value = getProperty(properties, path) + // remove parent from children + paths?.each { String propertyPath -> + removeProperty(properties, propertyPath) + } + + value = deepCopy(value) + // reorder + newOrder.properties = newOrder.properties ?: [:] + newOrder.properties[parent] = newOrder.properties[parent] ?: [:] + newOrder.properties[parent].putAll(value ?: [:]) + removeProperty(properties, path) + } else { + // if path is not found, then check if parent is in root model + newOrder.properties = newOrder.properties ?: [:] + newOrder.properties[parent] = newOrder.properties[parent] ?: [:] + properties = properties ?: [:] + newOrder.properties[parent].putAll((properties[parent] ?: [:])) + } + +// paths?.each { String propertyPath -> +// removeProperty(properties, propertyPath) +// } + + if (children) { + // if children are present, then recurse the process through each children + newOrder.properties[parent] = newOrder.properties[parent] ?: [type: "object", properties: [:]] + newOrder.properties[parent].properties.putAll(rearrangePropertiesAccordingToModelRelationship(properties, apiOutput, children, newOrder.properties[parent]).properties) + } + } + } + + newOrder + } + + /** + * Remove a property at a given path. + * i.e. if path is a.b.c and object is {a: {b: {c: [:]}}} then after removing the property, properties will be {a: {b: [:]}} + * @param object + * @param key + * @param parts + */ + void removeProperty(def object, String key, List parts = null) { + parts = parts ?: key.split(/\./) + String part = parts.remove(0) + if (parts.size() == 0) { + if (object instanceof Map) + object.remove(part) + else if (object instanceof List) { + object.each { def item -> + item.remove(part) + } + } + } else if (object instanceof Map) + removeProperty(object[part], key, parts) + else if (object instanceof List) { + object.each { def item -> + removeProperty(item[part], key, parts.clone()) + } + } + } + + Map getStringViewModel(Map dataModel, Map component, Map documentation, String name) { + Map viewModel + switch (component.format) { + case "date-time": + viewModel = addLabel([ + "type" : "date", + "source": name + ], component, name) + break + default: + if (dataModel.constraints) { + viewModel = addLabel([ + "type" : "selectOne", + "source": name + ], component, name) + } else { + viewModel = addLabel([ + "type" : "text", + "source": name + ], component, name) + } + + break + } + + viewModel + } + + Map getTimeDataType(Map component, Map documentation, String name) { + addUnitsAndDescription([ + "dataType" : "time", + "name" : name, + "dwcAttribute": "eventTime" + ], component) + } + + Map getTimeViewModel(Map dataModel, Map component, Map documentation, String name) { + addUnitsAndDescription([ + "dataType" : "time", + "name" : name, + "dwcAttribute": "eventTime" + ], component) + } + + Map getListDataType(Map component, Map documentation, String name, List columns, Boolean isObject = false) { + addUnitsAndDescription([ + "dataType": "list", + "name" : name, + "isObject": isObject, + "columns" : columns + ], component) + } + + Map getListViewModel(Map dataModel, Map component, Map documentation, String name, List columns) { + Map viewModel = addLabel([ + type : "section", + title: getLabel(component, name), + boxed: true, + items: [ + type : "repeat", + source: dataModel.name, + items : [ + type : "row", + "class": "output-section", + items : [ + type : "col", + items: columns + ] + ] + ] + ], component, name) + + if (dataModel.isObject) { + viewModel.items.userAddedRows = false + } + + viewModel + } + + Map getSpeciesDataType(Map component, Map documentation, String name) { + addUnitsAndDescription([ + "dataType" : "species", + "name" : name, + "dwcAttribute": "scientificName" + ], component) + } + + Map getSpeciesViewModel(Map dataModel, Map component, Map documentation, String name) { + addLabel([ + type : "speciesSelect", + source: dataModel.name + ], component, name) + } + + Map getImageDataType(Map component, Map documentation, String name) { + addUnitsAndDescription([ + "dataType": "image", + "name" : name + ], component) + } + + Map getImageViewModel(Map dataModel, Map component, Map documentation, String name) { + addLabel([ + type : "image", + source: dataModel.name + ], component, name) + } + + Map getDocumentDataType(Map component, Map documentation, String name) { + addUnitsAndDescription([ + "dataType": "boolean", + "name" : name + ], component) + } + + Map getDocumentViewModel(Map dataModel, Map component, Map documentation, String name) { + addLabel([ + type : "document", + source: dataModel.name + ], component, name) + } + + Map getBooleanDataType(Map component, Map documentation, String name) { + addUnitsAndDescription([ + "dataType": "boolean", + "name" : name + ], component) + } + + Map getBooleanViewModel(Map dataModel, Map component, Map documentation, String name) { + addLabel([ + type : "boolean", + source: dataModel.name + ], component, name) + } + + Map getFeatureDataType(Map component, Map documentation, String name) { + addUnitsAndDescription([ + "dataType": "feature", + "name" : name + ], component) + } + + Map getFeatureViewModel(Map dataModel, Map component, Map documentation, String name) { + addLabel([ + type : "feature", + source: dataModel.name + ], component, name) + } + + Map getIntegerDataType(Map component, Map documentation, String name) { + addUnitsAndDescription([ + "dataType" : "number", + "name" : name, + "decimalPlaces": 0 + ], component) + } + + Map getIntegerViewModel(Map dataModel, Map component, Map documentation, String name) { + addLabel([ + type : "number", + source: dataModel.name + ], component, name) + } + + Map getNumberDataType(Map component, Map documentation, String name) { + addUnitsAndDescription([ + "dataType" : "number", + "name" : name, + "decimalPlaces": PARATOO_FLOAT_DECIMAL_PLACES + ], component) + } + + Map getNumberViewModel(Map dataModel, Map component, Map documentation, String name) { + addLabel([ + type : "number", + source: dataModel.name + ], component, name) + } + + Map addLabel(Map viewModel, Map component, String name) { + viewModel.preLabel = getLabel(component, name) + viewModel + } + + /** + * Get label from swagger definition or from property name + * @param component + * @param name + * @return + */ + String getLabel(Map component, String name) { + String label + if (component[PARATOO_FIELD_LABEL]) { + label = component[PARATOO_FIELD_LABEL] + } else { + label = getLabelFromPropertyName(name) + } + + label + } + + /** + * Get label from property name. + * e.g. "bird-survey" will be converted to "Bird Survey" + * @param name + * @return + */ + String getLabelFromPropertyName(String name) { + String out = '' + out = name.replaceAll(/(-|_)/, '-') + .split('-') + .findAll { it } + .collect { it.capitalize() } + .join(' ') + + return out.trim() + } + + Map addUnitsAndDescription(Map dataType, Map component) { + if (component[PARATOO_UNIT_FIELD_NAME]) { + dataType.units = component[PARATOO_UNIT_FIELD_NAME] + } + + if (component[PARATOO_HINT]) { + dataType.description = component[PARATOO_HINT] + } + + dataType + } + + Map getColumns(Map component, Map documentation, Deque modelVisitStack, int depth = 0, String path, ParatooProtocolConfig config = null) { + Map template = [dataModel: [], viewModel: []] + Map properties = getModelStructureFromDefinition(component) + List required = getRequiredModels(component) + properties?.each { String name, def model -> + if (!model) + return + + if (PARATOO_IGNORE_MODEL_LIST.contains(name)) + return + + if(model instanceof Map) { + Map sections = convertToDataModelAndViewModel(model, documentation, name, required, modelVisitStack, depth, "${path}.${name}", config) + if (sections?.dataModel) + template.dataModel.addAll(sections.dataModel) + + if (sections?.viewModel) + template.viewModel.addAll(sections.viewModel) + } + } + + template + } + + /** + * Converts a model name to a name used in swagger documentation. + * e.g. a-model-name will be converted to AModelNameRequest + * @param modelName + * @return + */ + String getSchemaNameFromModelName(String modelName) { + capitalizeModelName(modelName) + PARATOO_SCHEMA_POSTFIX + } + + /** + * Capitalize a model name. + * e.g. a-model-name will be converted to AModelName + * @param modelName + * @return + */ + String capitalizeModelName(String modelName) { + modelName?.toLowerCase()?.replaceAll("[^a-zA-Z0-9]+", ' ')?.tokenize(' ')?.collect { it.capitalize() }?.join() + } + + private static def getProperty(Map surveyData, String path) { + if (!path) { + return null + } + new PropertyAccessor(path).get(surveyData) + } + + /** + * Transforms a species name to species object used by ecodata. + * e.g. Acacia glauca [Species] (scientific: Acacia glauca Willd.) + * [name: "Acacia glauca Willd.", scientificName: "Acacia glauca Willd.", guid: "A_GUID"] + * Guid is necessary to generate species occurrence record. Guid is found by searching the species name with BIE. If not found, then a default value is added. + * @param name + * @return + */ + Map transformSpeciesName(String name) { + String regex = "\\(scientific:\\s*(.*?)\\)" + Pattern pattern = Pattern.compile(regex) + Matcher matcher = pattern.matcher(name) + Map result = [name: name, scientificName: name, outputSpeciesId: UUID.randomUUID().toString()] + + if (matcher.find()) { + name = matcher.group(1) + result.scientificName = result.name = name + } + + metadataService.autoPopulateSpeciesData(result) + // record is only created if guid is present + result.guid = result.guid ?: "A_GUID" + result + } } + diff --git a/grails-app/services/au/org/ala/ecodata/WebService.groovy b/grails-app/services/au/org/ala/ecodata/WebService.groovy index 8bf425498..6f02db8dd 100644 --- a/grails-app/services/au/org/ala/ecodata/WebService.groovy +++ b/grails-app/services/au/org/ala/ecodata/WebService.groovy @@ -227,7 +227,7 @@ class WebService { } } - Map doPost(String url, Map postBody) { + Map doPost(String url, Map postBody, boolean addUserId = true, Map headers = null) { HttpURLConnection conn = null def charEncoding = 'utf-8' try { @@ -237,11 +237,20 @@ class WebService { conn.setRequestProperty("Content-Type", "application/json;charset=${charEncoding}"); conn.setRequestProperty("Authorization", "${grailsApplication.config.getProperty('api_key')}"); - def user = getUserService().getCurrentUserDetails() - if (user && user.userId) { - conn.setRequestProperty(grailsApplication.config.getProperty('app.http.header.userId'), user.userId) - conn.setRequestProperty("Cookie", "ALA-Auth="+java.net.URLEncoder.encode(user.userName, charEncoding)) + if (addUserId) { + def user = getUserService().getCurrentUserDetails() + if (user && user.userId) { + conn.setRequestProperty(grailsApplication.config.getProperty('app.http.header.userId'), user.userId) + conn.setRequestProperty("Cookie", "ALA-Auth=" + java.net.URLEncoder.encode(user.userName, charEncoding)) + } } + + if (headers) { + headers.each {key, value -> + conn.setRequestProperty(key, value) + } + } + OutputStreamWriter wr = new OutputStreamWriter(conn.getOutputStream(), charEncoding) wr.write((postBody as JSON).toString()) wr.flush() diff --git a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy index 72d89ec59..a299e4a8f 100644 --- a/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy +++ b/src/main/groovy/au/org/ala/ecodata/GeometryUtils.groovy @@ -1,22 +1,22 @@ package au.org.ala.ecodata -import org.locationtech.jts.geom.* -import org.locationtech.jts.io.WKTReader -import org.locationtech.jts.io.WKTWriter -import org.locationtech.jts.simplify.TopologyPreservingSimplifier -import org.locationtech.jts.util.GeometricShapeFactory import grails.converters.JSON import org.apache.commons.logging.Log import org.apache.commons.logging.LogFactory import org.geotools.geojson.geom.GeometryJSON import org.geotools.geometry.jts.JTS +import org.geotools.geometry.jts.JTSFactoryFinder import org.geotools.referencing.CRS import org.geotools.referencing.GeodeticCalculator +import org.locationtech.jts.geom.* +import org.locationtech.jts.io.WKTReader +import org.locationtech.jts.io.WKTWriter +import org.locationtech.jts.simplify.TopologyPreservingSimplifier +import org.locationtech.jts.util.GeometricShapeFactory import org.opengis.referencing.crs.CoordinateReferenceSystem import org.opengis.referencing.operation.MathTransform import java.awt.geom.Point2D - /** * Helper class for working with site geometry. */ @@ -39,6 +39,19 @@ class GeometryUtils { return result } + static Geometry getFeatureCollectionConvexHull (List features) { + // Extract geometries from features + List geometries = [] + features.each { feature -> geometries.add( geoJsonMapToGeometry(feature) ) } + + GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory() + GeometryCollection geometryCollection = geometryFactory.createGeometryCollection(geometries.toArray(new Geometry[0])) + Geometry unionGeometry = geometryCollection.union() + + // Compute convex hull + unionGeometry.convexHull() + } + private static MultiPolygon convertToMultiPolygon(Geometry geom) { MultiPolygon result switch (geom.geometryType) { diff --git a/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy new file mode 100644 index 000000000..03165ef20 --- /dev/null +++ b/src/main/groovy/au/org/ala/ecodata/converter/FeatureConverter.groovy @@ -0,0 +1,47 @@ +package au.org.ala.ecodata.converter + +class FeatureConverter implements RecordFieldConverter { + + List convert(Map data, Map metadata = [:]) { + Map record = [:] + + + Double latitude = getDecimalLatitude(data[metadata.name]) + Double longitude = getDecimalLongitude(data[metadata.name]) + + // Don't override decimalLongitud or decimalLatitude in case they are null, site info could've already set them + if(latitude != null) { + record.decimalLatitude = latitude + } + + if (longitude != null) { + record.decimalLongitude = longitude + } + + + Map dwcMappings = extractDwcMapping(metadata) + + record << getDwcAttributes(data, dwcMappings) + + if (data.dwcAttribute) { + record[data.dwcAttribute] = data.value + } + + [record] + } + + static Double getDecimalLatitude (Map data) { + switch (data.type) { + case 'Point': + return data.coordinates[1] + } + } + + static Double getDecimalLongitude (Map data) { + switch (data.type) { + case 'Point': + return data.coordinates[0] + } + + } +} diff --git a/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy index a4d24c835..8603d1203 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy @@ -14,6 +14,12 @@ class ListConverter implements RecordFieldConverter { // delegate the conversion of each column in each row to a specific converter for the column type data[outputMetadata.name].each { row -> Map baseRecord = [:] + List singleItemModels + List multiItemModels + (singleItemModels, multiItemModels) = outputMetadata?.columns?.split { + //check if dataType is null + !RecordConverter.MULTI_ITEM_DATA_TYPES.contains(it.dataType?.toLowerCase()) + } //Use the same approach as au.org.ala.ecodata.converter.RecordConverter.convertRecords() to convert // singleItemModels, ie, iterate over the datatype definitions rather than the values (data) @@ -26,7 +32,7 @@ class ListConverter implements RecordFieldConverter { // All records will share the same base record data List baseRecordModels List speciesModels - (baseRecordModels, speciesModels) = outputMetadata?.columns?.split { + (baseRecordModels, speciesModels) = singleItemModels?.split { it.dataType.toLowerCase() != "species" } @@ -36,7 +42,8 @@ class ListConverter implements RecordFieldConverter { baseRecordModels?.each { Map dataModel -> RecordFieldConverter converter = RecordConverter.getFieldConverter(dataModel.dataType) List recordFieldSets = converter.convert(row, dataModel) - baseRecord << recordFieldSets[0] + if (recordFieldSets[0]) + baseRecord << recordFieldSets[0] } // For each species dataType, where present we will generate a new record @@ -54,6 +61,26 @@ class ListConverter implements RecordFieldConverter { "This is most likely a bug.") } } + + if (multiItemModels) { + // For each multiItemModel, get the appropriate field converter for the data type and generate the list of field + // sets which will be converted into Records. For each field set, add a copy of the skeleton Record so it has + // all the common fields + multiItemModels?.each { Map dataModel -> + RecordFieldConverter converter = RecordConverter.getFieldConverter(dataModel.dataType) + List recordFieldSets = converter.convert(row, dataModel) + + recordFieldSets.each { + Map rowRecord = RecordConverter.overrideFieldValues(baseRecord, it) + if(rowRecord.guid && rowRecord.guid != "") { + records << rowRecord + } else { + log.warn("Multi item Record [${rowRecord}] does not contain species information, " + + "was the form intended to work like that?") + } + } + } + } } records diff --git a/src/main/groovy/au/org/ala/ecodata/metadata/OutputMetadata.groovy b/src/main/groovy/au/org/ala/ecodata/metadata/OutputMetadata.groovy index 16fb4931b..662f696f5 100644 --- a/src/main/groovy/au/org/ala/ecodata/metadata/OutputMetadata.groovy +++ b/src/main/groovy/au/org/ala/ecodata/metadata/OutputMetadata.groovy @@ -241,7 +241,7 @@ class OutputMetadata { } def isNestedDataModelType(node) { - return (node.columns != null && node.dataType != "geoMap") + return ((node.columns != null) && (node.columns.size() != 0) && node.dataType != "geoMap") } def isNestedViewModelType(node) { return (node.items != null || node.columns != null) @@ -256,7 +256,7 @@ class OutputMetadata { * ] * */ - Map getNamesForDataType(String type, context){ + Map getNamesForDataType(String type, context, int depth = 0, String path = ""){ Map names = [:], childrenNames if(!context && metadata){ @@ -265,8 +265,10 @@ class OutputMetadata { context?.each { data -> if(isNestedDataModelType(data)){ + String contextPath = "${path}.${data.name}" + log.info("${contextPath}") // recursive call for nested data model - childrenNames = getNamesForDataType(type, getNestedDataModelNodes(data)); + childrenNames = getNamesForDataType(type, getNestedDataModelNodes(data), depth + 1, contextPath); if(childrenNames?.size()){ names[data.name] = childrenNames } diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollection.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollection.groovy index fff87f0cd..8eca23482 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollection.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollection.groovy @@ -1,5 +1,6 @@ package au.org.ala.ecodata.paratoo +import grails.converters.JSON import grails.databinding.BindingFormat import grails.validation.Validateable import com.fasterxml.jackson.annotation.JsonIgnoreProperties @@ -20,4 +21,8 @@ class ParatooCollection implements Validateable { userId nullable: true eventTime nullable: true } + + Map parseOrgMintedIdentifier() { + JSON.parse(new String(orgMintedIdentifier.decodeBase64())) + } } diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 5c8a40347..0c6df8b7f 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -1,12 +1,17 @@ package au.org.ala.ecodata.paratoo +import au.org.ala.ecodata.ActivityForm import au.org.ala.ecodata.DateUtil +import au.org.ala.ecodata.FormSection +import au.org.ala.ecodata.GeometryUtils +import au.org.ala.ecodata.ParatooService +import au.org.ala.ecodata.metadata.OutputMetadata import au.org.ala.ecodata.metadata.PropertyAccessor -import groovy.util.logging.Slf4j import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import groovy.util.logging.Slf4j +import org.locationtech.jts.geom.Geometry import java.util.regex.Matcher - /** * Configuration about how to work with a Paratoo/Monitor protocol */ @@ -32,6 +37,8 @@ class ParatooProtocolConfig { String observationIndividualCountPath = 'attributes.number_of_individuals' String observationEventDatePath = 'attributes.date_time' String observationGeometryPath = 'attributes.location' + String plotVisitPath = 'attributes.plot_visit.data.attributes' + String plotLayoutPath = "${plotVisitPath}.plot_layout.data.attributes" String plotLayoutIdPath = 'attributes.plot_visit.data.attributes.plot_layout.data.id' String plotLayoutPointsPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_points' String plotSelectionPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_selection.data.attributes' @@ -40,6 +47,7 @@ class ParatooProtocolConfig { String getApiEndpoint(ParatooSurveyId surveyId) { apiEndpoint ?: defaultEndpoint(surveyId) } + Map overrides = [dataModel: [:], viewModel: [:]] private static String removeMilliseconds(String isoDateWithMillis) { if (!isoDateWithMillis) { @@ -134,6 +142,42 @@ class ParatooProtocolConfig { extractGeometryFromSiteData(geometryData) } + private List extractFeatures (Map observation, ActivityForm form) { + List features = [] + form.sections.each { FormSection section -> + OutputMetadata om = new OutputMetadata(section.template) + Map paths = om.getNamesForDataType("feature", null ) + features.addAll(getFeaturesFromPath(observation, paths)) + } + + features + } + + private List getFeaturesFromPath (Map output, Map paths) { + List features = [] + paths.each { String name, node -> + if (node instanceof Boolean) { + features.add(output[name]) + // todo later: add featureIds and modelId for compliance with feature behaviour of reports + } + + // recursive check for feature + if (node instanceof Map) { + if (output[name] instanceof Map) { + features.addAll(getFeaturesFromPath(output[name], node)) + } + + if (output[name] instanceof List) { + output[name].eachWithIndex { row, index -> + features.addAll(getFeaturesFromPath(row, node)) + } + } + } + } + + features + } + private Map extractGeometryFromSiteData(geometryData) { Map geometry = null if (geometryData) { @@ -157,14 +201,14 @@ class ParatooProtocolConfig { apiEndpoint } - private static def getProperty(Map surveyData, String path) { + static def getProperty(Map surveyData, String path) { if (!path) { return null } new PropertyAccessor(path).get(surveyData) } - Map getGeoJson(Map survey) { + Map getGeoJson(Map survey, Map observation = null, ActivityForm form = null) { if (!survey) { return null } @@ -172,13 +216,56 @@ class ParatooProtocolConfig { Map geoJson = null if (usesPlotLayout) { geoJson = extractSiteDataFromPlotVisit(survey) + // get list of all features associated with observation + if (form && observation) { + geoJson.features = extractFeatures(observation, form) + } } - else if (geometryPath) { - geoJson = extractSiteDataFromPath(survey) + else if (form && observation) { + List features = extractFeatures(observation, form) + if (features) { + Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(features) + geoJson = GeometryUtils.geometryToGeoJsonMap(geometry) + geoJson.features = features + } } geoJson } + Map getPlotVisit (Map surveyData) { + Map plotVisit = getProperty(surveyData, plotVisitPath) + List keys = plotVisit?.keySet().toList().minus(ParatooService.PARATOO_DATAMODEL_PLOT_LAYOUT) + Map result = [:] + keys.each { key -> + result[key] = plotVisit[key] + } + + result + } + + Map getPlotLayout (Map surveyData) { + Map plotLayout = getProperty(surveyData, plotLayoutPath) + List keys = plotLayout?.keySet().toList().minus(ParatooService.PARATOO_DATAMODEL_PLOT_SELECTION) + Map result = [:] + keys.each { key -> + result[key] = plotLayout[key] + } + + result + } + + Map getPlotSelection (Map surveyData) { + Map plotSelection = getProperty(surveyData, plotSelectionPath) + List keys = plotSelection?.keySet().toList() + Map result = [:] + keys.each { key -> + result[key] = plotSelection[key] + } + + result + } + + boolean matches(Map surveyData, ParatooSurveyId surveyId) { Map tmpSurveyId = getSurveyId(surveyData) surveyEqualityTest(tmpSurveyId, surveyId) @@ -238,6 +325,20 @@ class ParatooProtocolConfig { plotGeometry } + Map getSurveyDataFromObservation(Map observation) { + String surveyAttribute = apiEndpoint + if(surveyAttribute?.endsWith('s')) { + surveyAttribute = surveyAttribute.substring(0, surveyAttribute.length() - 1) + } + + def survey = observation[surveyAttribute] + if (survey instanceof List) { + return survey[0] + } + + survey + } + private static List closePolygonIfRequired(List points) { if (points[0][0] != points[-1][0] || points[0][1] != points[-1][1]) { points << points[0] diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 0f661525a..088d81b67 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -7,6 +7,7 @@ import grails.converters.JSON import grails.test.mongodb.MongoSpec import grails.testing.services.ServiceUnitTest import groovy.json.JsonSlurper +import org.codehaus.jackson.map.ObjectMapper import org.grails.web.converters.marshaller.json.CollectionMarshaller import org.grails.web.converters.marshaller.json.MapMarshaller @@ -14,7 +15,7 @@ import org.grails.web.converters.marshaller.json.MapMarshaller * Tests for the ParatooService. * The tests are incomplete as some of the behaviour needs to be specified. */ -class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest{ +class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest { String userId = 'u1' SiteService siteService = Mock(SiteService) @@ -23,8 +24,10 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> DUMMY_POLYGON + 1 * siteService.geometryAsGeoJson({ it.siteId == 's1' }) >> DUMMY_POLYGON } @@ -86,7 +92,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest projects = service.userProjects(userId) @@ -95,8 +101,8 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest projects = service.userProjects(userId) + when: + List projects = service.userProjects(userId) - then: - projects.size() == 1 - projects[0].accessLevel == AccessLevel.admin + then: + projects.size() == 1 + projects[0].accessLevel == AccessLevel.admin } @@ -122,15 +128,15 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> {data, pId, updateCollectory -> + 1 * projectService.update(_, projectId, false) >> { data, pId, updateCollectory -> Map dataSet = data.custom.dataSets[1] // The stubbed project already has a dataSet, so the new one will be index=1 assert dataSet.surveyId.time == surveyId.timeAsISOString() assert dataSet.surveyId.uuid == surveyId.uuid @@ -140,7 +146,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest= 0}, null, _, false) >> [data:[], meta:[pagination:[total:0]]] - 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) - 1 * projectService.update([custom:[dataSets:[expectedDataSet]]], 'p1', false) >> [status:'ok'] + 1 * webService.getJson({ it.indexOf('/s1s') >= 0 }, null, _, false) >> [data: [], meta: [pagination: [total: 0]]] + 1 * webService.doPost(*_) >> [resp: [collections: [s1: [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z"]]]] + 2 * tokenService.getAuthToken(true) >> Mock(AccessToken) + 1 * projectService.update([custom: [dataSets: [expectedDataSet]]], 'p1', false) >> [status: 'ok'] + 1 * activityService.create(_) >> [activityId: '123'] + 1 * recordService.getAllByActivity('123') >> [] + 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { + (["1": [ + "name" : "Opportune", + "usesPlotLayout": false, + "tags" : ["survey"], + "apiEndpoint" : "s1s", + "overrides" : [ + "dataModel": null, + "viewModel": null + ] + ]] as JSON).toString() + } and: - result == [status:'ok'] + result == [status: 'ok'] } - + void "The service can create a site from a submitted plot-selection"() { setup: Map data = [ - "plot_label":"CTMAUA2222", - "recommended_location":["lat":-35.2592424,"lng":149.0651439], - "uuid":"lmpisy5p9g896lad4ut", - "comment":"Test"] + "plot_label" : "CTMAUA2222", + "recommended_location": ["lat": -35.2592424, "lng": 149.0651439], + "uuid" : "lmpisy5p9g896lad4ut", + "comment" : "Test"] - Map expected = ['name':'CTMAUA2222', 'description':'CTMAUA2222', publicationStatus:'published', 'externalIds':[new ExternalId(externalId:'lmpisy5p9g896lad4ut', idType:ExternalId.IdType.MONITOR_PLOT_GUID)], 'notes':'Test', 'extent':['geometry':['type':'Point', 'coordinates':[149.0651439, -35.2592424], 'decimalLatitude':-35.2592424, 'decimalLongitude':149.0651439], 'source':'point'], 'projects':[], 'type':'surveyArea'] + Map expected = ['name': 'CTMAUA2222', 'description': 'CTMAUA2222', publicationStatus: 'published', 'externalIds': [new ExternalId(externalId: 'lmpisy5p9g896lad4ut', idType: ExternalId.IdType.MONITOR_PLOT_GUID)], 'notes': 'Test', 'extent': ['geometry': ['type': 'Point', 'coordinates': [149.0651439, -35.2592424], 'decimalLatitude': -35.2592424, 'decimalLongitude': 149.0651439], 'source': 'point'], 'projects': [], 'type': 'surveyArea'] String userId = 'u1' @@ -190,8 +211,8 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest= 0}, null, _, false) >> [data:[surveyData], meta:[pagination:[total:0]]] - 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) - 1 * projectService.update(_, projectId, false) >> [status:'ok'] - 1 * siteService.create(_) >> {site = it[0]; [siteId:'s1']} + 1 * webService.getJson({ it.indexOf('/basal-area-dbh-measure-survey') >= 0 }, null, _, false) >> [data: [surveyData], meta: [pagination: [total: 0]]] + 1 * webService.doPost(*_) >> [resp: [collections: ["basal-area-dbh-measure-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z"]]]] + 2 * tokenService.getAuthToken(true) >> Mock(AccessToken) + 1 * projectService.update(_, projectId, false) >> [status: 'ok'] + 1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] } + 1 * activityService.create(_) >> [activityId: '123'] + 1 * recordService.getAllByActivity('123') >> [] + 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { + (["1": [ + "name" : "Basal Area - DBH", + "usesPlotLayout": true, + "tags" : ["survey"], + "apiEndpoint" : "basal-area-dbh-measure-surveys", + "overrides" : [ + "dataModel": null, + "viewModel": null + ] + ]] as JSON).toString() + } and: site.name == "SATFLB0001 - Control (100 x 100)" @@ -257,47 +293,791 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [projectArea, plot] - Program program = new Program(programId: "prog1", name:"A program", config:[(ParatooService.PROGRAM_CONFIG_PARATOO_ITEM):true]) - program.save(failOnError:true, flush:true) + Program program = new Program(programId: "prog1", name: "A program", config: [(ParatooService.PROGRAM_CONFIG_PARATOO_ITEM): true]) + program.save(failOnError: true, flush: true) + + Service service = new Service(name: "S1", serviceId: '1', legacyId: 1, outputs: [new ServiceForm(externalId: "guid-2", formName: "aParatooForm", sectionName: null)]) + service.save(failOnError: true, flush: true) + + ActivityForm activityForm = new ActivityForm(name: "aParatooForm 1", type: 'EMSA', category: 'protocol category 1', external: true) + activityForm.externalIds = [new ExternalId(externalId: "guid-2", idType: ExternalId.IdType.MONITOR_PROTOCOL_GUID)] + activityForm.save(failOnError: true, flush: true) + + activityForm = new ActivityForm(name: "aParatooForm 2 ", type: 'EMSA', category: 'protocol category 2', external: true) + activityForm.externalIds = [new ExternalId(externalId: "guid-3", idType: ExternalId.IdType.MONITOR_PROTOCOL_GUID)] + activityForm.save(failOnError: true, flush: true) + + activityForm = new ActivityForm(name: "aParatooForm 3", type: 'EMSA', category: 'protocol category 3', external: true) + activityForm.externalIds = [new ExternalId(externalId: "guid-4", idType: ExternalId.IdType.MONITOR_PROTOCOL_GUID)] + activityForm.save(failOnError: true, flush: true) + + activityForm = new ActivityForm(name: "aParatooForm 4", type: 'EMSA', category: 'protocol category 4', external: true, + sections: [ + new FormSection(name: "section 1", type: "section", template: [ + dataModel : [ + [ + dataType: "list", + name : "bird-survey", + columns : [ + [ + dataType: "integer", + name : "bird-observation" + ] + ] + ] + ], + viewModel : [], + relationships: [ + ecodata : ["bird-survey": ["bird-observation": [:]]], + apiOutput: ["bird-survey.bird-observation": ["bird-observation": [:]]] + ] + ] + ) + ] + ) + activityForm.externalIds = [new ExternalId(externalId: "1", idType: ExternalId.IdType.MONITOR_PROTOCOL_GUID)] + activityForm.save(failOnError: true, flush: true) + + } + + void "capitalizeModelName should convert hyphenated name to capitalised name"() { + when: + String result = service.capitalizeModelName("paratoo-protocol") + + then: + result == "ParatooProtocol" + + when: + result = service.capitalizeModelName("paratoo-protocol-id") + + then: + result == "ParatooProtocolId" + + when: + result = service.capitalizeModelName(null) + + then: + result == null + + when: + result = service.capitalizeModelName("") - Service service = new Service(name:"S1", serviceId:'1', legacyId: 1, outputs:[new ServiceForm(externalId:"guid-2", formName:"aParatooForm", sectionName:null)]) - service.save(failOnError:true, flush:true) + then: + result == "" + } - ActivityForm activityForm = new ActivityForm(name:"aParatooForm 1", type:'EMSA', category:'protocol category 1', external: true) - activityForm.externalIds = [new ExternalId(externalId: "guid-2", idType:ExternalId.IdType.MONITOR_PROTOCOL_GUID)] - activityForm.save(failOnError:true, flush:true) + void "transformSpeciesName should convert paratoo species name to object correctly"() { + when: + Map result = service.transformSpeciesName("Acacia glauca [Species] (scientific: Acacia glauca Willd.)") + String outputSpeciesId = result.remove("outputSpeciesId") + then: + outputSpeciesId != null + result == [name: "Acacia glauca Willd.", scientificName: "Acacia glauca Willd.", guid: "A_GUID"] + 1 * metadataService.autoPopulateSpeciesData(_) >> null + } - activityForm = new ActivityForm(name:"aParatooForm 2 ", type:'EMSA', category:'protocol category 2', external: true) - activityForm.externalIds = [new ExternalId(externalId: "guid-3", idType:ExternalId.IdType.MONITOR_PROTOCOL_GUID)] - activityForm.save(failOnError:true, flush:true) + void "buildRelationshipTree should build relationship tree correctly"() { + given: + def properties = [ + "bird-observation" : [ + "type" : "integer", + "x-model-ref": "bird-survey" + ], + "bird-survey" : [ + "type" : "integer", + "x-model-ref": "plot-visit" + ], + "fauna-survey" : [ + "type" : "integer", + "x-model-ref": "plot-visit" + ], + "fauna-observation": [ + "type" : "integer", + "x-model-ref": "fauna-survey" + ], + "plot-visit" : [ + "type" : "integer", + "x-model-ref": "plot-layout" + ], + "plot-layout" : [ + "type" : "integer", + "x-model-ref": "plot-selection" + ], + "plot-selection" : [ + "type" : "object", + "properties": [ + "column": [ + "type": "string" + ] + ] + ] + ] + + + when: + def relationships = service.buildParentChildRelationship(properties) + + then: + relationships.size() == 2 + relationships["bird-survey"] != null + relationships["bird-survey"].size() == 1 + relationships["bird-survey"].contains("bird-observation") + relationships["fauna-survey"].size() == 1 + relationships["fauna-survey"].contains("fauna-observation") + } + + void "buildTreeFromParentChildRelationships should build tree correctly"() { + given: + def relationships = [ + "plot-layout" : ["plot-visit"], + "bird-survey" : ["bird-observation"], + "plot-visit" : ["bird-survey", "fauna-survey"], + "plot-selection": ["plot-layout"], + "fauna-survey" : ["fauna-observation"] + ] + + when: + def tree = service.buildTreeFrom2DRelationship(relationships) + + then: + tree.size() == 1 + tree["bird-survey"] == null + tree["plot-selection"].size() != 0 + tree["plot-selection"]["plot-layout"]["plot-visit"]["bird-survey"]["bird-observation"] != null + } + + void "findPathFromRelationship should find path to model from relationship"() { + setup: + Map relationship = ["aerial-observation": [survey: ["aerial-survey": [setup_ID: ["aerial-setup-desktop": [survey: [:]]]]]], "aerial-survey": [survey: [:]], "aerial-setup": [survey: ["aerial-survey": [setup_ID: ["aerial-setup-desktop": [survey: [:]]]]]]] + + when: + List paths = service.findPathFromRelationship("aerial-setup-desktop", relationship) + + then: + paths.size() == 2 + paths[0] == "aerial-observation.survey.aerial-survey.setup_ID" + paths[1] == "aerial-setup.survey.aerial-survey.setup_ID" + } + + void "removeProperty should remove property from nested object"() { + setup: "nested map" + def object = [a: 1, b: [d: [e: [f: ["toRemove"]]]], c: 3] + + when: + service.removeProperty(object, "b.d.e.f") + + then: + object.size() == 3 + object.b.d.e.f == null + + when: "nested list" + object = [a: 1, b: [[d: [[e: [[f: ["toRemove"]]]]]]], c: 3] + service.removeProperty(object, "b.d.e.f") + + then: + object.size() == 3 + object.b[0].d[0].e[0].f == null + + } + + void "rearrangePropertiesAccordingToModelRelationship should rearrange properties according to model relationship"() { + setup: + def relationship = '{\n' + + ' "ecodata": {\n' + + ' "vegetation-mapping-survey": {\n' + + ' "vegetation-mapping-observation": {}\n' + + ' }\n' + + ' },\n' + + ' "apiOutput": {\n' + + ' "vegetation-mapping-observation.properties.vegetation_mapping_survey": {\n' + + ' "vegetation-mapping-survey": {}\n' + + ' }\n' + + ' }\n' + + ' }' + ObjectMapper mapper = new ObjectMapper() + relationship = mapper.readValue(relationship, Map.class) + + def output = """ +{ + "vegetation-mapping-observation": { + "type": "object", + "properties": { + "vegetation_mapping_survey": { + "type": "object", + "properties": { + "test": { + "type": "string" + } + } + }, + "observation": { + "type": "object", + "properties": { + "survey": { + "type": "object", + "properties": {} + } + } + } + } + } +} +""" + + output = mapper.readValue(output, Map.class) + + when: + output = service.rearrangePropertiesAccordingToModelRelationship(output, relationship.apiOutput, relationship.ecodata) + + then: + output.size() == 2 + output["properties"]["vegetation-mapping-survey"]["properties"].size() == 2 + output["properties"]["vegetation-mapping-survey"]["properties"]["vegetation-mapping-observation"]["properties"].size() == 1 + output["properties"]["vegetation-mapping-survey"]["properties"]["test"] == ["type": "string"] + output["properties"]["vegetation-mapping-survey"]["properties"]["vegetation-mapping-observation"]["properties"]["observation"]["properties"]["survey"]["properties"].size() == 0 + } - activityForm = new ActivityForm(name:"aParatooForm 3", type:'EMSA', category:'protocol category 3', external: true) - activityForm.externalIds = [new ExternalId(externalId: "guid-4", idType:ExternalId.IdType.MONITOR_PROTOCOL_GUID)] - activityForm.save(failOnError:true, flush:true) + void "cleanSwaggerDefinition should standardize the swagger definition"() { + setup: + def definition = [ + "type" : "object", + "required" : ["data"], + "properties" : [ + "data": [ + "required" : ["transect_number", "date_time"], + "type" : "object", + "properties": [ + "plot" : [ + "type" : "integer", + "x-model-ref": "setup" + ], + "number" : ["type": "string"], + "photo" : [ + "type" : "integer", + "x-paratoo-file-type": ["images"], + "x-model-ref" : "file" + ], + "quad" : [ + "type" : "array", + "items": [ + "type" : "integer", + "x-model-ref": "quadrant" + ], + ], + "date_time": ["type": "string", "format": "date-time"] + ] + ] + ], + "additionalProperties": false + ] + + when: + def result = service.cleanSwaggerDefinition(definition) + + then: + result.size() == 3 + + result.properties.size() == 5 + result.properties.plot.size() == 2 + result.properties.number.type == "string" + result.properties.quad.properties.type == "integer" + result.required.size() == 2 + result.required[0] == "transect_number" + } + + void "buildChildParentRelationship should build child parent relationship from simplified data model"() { + setup: + def output = ' {\n' + + ' "type" : "object",\n' + + ' "properties" : {\n' + + ' "plot" : {\n' + + ' "type" : "integer",\n' + + ' "x-model-ref" : "setup"\n' + + ' },\n' + + ' "number" : {\n' + + ' "type" : "string"\n' + + ' },\n' + + ' "photo" : {\n' + + ' "type" : "integer",\n' + + ' "x-paratoo-file-type" : [ "images" ],\n' + + ' "x-model-ref" : "file"\n' + + ' },\n' + + ' "quad" : {\n' + + ' "type" : "array",\n' + + ' "properties" : {\n' + + ' "type" : "integer",\n' + + ' "x-model-ref" : "quadrant"\n' + + ' }\n' + + ' },\n' + + ' "date_time" : {\n' + + ' "type" : "string",\n' + + ' "format" : "date-time"\n' + + ' }\n' + + ' },\n' + + ' "required" : [ "transect_number", "date_time" ]\n' + + '}' + ObjectMapper mapper = new ObjectMapper() + def input = mapper.readValue(output, Map.class) + + when: + def result = service.buildChildParentRelationship(input) + + then: + result.size() == 2 + result["properties.plot"] == ["setup"] + result["properties.quad"] == ["quadrant"] + } + + void "buildParentChildRelationship should build parent child relationship from simplified data model"() { + setup: + def output = ' {\n' + + ' "type" : "object",\n' + + ' "properties" : {\n' + + ' "plot" : {\n' + + ' "type" : "integer",\n' + + ' "x-model-ref" : "setup"\n' + + ' },\n' + + ' "number" : {\n' + + ' "type" : "string"\n' + + ' },\n' + + ' "photo" : {\n' + + ' "type" : "integer",\n' + + ' "x-paratoo-file-type" : [ "images" ],\n' + + ' "x-model-ref" : "file"\n' + + ' },\n' + + ' "quad" : {\n' + + ' "type" : "array",\n' + + ' "properties" : {\n' + + ' "type" : "integer",\n' + + ' "x-model-ref" : "quadrant"\n' + + ' }\n' + + ' },\n' + + ' "date_time" : {\n' + + ' "type" : "string",\n' + + ' "format" : "date-time"\n' + + ' }\n' + + ' },\n' + + ' "required" : [ "transect_number", "date_time" ]\n' + + '}' + ObjectMapper mapper = new ObjectMapper() + def input = mapper.readValue(output, Map.class) + + when: + def result = service.buildParentChildRelationship(input) + + then: + result.size() == 2 + result["setup"] == ["plot"] + result["quadrant"] == ["quad"] + } + + void "rearrangeSurveyData should reorder api output according to provided relationship"() { + setup: + def mapper = new ObjectMapper() + def output = '{"a": {}, "f": 4, "b": {"c": 1, "d": 2}, "e": {"g": 3}}' + output = mapper.readValue(output, Map.class) + def relationship = '{"b": {"e": {"a": {} } }, "c": {} }' + relationship = mapper.readValue(relationship, Map.class) + + when: + def result = service.rearrangeSurveyData(output, output, relationship) + + then: + result.size() == 2 + result["b"].size() == 3 + result["b"]["c"] == 1 + result["b"]["d"] == 2 + result["b"]["e"].size() == 2 + result["b"]["e"]["a"].size() == 0 + result["b"]["e"]["g"] == 3 + result["f"] == 4 + } + + void "resolveModelReferences should swap model with definitions"() { + setup: + def dataModel = [ + "type" : "object", + "properties": [ + "plot" : [ + "type" : "integer", + "x-model-ref": "setup" + ], + "number" : ["type": "string"], + "photo" : [ + "type" : "integer", + "x-paratoo-file-type": ["images"], + "x-model-ref" : "file" + ], + "quad" : [ + "type" : "array", + "properties": [ + "type" : "integer", + "x-model-ref": "quadrant" + ], + ], + "date_time": ["type": "string", "format": "date-time"] + ] + ] + + def components = [ + "SetupRequest" : [ + "type" : "object", + "properties": [ + "transect_number": ["type": "string"], + "date_time" : ["type": "string", "format": "date-time"] + ] + ], + "QuadrantRequest": [ + "type" : "object", + "properties": [ + "transect": ["type": "string"], + "date" : ["type": "string", "format": "date"] + ] + ] + ] + + when: + def result = service.resolveModelReferences(dataModel, components) + + then: + result.size() == 2 + result.properties.plot.properties.transect_number.type == "string" + result.properties.plot.properties.date_time.format == "date-time" + result.properties.quad.properties.properties.date.format == "date" + result.properties.quad.properties.properties.transect.type == "string" + result.properties.photo.type == "integer" + result.properties.photo["x-model-ref"] == "file" + } + + void "isPlotLayoutNeededByProtocol checks if plot is required by protocol"() { + given: + def protocol1 = [attributes: [workflow: [[modelName: 'plot-layout'], [modelName: 'other-model']]]] + def expected1 = true + + when: + def result = service.isPlotLayoutNeededByProtocol(protocol1) + + then: + result == expected1 + + when: + def protocol2 = [attributes: [workflow: [[modelName: 'other-model'], [modelName: 'another-model']]]] + def expected2 = false + result = service.isPlotLayoutNeededByProtocol(protocol2) + + then: + result == expected2 + } + + void "findProtocolEndpointDefinition should find the protocol endpoint definition"() { + given: + def protocol = [attributes: [endpointPrefix: "opportunes", workflow: [[modelName: 'plot-layout'], [modelName: 'other-model']]]] + def documentation = [ + paths: [ + "opportunes/bulk": [ + post: [ + requestBody: + [ + content: [ + "application/json": [ + schema: [ + "properties": [ + "data": [ + "properties": [ + "collections": [ + "items": [ + "properties": [ + "surveyId": [ + "properties": [ + "projectId": [ + "type": "string" + ], + "protocol" : [ + "properties": [ + "id" : [ + "type": "string" + ], + "version": [ + "type": "integer" + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + + when: + def result = service.findProtocolEndpointDefinition(protocol, documentation) + + then: + result != null + result.surveyId != null + + when: // no endpoint path is found should return null + documentation = [ + paths: [ + "test/bulk": [ + post: [ + requestBody: + [ + content: [ + "application/json": [ + schema: [ + "properties": [ + "data": [ + "properties": [ + "collections": [ + "items": [ + "properties": [ + "surveyId": [ + "properties": [ + "projectId": [ + "type": "string" + ], + "protocol" : [ + "properties": [ + "id" : [ + "type": "string" + ], + "version": [ + "type": "integer" + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + ] + result = service.findProtocolEndpointDefinition(protocol, documentation) + + then: + result == null + + when: // schema has changed should throw exception + documentation = [ + paths: [ + "opportunes/bulk": [ + post: [ + requestBody: + [ + content: [ + "application/json": [:] + ] + ] + ] + ] + ] + ] + service.findProtocolEndpointDefinition(protocol, documentation) + + then: + thrown(NullPointerException) + } + + void "resolveReferences should resolve \$ref variables in schema"() { + given: + def schema = [ + "plot-layout" : [ + "\$ref": "#/components/schemas/PlotLayoutRequest" + ], + "basal-area-dbh-measure-observation": [ + "type" : "array", + "items": [ + "\$ref": "#/components/schemas/BasalAreaDbhMeasureObservationRequest" + ] + ] + ] + def components = [ + "PlotLayoutRequest" : [ + "type" : "object", + "properties": [ + "transect_number": ["type": "string"], + "date_time" : ["type": "string", "format": "date-time"] + ] + ], + "BasalAreaDbhMeasureObservationRequest": [ + "type" : "object", + "properties": [ + "transect": ["type": "string"], + "date" : ["type": "string", "format": "date"] + ] + ] + ] + + when: + def result = service.resolveReferences(schema, components) + + then: + result.size() == 2 + result["plot-layout"].size() == 2 + result["plot-layout"].properties.transect_number.type == "string" + result["plot-layout"].properties.date_time.format == "date-time" + result["basal-area-dbh-measure-observation"].size() == 2 + result["basal-area-dbh-measure-observation"].items.properties.date.format == "date" + result["basal-area-dbh-measure-observation"].items.properties.transect.type == "string" + } + + void "convertToDataModelAndViewModel should return correct data model"() { + when:// when integer type is provided + String name = "height" + def component = [ + "type" : "integer", + "x-paratoo-unit": "m", + "x-paratoo-hint": "height" + ] + def config = new ParatooProtocolConfig( + "name" : "Opportune", + "usesPlotLayout": false, + "tags" : ["survey"], + "apiEndpoint" : "s1s", + "overrides" : [ + "dataModel": null, + "viewModel": null + ] + ) + def result = service.convertToDataModelAndViewModel(component, [:], name, null, null, 0, "", config) + service.cacheService.clear() + then: + result.dataModel[0] == [ + "dataType" : "number", + "units" : "m", + "name" : "height", + "description" : "height", + "decimalPlaces": 0 + ] + result.viewModel[0] == [ + "type" : "number", + "source" : "height", + "preLabel": "Height" + ] + when:// when number type is provided + component = [ + "type" : "number", + "x-paratoo-unit": "m", + "x-paratoo-hint": "height" + ] + result = service.convertToDataModelAndViewModel(component, [:], name, null, null, 0, "", config) + then: + result.dataModel[0] == [ + "dataType" : "number", + "units" : "m", + "name" : "height", + "description" : "height", + "decimalPlaces": 6 + ] + result.viewModel[0] == [ + "type" : "number", + "source" : "height", + "preLabel": "Height" + ] + + when:// when string type is provided + service.cacheService.clear() + component = [ + "type" : "string", + "x-paratoo-unit": "m", + "x-paratoo-hint": "height", + "x-lut-ref" : "lut1" + ] + result = service.convertToDataModelAndViewModel(component, [:], name, null, null, 0, "", config) + then: + 1 * webService.getJson( {it.indexOf('/lut1s') >= 0}, _, _, _) >> [data: [ + [ + attributes: [ + "symbol": "1", + "label" : "one" + ] + ], + [ + attributes: [ + "symbol": "2", + "label" : "two" + ] + ] + ], meta : [pagination: [total: 0]]] + 1 * webService.getJson({ it.indexOf('/documentation/swagger.json') >= 0 }, _, _, _) >> [ + paths: ["/lut1s": [:]] + ] + result.dataModel[0] == [ + "dataType" : "text", + "units" : "m", + "name" : "height", + "description" : "height", + "constraints" : [ + "textProperty" : "label", + "type" : "literal", + "valueProperty": "value", + "literal" : [ + [ + label: "one", + value: "1" + ], + [ + label: "two", + value: "2" + ] + ] + ], + "displayOptions": [ + "placeholder": "Select an option", + "tags" : true + ], + "x-lut-ref":"lut1" + ] + result.viewModel[0] == [ + "type" : "selectOne", + "source" : "height", + "preLabel": "Height" + ] } } diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index 741d547a9..c0b6b1644 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -1,6 +1,7 @@ package au.org.ala.ecodata.paratoo - +import au.org.ala.ecodata.ActivityForm +import au.org.ala.ecodata.ExternalId import grails.converters.JSON import groovy.json.JsonSlurper import org.grails.web.converters.marshaller.json.CollectionMarshaller @@ -39,6 +40,26 @@ class ParatooProtocolConfigSpec extends Specification { endDatePath: null ] ParatooProtocolConfig config = new ParatooProtocolConfig(vegetationMappingConfig) + ActivityForm activityForm = new ActivityForm( + name: "aParatooForm 1", + type: 'EMSA', + category: 'protocol category 1', + external: true, + sections: [ + [ + name: "section 1", + template: [ + dataModel: [[ + name: "field 1", + type: "text", + required: true, + external: true + ]] + ] + ] + ] + ) + activityForm.externalIds = [new ExternalId(externalId: "guid-2", idType: ExternalId.IdType.MONITOR_PROTOCOL_GUID)] expect: config.getStartDate(surveyData) == '2023-09-08T23:39:00Z' @@ -104,7 +125,6 @@ class ParatooProtocolConfigSpec extends Specification { ParatooSurveyId paratooSurveyId = new ParatooSurveyId( surveyType: 'opportunistic-survey', time: parse("2023-10-24T00:59:48.456Z").toDate(), - randNum: 80454523, projectId: '0d02b422-5bf7-495f-b9f2-fa0a3046937f', protocol: new ParatooProtocolId(id: "068d17e8-e042-ae42-1e42-cff4006e64b0", version: 1) ) From 7c2918c455a55fa35f6e651e5d302087d6e1d941 Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 25 Mar 2024 14:00:11 +1100 Subject: [PATCH 16/52] AtlasOfLivingAustralia/fieldcapture#3049 - added vegetation observation records - fixed broken tests --- .../au/org/ala/ecodata/ParatooService.groovy | 3 +- .../paratoo/ParatooProtocolConfig.groovy | 4 + .../paratoo/ParatooProtocolConfigSpec.groovy | 9 +- .../opportunisticSurveyObservations.json | 3 + .../paratoo/vegetationMappingObservation.json | 150 ++++++++++++++++++ 5 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/paratoo/vegetationMappingObservation.json diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 079dcfd83..9833a989f 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -67,6 +67,7 @@ class ParatooService { ActivityService activityService RecordService recordService MetadataService metadataService + UserService userService /** * The rules we use to find projects eligible for use by paratoo are: @@ -375,7 +376,7 @@ class ParatooService { projectId : collection.projectId, publicationStatus: "published", siteId : siteId, - userId : collection.parseOrgMintedIdentifier()?.userId, + userId : userService.getCurrentUserDetails()?.userId, outputs : [[ data: surveyObservations, name: activityForm.name diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 0c6df8b7f..65ff1ced4 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -229,6 +229,10 @@ class ParatooProtocolConfig { geoJson.features = features } } + else if (geometryPath) { + geoJson = extractSiteDataFromPath(survey) + } + geoJson } diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index c0b6b1644..70976af10 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -31,6 +31,7 @@ class ParatooProtocolConfigSpec extends Specification { setup: Map surveyData = readSurveyData('vegetationMappingSurvey') + Map observation = readSurveyData('vegetationMappingObservation') Map vegetationMappingConfig = [ apiEndpoint:'vegetation-mapping-observations', usesPlotLayout:false, @@ -50,8 +51,8 @@ class ParatooProtocolConfigSpec extends Specification { name: "section 1", template: [ dataModel: [[ - name: "field 1", - type: "text", + name: "position", + dataType: "feature", required: true, external: true ]] @@ -65,6 +66,7 @@ class ParatooProtocolConfigSpec extends Specification { config.getStartDate(surveyData) == '2023-09-08T23:39:00Z' config.getEndDate(surveyData) == null config.getGeoJson(surveyData) == [type:'Point', coordinates:[149.0651536, -35.2592398]] + config.getGeoJson(surveyData, observation, activityForm).features == [[type:'Point', coordinates:[149.0651536, -35.2592398]]] } def "The floristics-standard survey can be used with this config"() { @@ -126,7 +128,8 @@ class ParatooProtocolConfigSpec extends Specification { surveyType: 'opportunistic-survey', time: parse("2023-10-24T00:59:48.456Z").toDate(), projectId: '0d02b422-5bf7-495f-b9f2-fa0a3046937f', - protocol: new ParatooProtocolId(id: "068d17e8-e042-ae42-1e42-cff4006e64b0", version: 1) + protocol: new ParatooProtocolId(id: "068d17e8-e042-ae42-1e42-cff4006e64b0", version: 1), + uuid: "10a03062-2b0d-40bb-a6d7-e72f06788b94" ) List data = config.findObservationsBelongingToSurvey(surveyObservations.data, paratooSurveyId) diff --git a/src/test/resources/paratoo/opportunisticSurveyObservations.json b/src/test/resources/paratoo/opportunisticSurveyObservations.json index 8253c6361..f67ff81c3 100644 --- a/src/test/resources/paratoo/opportunisticSurveyObservations.json +++ b/src/test/resources/paratoo/opportunisticSurveyObservations.json @@ -87,6 +87,7 @@ "surveyId": { "time": "2023-10-24T00:59:48.456Z", "randNum": 80454523, + "uuid": "10a03062-2b0d-40bb-a6d7-e72f06788b94", "protocol": { "id": "068d17e8-e042-ae42-1e42-cff4006e64b0", "version": 1 @@ -184,6 +185,7 @@ "attributes": { "surveyId": { "time": "2023-10-24T00:59:48.456Z", + "uuid": "10a03062-2b0d-40bb-a6d7-e72f06788b49", "randNum": 80775523, "protocol": { "id": "068d17e8-e042-ae42-1e42-cff4006e64b0", @@ -283,6 +285,7 @@ "surveyId": { "time": "2023-10-24T00:59:48.456Z", "randNum": 804993, + "uuid": "10a03062-2b0d-40bb-a6d7-e72f06788b34", "protocol": { "id": "068d17e8-e042-ae42-1e42-cff4006e64b0", "version": 1 diff --git a/src/test/resources/paratoo/vegetationMappingObservation.json b/src/test/resources/paratoo/vegetationMappingObservation.json new file mode 100644 index 000000000..64e4e4792 --- /dev/null +++ b/src/test/resources/paratoo/vegetationMappingObservation.json @@ -0,0 +1,150 @@ +{ + "date_time": "2023-09-08T23:39:02.792Z", + "field_id": "This is a test 2", + "homogeneity_measure": 10, + "comment": "Test", + "NVIS_level_5": null, + "createdAt": "2023-09-09T00:51:43.722Z", + "updatedAt": "2023-09-09T00:51:43.722Z", + "position": { + "type": "Point", + "coordinates": [ + 149.0651536, + -35.2592398 + ] + }, + "photo": { + "id": 11, + "description": "Test", + "photo": { + "data": { + "id": 25, + "attributes": { + "name": "48273978.png", + "alternativeText": null, + "caption": null, + "width": 576, + "height": 133, + "formats": { + "small": { + "ext": ".png", + "url": "https://paratoo-uploads-dev.s3.ap-southeast-2.amazonaws.com/small_48273978_f2b4e1fa84.png", + "hash": "small_48273978_f2b4e1fa84", + "mime": "image/png", + "name": "small_48273978.png", + "path": null, + "size": 22.27, + "width": 500, + "height": 115 + }, + "thumbnail": { + "ext": ".png", + "url": "https://paratoo-uploads-dev.s3.ap-southeast-2.amazonaws.com/thumbnail_48273978_f2b4e1fa84.png", + "hash": "thumbnail_48273978_f2b4e1fa84", + "mime": "image/png", + "name": "thumbnail_48273978.png", + "path": null, + "size": 7.5, + "width": 245, + "height": 57 + } + }, + "hash": "48273978_f2b4e1fa84", + "ext": ".png", + "mime": "image/png", + "size": 7.32, + "url": "https://paratoo-uploads-dev.s3.ap-southeast-2.amazonaws.com/48273978_f2b4e1fa84.png", + "previewUrl": null, + "provider": "aws-s3", + "provider_metadata": null, + "createdAt": "2023-09-09T00:51:42.889Z", + "updatedAt": "2023-09-09T00:51:42.889Z" + } + } + }, + "direction": { + "data": { + "id": 1, + "attributes": { + "symbol": "S", + "label": "South", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/c02d24ff-488e-5039-b2f0-c8c304c273e6", + "createdAt": "2023-09-08T07:02:37.783Z", + "updatedAt": "2023-09-08T07:02:37.783Z" + } + } + } + }, + "vegetation_growth_stage": { + "data": { + "id": 1, + "attributes": { + "symbol": "ER", + "label": "Early Regeneration", + "description": "", + "uri": "", + "createdAt": "2023-09-08T07:03:09.387Z", + "updatedAt": "2023-09-08T07:03:09.387Z" + } + } + }, + "disturbance": { + "data": { + "id": 12, + "attributes": { + "symbol": "NC", + "label": "Not Collected", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/94ee6b46-e2f1-5101-8666-3cbcd8697f0f", + "createdAt": "2023-09-08T07:02:37.931Z", + "updatedAt": "2023-09-08T07:02:37.931Z" + } + } + }, + "fire_history": { + "data": { + "id": 2, + "attributes": { + "symbol": "U", + "label": "Unknown", + "description": "", + "uri": "", + "createdAt": "2023-09-08T07:02:40.350Z", + "updatedAt": "2023-09-08T07:02:40.350Z" + } + } + }, + "vegetation_mapping_survey": { + "data": { + "id": 8, + "attributes": { + "start_date_time": "2023-09-08T23:39:00.520Z", + "surveyId": { + "time": "2023-09-09T00:51:34.001Z", + "uuid": "39297148", + "protocol": { + "id": "cd2cbbc7-2f17-4b0f-91b4-06f46e9c90f2", + "version": 1 + }, + "projectId": "0d02b422-5bf7-495f-b9f2-fa0a3046937f", + "surveyType": "vegetation-mapping-survey" + }, + "createdAt": "2023-09-09T00:51:43.687Z", + "updatedAt": "2023-09-09T00:51:43.687Z" + } + } + }, + "NVIS_level_5_vegetation_association_information": [], + "substrate_cover": { + "id": 9, + "bare_cover_percent": 2, + "cryptogam_cover_percent": 3, + "outcrop_cover_percent": 4, + "litter_cover_percent": 5, + "rock_cover_percent": 6, + "coarse_woody_debris_cover_percent": 7, + "gravel_cover_percent": 8, + "unknown_cover_percent": 9 + } +} \ No newline at end of file From 00deb0b45375907dbfaa9093c1d88a9750beb54f Mon Sep 17 00:00:00 2001 From: chrisala Date: Mon, 25 Mar 2024 14:48:06 +1100 Subject: [PATCH 17/52] 4.5-SNAPSHOT --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8f4711fa8..755d03048 100644 --- a/build.gradle +++ b/build.gradle @@ -20,7 +20,7 @@ plugins { id "com.gorylenko.gradle-git-properties" version "2.4.1" } -version "4.4" +version "4.5-SNAPSHOT" group "au.org.ala" description "Ecodata" From 0fd7391f4807a0720e2f1c958de38dd088eebb63 Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 26 Mar 2024 09:12:17 +1100 Subject: [PATCH 18/52] Added fundingVerificationDate to Project AtlasOfLivingAustralia/fieldcapture#2973 --- grails-app/conf/data/mapping.json | 4 ++++ grails-app/domain/au/org/ala/ecodata/Project.groovy | 3 +++ 2 files changed, 7 insertions(+) diff --git a/grails-app/conf/data/mapping.json b/grails-app/conf/data/mapping.json index 980d85eb1..39e8c094a 100644 --- a/grails-app/conf/data/mapping.json +++ b/grails-app/conf/data/mapping.json @@ -60,6 +60,10 @@ "fundingType": { "type" : "keyword" }, + "fundingVerificationDate": { + "type": "date", + "ignore_malformed": true + }, "status": { "type" : "keyword" }, diff --git a/grails-app/domain/au/org/ala/ecodata/Project.groovy b/grails-app/domain/au/org/ala/ecodata/Project.groovy index e78edf691..0bfd23021 100644 --- a/grails-app/domain/au/org/ala/ecodata/Project.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Project.groovy @@ -70,6 +70,8 @@ class Project { List ecoScienceType = [] List tags = [] double funding + /** The most recent date the correctness of the project funding was checked */ + Date fundingVerificatonDate String orgIdGrantee, orgIdSponsor, orgIdSvcProvider String userCreated, userLastModified boolean isExternal = false // An external project only has a listing with the ALA and is not using data capture capabilities @@ -236,6 +238,7 @@ class Project { mapDisplays nullable: true terminationReason nullable: true fundingType nullable: true + fundingVerificatonDate nullable: true electionCommitmentYear nullable: true geographicInfo nullable:true portfolio nullable: true From 70128fdc8334636ad4182654d075355f869470a1 Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 26 Mar 2024 13:19:50 +1100 Subject: [PATCH 19/52] Added logging fieldcapture#3049 --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 030cd6f02..24e311fa1 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -676,6 +676,7 @@ class ParatooService { String url = paratooBaseUrl + apiEndpoint Map response = webService.doPost(url, payload, false, authHeader)?.resp + log.debug((response as JSON).toString()) response?.collections } From 78a44085e1a82f25ae7c7c06f62865e03d34bac5 Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 26 Mar 2024 13:47:55 +1100 Subject: [PATCH 20/52] Added logging fieldcapture#3049 --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 24e311fa1..dc1c905d2 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -675,9 +675,10 @@ class ParatooService { Map authHeader = getAuthHeader() String url = paratooBaseUrl + apiEndpoint - Map response = webService.doPost(url, payload, false, authHeader)?.resp + Map response = webService.doPost(url, payload, false, authHeader) log.debug((response as JSON).toString()) - response?.collections + + response?.resp?.collections } From cb01b50b588b7d721482081114033be32639d5f7 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 28 Mar 2024 12:32:56 +1100 Subject: [PATCH 21/52] AtlasOfLivingAustralia/fieldcapture#3049 - async fetch from monitor - refactored code --- .../org/ala/ecodata/ParatooController.groovy | 4 +- .../au/org/ala/ecodata/ParatooService.groovy | 212 ++++++++++++------ .../converter/GenericFieldConverter.groovy | 10 +- .../ecodata/converter/ListConverter.groovy | 4 + .../ecodata/converter/SpeciesConverter.groovy | 4 + .../ecodata/metadata/OutputMetadata.groovy | 1 - .../paratoo/ParatooProtocolConfig.groovy | 124 ++-------- .../ala/ecodata/ParatooControllerSpec.groovy | 4 +- .../org/ala/ecodata/ParatooServiceSpec.groovy | 36 +-- .../paratoo/ParatooProtocolConfigSpec.groovy | 21 +- 10 files changed, 202 insertions(+), 218 deletions(-) diff --git a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy index b36cba548..0521d8cec 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy @@ -289,10 +289,10 @@ class ParatooController { boolean hasProtocol = paratooService.protocolWriteCheck(userId, dataSet.project.id, collectionId.protocolId) if (hasProtocol) { Map result = paratooService.submitCollection(collection, dataSet.project) - if (!result.error) { + if (!result.updateResult.error) { respond([success: true]) } else { - error(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.error) + error(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.updateResult.error) } } else { error(HttpStatus.SC_FORBIDDEN, "Project / protocol combination not available") diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index dc1c905d2..08aaba962 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -1,20 +1,23 @@ package au.org.ala.ecodata -import au.org.ala.ecodata.metadata.PropertyAccessor + import au.org.ala.ecodata.paratoo.* import au.org.ala.ws.tokens.TokenService +import grails.async.Promise import grails.converters.JSON import grails.core.GrailsApplication import groovy.util.logging.Slf4j import java.util.regex.Matcher import java.util.regex.Pattern + +import static grails.async.Promises.task /** * Supports the implementation of the paratoo "org" interface */ @Slf4j class ParatooService { - + static final int PARATOO_MAX_RETRIES = 3 static final String PARATOO_PROTOCOL_PATH = '/protocols' static final String PARATOO_DATA_PATH = '/protocols/reverse-lookup' static final String PARATOO_PROTOCOL_FORM_TYPE = 'EMSA' @@ -68,6 +71,7 @@ class ParatooService { RecordService recordService MetadataService metadataService UserService userService + ElasticSearchService elasticSearchService /** * The rules we use to find projects eligible for use by paratoo are: @@ -207,8 +211,6 @@ class ParatooService { * @return */ Map submitCollection(ParatooCollection collection, ParatooProject project) { - boolean forceActivityCreation = false - Map dataSet = project.project.custom?.dataSets?.find{it.dataSetId == collection.orgMintedUUID} if (!dataSet) { @@ -217,43 +219,75 @@ class ParatooService { dataSet.progress = Activity.STARTED dataSet.surveyId.coreSubmitTime = new Date() dataSet.surveyId.survey_metadata.provenance.putAll(collection.coreProvenance) + String userId = userService.currentUserDetails?.userId + Map authHeader = getAuthHeader() + Promise promise = task { + asyncFetchCollection(collection, authHeader, userId, project) + } + promise.onError { Throwable e -> + log.error("An error occurred feching ${collection.orgMintedUUID}: ${e.message}", e) + } + def result = projectService.update([custom: project.project.custom], project.id, false) + [updateResult: result, promise: promise] + } - ParatooCollectionId surveyId = ParatooCollectionId.fromMap(dataSet.surveyId) - - ParatooProtocolConfig config = getProtocolConfig(surveyId.protocolId) - ActivityForm form = ActivityForm.findByExternalId(surveyId.protocolId) - // reverse lookup gets survey and observations but not plot based data - Map surveyDataAndObservations = retrieveSurveyAndObservations(collection) - // get survey data from reverse lookup response - Map surveyData = config.getSurveyDataFromObservation(surveyDataAndObservations) - // get plot data by querying protocol api endpoint - Map surveyDataWithPlotInfo = retrieveSurveyData(surveyId, config, surveyData.createdAt) - // add plot data to survey observations - addPlotDataToObservations(surveyDataWithPlotInfo, surveyDataAndObservations, config) - if (surveyDataAndObservations) { + Map asyncFetchCollection(ParatooCollection collection, Map authHeader, String userId, ParatooProject project) { + int counter = 0 + boolean forceActivityCreation = false + Map surveyDataAndObservations = null + Map dataSet = project.project.custom?.dataSets?.find{it.dataSetId == collection.orgMintedUUID} + + if (!dataSet) { + throw new RuntimeException("Unable to find data set with orgMintedUUID: "+collection.orgMintedUUID) + } + + // wait for 5 seconds before fetching data + while(surveyDataAndObservations == null && counter < PARATOO_MAX_RETRIES) { + sleep(5 * 1000) + try { + surveyDataAndObservations = retrieveSurveyAndObservations(collection, authHeader) + } catch (Exception e) { + log.error("Error fetching collection data for ${collection.orgMintedUUID}: ${e.message}") + } + + counter++ + } + + if (surveyDataAndObservations == null) { + log.error("Unable to fetch collection data for ${collection.orgMintedUUID}") + return + } else { + ParatooCollectionId surveyId = ParatooCollectionId.fromMap(dataSet.surveyId) + ParatooProtocolConfig config = getProtocolConfig(surveyId.protocolId) + ActivityForm form = ActivityForm.findByExternalId(surveyId.protocolId) + // get survey data from reverse lookup response + Map surveyData = config.getSurveyDataFromObservation(surveyDataAndObservations) + // get plot data by querying protocol api endpoint + Map surveyDataWithPlotInfo = retrieveSurveyData(surveyId, config, surveyData.createdAt, authHeader) + // add plot data to survey observations + addPlotDataToObservations(surveyDataWithPlotInfo, surveyDataAndObservations, config) + rearrangeSurveyData(surveyDataAndObservations, surveyDataAndObservations, form.sections[0].template.relationships.ecodata, form.sections[0].template.relationships.apiOutput) // transform data to make it compatible with data model surveyDataAndObservations = recursivelyTransformData(form.sections[0].template.dataModel, surveyDataAndObservations) - rearrangeSurveyData(surveyDataAndObservations, surveyDataAndObservations, form.sections[0].template.relationships.ecodata) // If we are unable to create a site, null will be returned - assigning a null siteId is valid. if (!dataSet.siteId) dataSet.siteId = createSiteFromSurveyData(surveyDataWithPlotInfo, surveyDataAndObservations, collection, surveyId, project.project, config, form) // make sure activity has not been created for this data set if (!dataSet.activityId || forceActivityCreation) { - String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet.siteId) - List records = recordService.getAllByActivity(activityId) - dataSet.areSpeciesRecorded = records?.size() > 0 - dataSet.activityId = activityId + Activity.withSession { + String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet.siteId, userId) + List records = recordService.getAllByActivity(activityId) + dataSet.areSpeciesRecorded = records?.size() > 0 + dataSet.activityId = activityId + } } dataSet.startDate = config.getStartDate(surveyDataAndObservations) dataSet.endDate = config.getEndDate(surveyDataAndObservations) - } else { - log.warn("Unable to retrieve survey data for: " + collection.orgMintedUUID) - log.debug(surveyData) - } - projectService.update([custom: project.project.custom], project.id, false) + projectService.update([custom: project.project.custom], project.id, false) + } } /** @@ -268,23 +302,32 @@ class ParatooService { * @param isChild * @return */ - Map rearrangeSurveyData (Map properties, Map rootProperties, Map relationship, List nodesToRemove = [], List ancestors = [] , Boolean isChild = false) { + Map rearrangeSurveyData (Map properties, Map rootProperties, Map relationship, Map apiOutputRelationship, List nodesToRemove = [], List ancestors = [] , Boolean isChild = false) { if (relationship instanceof Map) { relationship.each { String nodeName, Map children -> ancestors.add(nodeName) - def nodeObject = rootProperties[nodeName] ?: [:] - if (isChild) { + def nodeObject = rootProperties[nodeName] + if (nodeObject != null) { properties[nodeName] = nodeObject + } + else { + nodeObject = findObservationDataFromAPIOutput(nodeName, apiOutputRelationship, rootProperties) + if (nodeObject != null) { + properties[nodeName] = nodeObject + } + } + + if (isChild) { nodesToRemove.add(nodeName) } if (children) { if (nodeObject instanceof Map) { - rearrangeSurveyData(nodeObject, rootProperties, children, nodesToRemove, ancestors, true ) + rearrangeSurveyData(nodeObject, rootProperties, children, apiOutputRelationship, nodesToRemove, ancestors, true ) } else if (nodeObject instanceof List) { nodeObject.each { Map node -> - rearrangeSurveyData(node, rootProperties, children, nodesToRemove, ancestors, true ) + rearrangeSurveyData(node, rootProperties, children, apiOutputRelationship, nodesToRemove, ancestors, true ) } } } @@ -370,7 +413,7 @@ class ParatooService { * @param siteId * @return */ - private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, String siteId) { + private String createActivityFromSurveyData(ActivityForm activityForm, Map surveyObservations, ParatooCollectionId collection, String siteId, String userId) { Map activityProps = [ type : activityForm.name, formVersion : activityForm.formVersion, @@ -378,7 +421,7 @@ class ParatooService { projectId : collection.projectId, publicationStatus: "published", siteId : siteId, - userId : userService.getCurrentUserDetails()?.userId, + userId : userId, outputs : [[ data: surveyObservations, name: activityForm.name @@ -401,18 +444,32 @@ class ParatooService { switch (model.dataType) { case "list": String updatedPath = model.name - def rows = getProperty(output, updatedPath) + def rows =[] + try { + rows = getProperty(output, updatedPath)?.first() + } + catch (Exception e) { + log.info("Error getting list for ${model.name}: ${e.message}") + } + if (rows instanceof Map) { output[updatedPath] = rows = [rows] } rows?.each { row -> - recursivelyTransformData(model.columns, row) + if (row != null) { + recursivelyTransformData(model.columns, row) + } } break case "species": - String speciesName = getProperty(output, model.name) - output[model.name] = transformSpeciesName(speciesName) + String speciesName + try { + speciesName = getProperty(output, model.name)?.first() + output[model.name] = transformSpeciesName(speciesName) + } catch (Exception e) { + log.info("Error getting species name for ${model.name}: ${e.message}") + } break case "feature": // used by protocols like bird survey where a point represents a sight a bird has been observed in a @@ -638,10 +695,11 @@ class ParatooService { "?populate=deep&sort=updatedAt&pagination[start]=$start&pagination[limit]=$limit&filters[createdAt][\$eq]=$createdAt" } - Map retrieveSurveyData(ParatooCollectionId surveyId, ParatooProtocolConfig config, String createdAt) { - + Map retrieveSurveyData(ParatooCollectionId surveyId, ParatooProtocolConfig config, String createdAt, Map authHeader = null) { String apiEndpoint = config.getApiEndpoint(surveyId) - Map authHeader = getAuthHeader() + if (!authHeader) { + authHeader = getAuthHeader() + } int start = 0 int limit = 10 @@ -666,13 +724,15 @@ class ParatooService { survey } - Map retrieveSurveyAndObservations(ParatooCollection collection) { + Map retrieveSurveyAndObservations(ParatooCollection collection, Map authHeader = null) { String apiEndpoint = PARATOO_DATA_PATH Map payload = [ org_minted_uuid: collection.orgMintedUUID ] - Map authHeader = getAuthHeader() + if (!authHeader) { + authHeader = getAuthHeader() + } String url = paratooBaseUrl + apiEndpoint Map response = webService.doPost(url, payload, false, authHeader) @@ -681,7 +741,6 @@ class ParatooService { response?.resp?.collections } - private static Map findMatchingSurvey(ParatooCollectionId surveyId, List data, ParatooProtocolConfig config) { data?.find { config.matches(it, surveyId) } } @@ -1039,7 +1098,7 @@ class ParatooService { childrenNode } - public static def deepCopy(def original) { + static def deepCopy(def original) { def copy if (original instanceof Map) { @@ -1216,17 +1275,6 @@ class ParatooService { ref.replace("#/components/schemas/", "") } - Map getActivityFormForComponent(String modelName) { - Map documentation = getParatooSwaggerDocumentation() - Map components = getComponents(documentation) - Map component = components[modelName] - - if (component) { - log.debug((component as JSON).toString(true)) - convertToDataModelAndViewModel(component, documentation, modelName) - } - } - Map convertToDataModelAndViewModel(Map component, Map documentation, String name, List required = null, Deque modelVisitStack = null, int depth = 0, String path = "", ParatooProtocolConfig config = null) { boolean modelNameStacked = false Map model = [dataModel: [], viewModel: []], dataType, viewModel, template @@ -1493,13 +1541,21 @@ class ParatooService { if (path) { newOrder["type"] = "object" // get model definition for the parent - def value = getProperty(properties, path) - // remove parent from children - paths?.each { String propertyPath -> - removeProperty(properties, propertyPath) + def value = [:] + try { + value = getProperty(properties, path)?.first() ?: [:] + // remove parent from children + paths?.each { String propertyPath -> + removeProperty(properties, propertyPath) + } + + value = deepCopy(value) + + } + catch (Exception e) { + log.info("Error getting property for path: ${path}") } - value = deepCopy(value) // reorder newOrder.properties = newOrder.properties ?: [:] newOrder.properties[parent] = newOrder.properties[parent] ?: [:] @@ -1513,10 +1569,6 @@ class ParatooService { newOrder.properties[parent].putAll((properties[parent] ?: [:])) } -// paths?.each { String propertyPath -> -// removeProperty(properties, propertyPath) -// } - if (children) { // if children are present, then recurse the process through each children newOrder.properties[parent] = newOrder.properties[parent] ?: [type: "object", properties: [:]] @@ -1528,6 +1580,24 @@ class ParatooService { newOrder } + def findObservationDataFromAPIOutput(String modelToFind, Map apiOutputRelationship, Map data) { + List paths = findPathFromRelationship(modelToFind, apiOutputRelationship) + String path = paths.size() ? paths.first() : null + if (path) { + path = path.replaceAll(".properties", "") + // get model definition for the parent + try { + return getProperty(data, path)?.first() + } + catch (Exception e) { + log.info("Error getting property for path: ${path}") + } + } + else { + return data[modelToFind] + } + } + /** * Remove a property at a given path. * i.e. if path is a.b.c and object is {a: {b: {c: [:]}}} then after removing the property, properties will be {a: {b: [:]}} @@ -1830,11 +1900,13 @@ class ParatooService { modelName?.toLowerCase()?.replaceAll("[^a-zA-Z0-9]+", ' ')?.tokenize(' ')?.collect { it.capitalize() }?.join() } - private static def getProperty(Map surveyData, String path) { - if (!path) { + private def getProperty(def surveyData, String path) { + if (!path || surveyData == null) { return null } - new PropertyAccessor(path).get(surveyData) + + List parts = path.split(/\./) + deepCopy(elasticSearchService.getDataFromPath(surveyData, parts)) } /** @@ -1846,6 +1918,10 @@ class ParatooService { * @return */ Map transformSpeciesName(String name) { + if (!name) { + return null + } + String regex = "\\(scientific:\\s*(.*?)\\)" Pattern pattern = Pattern.compile(regex) Matcher matcher = pattern.matcher(name) diff --git a/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy index b8fe8633b..23e07b096 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy @@ -23,7 +23,7 @@ class GenericFieldConverter implements RecordFieldConverter { record << getDwcAttributes(data, dwcMappings) - if (data.dwcAttribute) { + if (data?.dwcAttribute) { record[data.dwcAttribute] = data.value } @@ -34,9 +34,9 @@ class GenericFieldConverter implements RecordFieldConverter { private Double getLatitude(Map data) { Double latitude = null - if (data.decimalLatitude) { + if (data?.decimalLatitude) { latitude = toDouble(data.decimalLatitude) - } else if (data.locationLatitude) { + } else if (data?.locationLatitude) { latitude = toDouble(data.locationLatitude) } @@ -46,9 +46,9 @@ class GenericFieldConverter implements RecordFieldConverter { private Double getLongitude(Map data) { Double longitude = null - if (data.decimalLongitude) { + if (data?.decimalLongitude) { longitude = toDouble(data.decimalLongitude) - } else if (data.locationLongitude) { + } else if (data?.locationLongitude) { longitude = toDouble(data.locationLongitude) } diff --git a/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy index 8603d1203..daedd0ec0 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy @@ -13,6 +13,10 @@ class ListConverter implements RecordFieldConverter { // delegate the conversion of each column in each row to a specific converter for the column type data[outputMetadata.name].each { row -> + if (row == null) { + return + } + Map baseRecord = [:] List singleItemModels List multiItemModels diff --git a/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy index 5438e2327..a11f5a929 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy @@ -6,6 +6,10 @@ class SpeciesConverter implements RecordFieldConverter { ] List convert(Map data, Map metadata = [:]) { + if (data == null) { + return [] + } + Map record = [:] record.scientificNameID = record.guid = data[metadata.name].guid diff --git a/src/main/groovy/au/org/ala/ecodata/metadata/OutputMetadata.groovy b/src/main/groovy/au/org/ala/ecodata/metadata/OutputMetadata.groovy index 662f696f5..f9c0ff1e5 100644 --- a/src/main/groovy/au/org/ala/ecodata/metadata/OutputMetadata.groovy +++ b/src/main/groovy/au/org/ala/ecodata/metadata/OutputMetadata.groovy @@ -266,7 +266,6 @@ class OutputMetadata { context?.each { data -> if(isNestedDataModelType(data)){ String contextPath = "${path}.${data.name}" - log.info("${contextPath}") // recursive call for nested data model childrenNames = getNamesForDataType(type, getNestedDataModelNodes(data), depth + 1, contextPath); if(childrenNames?.size()){ diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 29436c0e4..6dcb1753b 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -1,17 +1,11 @@ package au.org.ala.ecodata.paratoo -import au.org.ala.ecodata.ActivityForm -import au.org.ala.ecodata.DateUtil -import au.org.ala.ecodata.FormSection -import au.org.ala.ecodata.GeometryUtils -import au.org.ala.ecodata.ParatooService +import au.org.ala.ecodata.* import au.org.ala.ecodata.metadata.OutputMetadata import au.org.ala.ecodata.metadata.PropertyAccessor import com.fasterxml.jackson.annotation.JsonIgnoreProperties import groovy.util.logging.Slf4j import org.locationtech.jts.geom.Geometry - -import java.util.regex.Matcher /** * Configuration about how to work with a Paratoo/Monitor protocol */ @@ -21,8 +15,6 @@ class ParatooProtocolConfig { String name String apiEndpoint - String observationEndpoint - String surveyType boolean usesPlotLayout = true List tags String geometryType = 'Polygon' @@ -31,12 +23,6 @@ class ParatooProtocolConfig { String startDatePath = 'attributes.start_date_time' String endDatePath = 'attributes.end_date_time' String surveyIdPath = 'attributes.survey_metadata' - String observationSurveyIdPath = '' - String observationSpeciesPath = 'attributes.species' - String observationRecordedByPath = 'attributes.observers.observer' - String observationIndividualCountPath = 'attributes.number_of_individuals' - String observationEventDatePath = 'attributes.date_time' - String observationGeometryPath = 'attributes.location' String plotVisitPath = 'attributes.plot_visit.data.attributes' String plotLayoutPath = "${plotVisitPath}.plot_layout.data.attributes" String plotLayoutIdPath = 'attributes.plot_visit.data.attributes.plot_layout.data.id' @@ -57,84 +43,36 @@ class ParatooProtocolConfig { } String getStartDate(Map surveyData) { - removeMilliseconds(getProperty(surveyData, startDatePath)) - } - - String getEndDate(Map surveyData) { - removeMilliseconds(getProperty(surveyData, endDatePath)) - } - - Map getSurveyId(Map surveyData) { - getProperty(surveyData, surveyIdPath) - } - - String getSpecies (Map observation) { - getProperty(observation, observationSpeciesPath) - } - - Map parseSpecies (String species) { - extractSpeciesName(species) - } - - List findObservationsBelongingToSurvey (List surveyData, ParatooCollectionId surveyId) { - surveyData?.findAll { Map data -> - Map dataLinkedToSurvey = getSurveyIdOfObservation( data ) - surveyEqualityTest(dataLinkedToSurvey, surveyId) + String date = getProperty(surveyData, startDatePath) + if (date == null) { + date = getPropertyFromSurvey(surveyData, startDatePath) } - } - Map getSurveyIdOfObservation (Map data) { - getProperty(data, observationSurveyIdPath) + removeMilliseconds(date) } - Map getEventDate (Map data) { - getProperty(data, observationEventDatePath) + def getPropertyFromSurvey(Map surveyData, String path) { + surveyData = getSurveyDataFromObservation(surveyData) + path = path.replaceFirst("^attributes.", '') + getProperty(surveyData, path) } - Double getDecimalLatitude (Map data) { - def shape = extractObservationSiteDataFromPath(data) - switch (shape.type) { - case 'Point': - return shape.coordinates[1] - } - } - - Double getDecimalLongitude (Map data) { - def shape = extractObservationSiteDataFromPath(data) - switch (shape.type) { - case 'Point': - return shape.coordinates[0] + String getEndDate(Map surveyData) { + String date = getProperty(surveyData, endDatePath) + if (date == null) { + date = getPropertyFromSurvey(surveyData, endDatePath) } + removeMilliseconds(date) } - Integer getIndividualCount (Map data) { - getProperty(data, observationIndividualCountPath) - } - - /** - * todo: this is a temporary hack to get around the fact that the survey doesn't have a field for this and it has to - * be interpreted from the protocol - * @param data - * @return - */ - String getIndividualOrGroup (Map data) { - 'Individuals' - } - - String getRecordedBy (Map data) { - def result = getProperty(data, observationRecordedByPath) - if (result instanceof List) { - return result.join(', ') - } - else if (result instanceof String) { - return result + Map getSurveyId(Map surveyData) { + Map result = getProperty(surveyData, surveyIdPath) + if (result == null) { + result = getPropertyFromSurvey(surveyData, surveyIdPath) } - } - private Map extractObservationSiteDataFromPath(Map surveyData) { - def geometryData = getProperty(surveyData, observationGeometryPath) - extractGeometryFromSiteData(geometryData) + result } private Map extractSiteDataFromPath(Map surveyData) { @@ -376,28 +314,4 @@ class ParatooProtocolConfig { ] geoJson } - - static Map extractSpeciesName (String species) { - Map result - Matcher m = (species =~ /([^\[]*)\[([^\]]*)\]\s*\(scientific: ([^\)]*)\)/) - if (m.matches()) { - def scientificName = m.group(3).trim() - def commonName = m.group(1)?.trim() - def name = "$scientificName ($commonName)" - result = [ - vernacularName: commonName, - scientificName: scientificName, - name: name - ] - } - else { - result = [ - vernacularName: species, - scientificName: species, - name: species - ] - } - - result - } } diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooControllerSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooControllerSpec.groovy index 158a13389..e1be30204 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooControllerSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooControllerSpec.groovy @@ -204,7 +204,7 @@ class ParatooControllerSpec extends Specification implements ControllerUnitTest< 1 * userService.currentUserDetails >> [userId:userId] 1 * paratooService.findDataSet(userId, collection.orgMintedUUID) >> searchResults 1 * paratooService.protocolWriteCheck(userId, 'p1', "guid-1") >> true - 1 * paratooService.submitCollection({it.orgMintedUUID == "c1"}, searchResults.project) >> [:] + 1 * paratooService.submitCollection({it.orgMintedUUID == "c1"}, searchResults.project) >> [updateResult: [:], promise: null] and: response.status == HttpStatus.SC_OK @@ -230,7 +230,7 @@ class ParatooControllerSpec extends Specification implements ControllerUnitTest< 1 * userService.currentUserDetails >> [userId:userId] 1 * paratooService.findDataSet(userId, collection.orgMintedUUID) >> searchResults 1 * paratooService.protocolWriteCheck(userId, 'p1', "guid-1") >> true - 1 * paratooService.submitCollection({it.orgMintedUUID == "c1"}, searchResults.project) >> [error:"Error"] + 1 * paratooService.submitCollection({it.orgMintedUUID == "c1"}, searchResults.project) >> [updateResult: [error:"Error"], promise: null] and: response.status == HttpStatus.SC_INTERNAL_SERVER_ERROR diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index d1dfad81e..a1137f185 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -10,6 +10,7 @@ import groovy.json.JsonSlurper import org.codehaus.jackson.map.ObjectMapper import org.grails.web.converters.marshaller.json.CollectionMarshaller import org.grails.web.converters.marshaller.json.MapMarshaller +import static grails.async.Promises.waitAll /** * Tests for the ParatooService. @@ -47,6 +48,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest= 0}, null, _, false) >> [data:[], meta:[pagination:[total:0]]] 1 * webService.doPost(*_) >> [resp: [collections: ["coarse-woody-debris-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z"]]]] - 2 * tokenService.getAuthToken(true) >> Mock(AccessToken) - 1 * projectService.update([custom: [dataSets: [expectedDataSet]]], 'p1', false) >> [status: 'ok'] + 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) + 1 * projectService.update([custom: [dataSets: [expectedDataSetAsync]]], 'p1', false) >> [status: 'ok'] + 1 * projectService.update([custom: [dataSets: [expectedDataSetSync]]], 'p1', false) >> [status: 'ok'] 1 * activityService.create(_) >> [activityId: '123'] 1 * recordService.getAllByActivity('123') >> [] 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { @@ -205,7 +210,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [userId: userId] and: - result == [status: 'ok'] + result.updateResult == [status: 'ok'] } void "The service can create a site from a submitted plot-selection"() { @@ -289,12 +294,13 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest= 0 }, null, _, false) >> [data: [surveyData], meta: [pagination: [total: 0]]] 1 * webService.doPost(*_) >> [resp: [collections: ["basal-area-dbh-measure-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z"]]]] - 2 * tokenService.getAuthToken(true) >> Mock(AccessToken) - 1 * projectService.update(_, projectId, false) >> [status: 'ok'] + 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) + 2 * projectService.update(_, projectId, false) >> [status: 'ok'] 1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] } 1 * activityService.create(_) >> [activityId: '123'] 1 * recordService.getAllByActivity('123') >> [] @@ -321,7 +327,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Thu, 28 Mar 2024 13:20:42 +1100 Subject: [PATCH 22/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixed test --- .../ala/ecodata/paratoo/ParatooProtocolConfig.groovy | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 6dcb1753b..aa0fbee11 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -43,6 +43,10 @@ class ParatooProtocolConfig { } String getStartDate(Map surveyData) { + if(startDatePath == null || surveyData == null) { + return null + } + String date = getProperty(surveyData, startDatePath) if (date == null) { date = getPropertyFromSurvey(surveyData, startDatePath) @@ -58,6 +62,10 @@ class ParatooProtocolConfig { } String getEndDate(Map surveyData) { + if(endDatePath == null || surveyData == null) { + return null + } + String date = getProperty(surveyData, endDatePath) if (date == null) { date = getPropertyFromSurvey(surveyData, endDatePath) @@ -67,6 +75,10 @@ class ParatooProtocolConfig { } Map getSurveyId(Map surveyData) { + if(surveyIdPath == null || surveyData == null) { + return null + } + Map result = getProperty(surveyData, surveyIdPath) if (result == null) { result = getPropertyFromSurvey(surveyData, surveyIdPath) From 36b9b2ddd087117a8732c2e1304e7a2e1942da53 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 28 Mar 2024 13:42:09 +1100 Subject: [PATCH 23/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixed borken test --- .../ecodata/paratoo/ParatooProtocolConfigSpec.groovy | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index f8f682029..ae021cc7b 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -116,8 +116,7 @@ class ParatooProtocolConfigSpec extends Specification { usesPlotLayout:false, geometryType: 'Point', startDatePath: 'attributes.startdate', - endDatePath: 'attributes.updatedAt', - observationSurveyIdPath: 'attributes.opportunistic_survey.data.attributes.survey_metadata' + endDatePath: 'attributes.updatedAt' ] ParatooProtocolConfig config = new ParatooProtocolConfig(opportunisticSurveyConfig) ParatooCollectionId paratooSurveyId = new ParatooCollectionId( @@ -129,12 +128,12 @@ class ParatooProtocolConfigSpec extends Specification { ] ] ) - def start_date = config.getStartDate(surveyObservations) - def end_date = config.getEndDate(surveyObservations) + def start_date = config.getStartDate(surveyObservations.data[0]) + def end_date = config.getEndDate(surveyObservations.data[0]) expect: - start_date != null - end_date != null + start_date == null + end_date == "2023-10-24T01:01:56Z" } } From 67d3aef18a5ce54fce1311a6949afc91a20f4c8a Mon Sep 17 00:00:00 2001 From: chrisala Date: Tue, 2 Apr 2024 13:46:47 +1100 Subject: [PATCH 24/52] Get site/dates from reverse lookup fieldcapture#3049 --- .../au/org/ala/ecodata/ParatooService.groovy | 54 +- .../paratoo/ParatooPlotSelection.groovy | 2 +- .../paratoo/ParatooProtocolConfig.groovy | 63 +- .../org/ala/ecodata/ParatooServiceSpec.groovy | 16 +- .../paratoo/basalAreaDbhReverseLookup.json | 1809 +++++++++++++++++ 5 files changed, 1861 insertions(+), 83 deletions(-) create mode 100644 src/test/resources/paratoo/basalAreaDbhReverseLookup.json diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 08aaba962..35964c921 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -259,20 +259,19 @@ class ParatooService { } else { ParatooCollectionId surveyId = ParatooCollectionId.fromMap(dataSet.surveyId) ParatooProtocolConfig config = getProtocolConfig(surveyId.protocolId) + config.surveyId = surveyId ActivityForm form = ActivityForm.findByExternalId(surveyId.protocolId) - // get survey data from reverse lookup response - Map surveyData = config.getSurveyDataFromObservation(surveyDataAndObservations) - // get plot data by querying protocol api endpoint - Map surveyDataWithPlotInfo = retrieveSurveyData(surveyId, config, surveyData.createdAt, authHeader) - // add plot data to survey observations - addPlotDataToObservations(surveyDataWithPlotInfo, surveyDataAndObservations, config) + + // addPlotDataToObservations mutates the data at the top level so a shallow copy is OK + Map surveyData = new HashMap(surveyDataAndObservations) + addPlotDataToObservations(surveyData, surveyDataAndObservations, config) rearrangeSurveyData(surveyDataAndObservations, surveyDataAndObservations, form.sections[0].template.relationships.ecodata, form.sections[0].template.relationships.apiOutput) // transform data to make it compatible with data model surveyDataAndObservations = recursivelyTransformData(form.sections[0].template.dataModel, surveyDataAndObservations) // If we are unable to create a site, null will be returned - assigning a null siteId is valid. - if (!dataSet.siteId) - dataSet.siteId = createSiteFromSurveyData(surveyDataWithPlotInfo, surveyDataAndObservations, collection, surveyId, project.project, config, form) - + if (!dataSet.siteId) { + dataSet.siteId = createSiteFromSurveyData(surveyData, surveyDataAndObservations, collection, surveyId, project.project, config, form) + } // make sure activity has not been created for this data set if (!dataSet.activityId || forceActivityCreation) { Activity.withSession { @@ -283,8 +282,8 @@ class ParatooService { } } - dataSet.startDate = config.getStartDate(surveyDataAndObservations) - dataSet.endDate = config.getEndDate(surveyDataAndObservations) + dataSet.startDate = config.getStartDate(surveyData) + dataSet.endDate = config.getEndDate(surveyData) projectService.update([custom: project.project.custom], project.id, false) } @@ -695,35 +694,6 @@ class ParatooService { "?populate=deep&sort=updatedAt&pagination[start]=$start&pagination[limit]=$limit&filters[createdAt][\$eq]=$createdAt" } - Map retrieveSurveyData(ParatooCollectionId surveyId, ParatooProtocolConfig config, String createdAt, Map authHeader = null) { - String apiEndpoint = config.getApiEndpoint(surveyId) - if (!authHeader) { - authHeader = getAuthHeader() - } - - int start = 0 - int limit = 10 - - String url = paratooBaseUrl + '/' + apiEndpoint - String query = buildSurveyQueryString(start, limit, createdAt) - log.debug("Retrieving survey data from: "+url+query) - Map response = webService.getJson(url + query, null, authHeader, false) - log.debug((response as JSON).toString()) - Map survey = findMatchingSurvey(surveyId, response.data, config) - - int total = response.meta?.pagination?.total ?: 0 - while (!survey && start + limit < total) { - start += limit - query = buildSurveyQueryString(start, limit, createdAt) - log.debug("Retrieving survey data from: "+url+query) - response = webService.getJson(url + query, null, authHeader, false) - log.debug((response as JSON).toString()) - survey = findMatchingSurvey(surveyId, response.data, config) - } - - survey - } - Map retrieveSurveyAndObservations(ParatooCollection collection, Map authHeader = null) { String apiEndpoint = PARATOO_DATA_PATH Map payload = [ @@ -741,10 +711,6 @@ class ParatooService { response?.resp?.collections } - private static Map findMatchingSurvey(ParatooCollectionId surveyId, List data, ParatooProtocolConfig config) { - data?.find { config.matches(it, surveyId) } - } - Map addOrUpdatePlotSelections(String userId, ParatooPlotSelectionData plotSelectionData) { List projects = userProjects(userId) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooPlotSelection.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooPlotSelection.groovy index 9a0146f9c..7796caf93 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooPlotSelection.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooPlotSelection.groovy @@ -13,7 +13,7 @@ class ParatooPlotSelectionData { String plot_label ParatooPlotSelectionLocation recommended_location String uuid - String comment + String comments } @JsonIgnoreProperties(['metaClass', 'errors', 'expandoMetaClass']) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index aa0fbee11..b55d56f7d 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -20,21 +20,23 @@ class ParatooProtocolConfig { String geometryType = 'Polygon' String geometryPath - String startDatePath = 'attributes.start_date_time' - String endDatePath = 'attributes.end_date_time' - String surveyIdPath = 'attributes.survey_metadata' - String plotVisitPath = 'attributes.plot_visit.data.attributes' - String plotLayoutPath = "${plotVisitPath}.plot_layout.data.attributes" - String plotLayoutIdPath = 'attributes.plot_visit.data.attributes.plot_layout.data.id' - String plotLayoutPointsPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_points' - String plotSelectionPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_selection.data.attributes' - String plotLayoutDimensionLabelPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_dimensions.data.attributes.label' - String plotLayoutTypeLabelPath = 'attributes.plot_visit.data.attributes.plot_layout.data.attributes.plot_type.data.attributes.label' + String startDatePath = 'start_date_time' + String endDatePath = 'end_date_time' + String surveyIdPath = 'survey_metadata' + String plotVisitPath = 'plot_visit' + String plotLayoutPath = "${plotVisitPath}.plot_layout" + String plotLayoutIdPath = "${plotLayoutPath}.id" + String plotLayoutPointsPath = "${plotLayoutPath}.plot_points" + String plotSelectionPath = "${plotLayoutPath}.plot_selection" + String plotLayoutDimensionLabelPath = "${plotLayoutPath}.plot_dimensions.label" + String plotLayoutTypeLabelPath = "${plotLayoutPath}.plot_type.label" String getApiEndpoint(ParatooCollectionId surveyId) { apiEndpoint ?: defaultEndpoint(surveyId) } Map overrides = [dataModel: [:], viewModel: [:]] + ParatooCollectionId surveyId + private static String removeMilliseconds(String isoDateWithMillis) { if (!isoDateWithMillis) { return isoDateWithMillis @@ -151,7 +153,11 @@ class ParatooProtocolConfig { apiEndpoint } - static def getProperty(Map surveyData, String path) { + def getProperty(Map surveyData, String path) { + if (surveyId) { + path = surveyId.survey_metadata.survey_details.survey_model+'.'+path + } + if (!path) { return null } @@ -167,7 +173,7 @@ class ParatooProtocolConfig { if (usesPlotLayout) { geoJson = extractSiteDataFromPlotVisit(survey) // get list of all features associated with observation - if (form && observation) { + if (geoJson && form && observation) { geoJson.features = extractFeatures(observation, form) } } @@ -188,32 +194,30 @@ class ParatooProtocolConfig { Map getPlotVisit (Map surveyData) { Map plotVisit = getProperty(surveyData, plotVisitPath) - List keys = plotVisit?.keySet().toList().minus(ParatooService.PARATOO_DATAMODEL_PLOT_LAYOUT) - Map result = [:] - keys.each { key -> - result[key] = plotVisit[key] - } - - result + copyWithExcludedProperty(plotVisit, ParatooService.PARATOO_DATAMODEL_PLOT_LAYOUT) } Map getPlotLayout (Map surveyData) { Map plotLayout = getProperty(surveyData, plotLayoutPath) - List keys = plotLayout?.keySet().toList().minus(ParatooService.PARATOO_DATAMODEL_PLOT_SELECTION) - Map result = [:] - keys.each { key -> - result[key] = plotLayout[key] - } - - result + copyWithExcludedProperty(plotLayout, ParatooService.PARATOO_DATAMODEL_PLOT_SELECTION) } Map getPlotSelection (Map surveyData) { Map plotSelection = getProperty(surveyData, plotSelectionPath) - List keys = plotSelection?.keySet().toList() + copyWithExcludedProperty(plotSelection) + } + + private Map copyWithExcludedProperty(Map map, String property = null) { + if (!map) { + return [:] + } + List keys = map.keySet().toList() + if (property) { + keys = keys.minus(property) + } Map result = [:] keys.each { key -> - result[key] = plotSelection[key] + result[key] = map[key] } result @@ -238,6 +242,7 @@ class ParatooProtocolConfig { } private Map extractSiteDataFromPlotVisit(Map survey) { + def plotLayoutId = getProperty(survey, plotLayoutIdPath) // Currently an int, may become uuid? if (!plotLayoutId) { @@ -322,7 +327,7 @@ class ParatooProtocolConfig { name : plotSelectionData.plot_label, externalId: plotSelectionData.uuid, description: plotSelectionData.plot_label, - notes: plotSelectionData.comment + notes: plotSelectionData.comments ] geoJson } diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index a1137f185..121e74d7c 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -188,7 +188,6 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest= 0}, null, _, false) >> [data:[], meta:[pagination:[total:0]]] 1 * webService.doPost(*_) >> [resp: [collections: ["coarse-woody-debris-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z"]]]] 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) 1 * projectService.update([custom: [dataSets: [expectedDataSetAsync]]], 'p1', false) >> [status: 'ok'] @@ -219,7 +218,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest= 0 }, null, _, false) >> [data: [surveyData], meta: [pagination: [total: 0]]] - 1 * webService.doPost(*_) >> [resp: [collections: ["basal-area-dbh-measure-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z"]]]] + 1 * webService.doPost(*_) >> [resp: surveyData] 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) 2 * projectService.update(_, projectId, false) >> [status: 'ok'] 1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] } @@ -319,12 +317,12 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [userId: userId] and: - site.name == "SATFLB0001 - Control (100 x 100)" - site.description == "SATFLB0001 - Control (100 x 100)" - site.notes == "some comment" + site.name == "CTMSEH4221 - Control (100 x 100)" + site.description == "CTMSEH4221 - Control (100 x 100)" + site.notes == "Test again 2024-03-26" site.type == "surveyArea" site.publicationStatus == "published" - site.externalIds[0].externalId == "4" + site.externalIds[0].externalId == "12" site.externalIds[0].idType == ExternalId.IdType.MONITOR_PLOT_GUID result.updateResult == [status: 'ok'] diff --git a/src/test/resources/paratoo/basalAreaDbhReverseLookup.json b/src/test/resources/paratoo/basalAreaDbhReverseLookup.json new file mode 100644 index 000000000..09731cb06 --- /dev/null +++ b/src/test/resources/paratoo/basalAreaDbhReverseLookup.json @@ -0,0 +1,1809 @@ +{ + "survey_metadata": { + "id": 8, + "org_minted_uuid": "9beeba20-374a-470a-b4f4-caed34e07572", + "createdAt": "2024-03-28T03:17:25.302Z", + "updatedAt": "2024-03-28T03:17:25.302Z", + "survey_details": { + "id": 59, + "survey_model": "basal-area-dbh-measure-survey", + "time": "2024-03-28T03:17:01.727Z", + "uuid": "263f710d-9af5-456a-9c68-b2b675763220", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "5005b0af-4360-4a8c-a203-b2c9e440547e", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 59, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": "0.1.0-a2ba7a48", + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": "Monitor FDCP API-production", + "system_org": null + } + }, + "collections": { + "basal-area-dbh-measure-survey": { + "id": 2, + "createdAt": "2024-03-28T03:17:22.980Z", + "updatedAt": "2024-03-28T03:17:22.980Z", + "plot_size": { + "id": 2, + "symbol": "P100", + "label": "100 x 100 m", + "description": "The 100 x 100 m (1 ha) core monitoring plot is the survey area for the Enhanced protocol.", + "uri": "", + "createdAt": "2024-03-26T02:39:43.499Z", + "updatedAt": "2024-03-26T02:39:43.499Z", + "survey_method": { + "id": 1, + "symbol": "T", + "label": "Transects", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:04.143Z", + "updatedAt": "2024-03-26T02:39:04.143Z" + }, + "transect_number": null + }, + "basal_dbh_instrument": { + "id": 1, + "symbol": "DIA", + "label": "Diameter Tape Measure", + "description": "The type of instrument used to measure the diameter of a stem/tree trunk, usually at 1.3 m height above ground as diameter at breast height (DBH).", + "uri": "https://linked.data.gov.au/def/nrm/237b799a-a1f3-5a1e-b88a-69429d05523a", + "createdAt": "2024-03-26T02:39:00.432Z", + "updatedAt": "2024-03-26T02:39:00.432Z" + }, + "plot_visit": { + "id": 12, + "start_date": "2024-03-26T03:03:26.025Z", + "end_date": null, + "visit_field_name": "Chris test 2024-03-26", + "createdAt": "2024-03-26T03:03:57.778Z", + "updatedAt": "2024-03-26T03:03:57.778Z", + "plot_layout": { + "id": 12, + "orientation": null, + "replicate": 1, + "is_the_plot_permanently_marked": false, + "createdAt": "2024-03-26T03:03:57.623Z", + "updatedAt": "2024-03-26T03:03:57.623Z", + "plot_selection": { + "id": 16, + "plot_label": "CTMSEH4221", + "uuid": "cf23aacf-1f02-43fb-8a78-559d3261838b", + "comments": "Test again 2024-03-26", + "date_time": "2024-03-26T03:01:18.936Z", + "createdAt": "2024-03-26T03:02:01.377Z", + "updatedAt": "2024-03-26T03:02:01.377Z", + "plot_name": { + "id": 16, + "unique_digits": "4221", + "state": { + "id": 1, + "symbol": "CT", + "label": "Australian Capital Territory", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/4253c9af-b390-5b69-aaf3-ce5cda0c36bf", + "createdAt": "2024-03-26T02:39:32.171Z", + "updatedAt": "2024-03-26T02:39:32.171Z" + }, + "program": { + "id": 9, + "symbol": "M", + "label": "MERIT", + "description": "A generic program for MERIT use, as they can have many 'programs' for a given plot", + "uri": "", + "createdAt": "2024-03-26T02:39:18.503Z", + "updatedAt": "2024-03-26T02:39:18.503Z" + }, + "bioregion": { + "id": 83, + "symbol": "SEH", + "label": "South Eastern Highlands", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:41.803Z", + "updatedAt": "2024-03-26T02:39:41.803Z", + "states": [ + { + "id": 2, + "symbol": "NS", + "label": "New South Wales", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/14b8c3fe-3a60-5f3a-bb7a-b9cf5df49b4e", + "createdAt": "2024-03-26T02:39:32.183Z", + "updatedAt": "2024-03-26T02:39:32.183Z" + }, + { + "id": 7, + "symbol": "VC", + "label": "Victoria", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/5e6382e6-d26c-5889-ba7e-f051d82d6173", + "createdAt": "2024-03-26T02:39:32.246Z", + "updatedAt": "2024-03-26T02:39:32.246Z" + }, + { + "id": 1, + "symbol": "CT", + "label": "Australian Capital Territory", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/4253c9af-b390-5b69-aaf3-ce5cda0c36bf", + "createdAt": "2024-03-26T02:39:32.171Z", + "updatedAt": "2024-03-26T02:39:32.171Z" + } + ] + } + }, + "recommended_location": { + "id": 34, + "lat": -35.2592561, + "lng": 149.0651502 + }, + "recommended_location_point": { + "id": 5, + "symbol": "C", + "label": "Centre", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:17.341Z", + "updatedAt": "2024-03-26T02:39:17.341Z" + }, + "plot_selection_survey": { + "id": 7, + "startdate": null, + "createdAt": "2024-03-26T03:02:00.063Z", + "updatedAt": "2024-03-26T03:02:00.063Z", + "survey_metadata": { + "id": 44, + "orgMintedUUID": null, + "survey_details": { + "id": 46, + "survey_model": "plot-selection-survey", + "time": "2024-03-26T03:01:57.093Z", + "uuid": "0d47c9c1-5842-4a02-b8f6-e95a92916b9f", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "a9cb9e38-690f-41c9-8151-06108caf539d", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 46, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": null, + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": null, + "system_org": null + } + } + } + }, + "plot_type": { + "id": 1, + "symbol": "C", + "label": "Control", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:18.129Z", + "updatedAt": "2024-03-26T02:39:18.129Z" + }, + "permanently_marked": null, + "plot_dimensions": { + "id": 1, + "symbol": "100m", + "label": "100 x 100", + "description": "100 m by 100 m (1 hectare)", + "uri": "", + "createdAt": "2024-03-26T02:39:17.254Z", + "updatedAt": "2024-03-26T02:39:17.254Z" + }, + "plot_points": [ + { + "id": 236, + "lat": -35.2592569, + "lng": 149.0651452, + "name": { + "id": 22, + "symbol": "SW", + "description": "South West", + "label": "South West", + "uri": "", + "createdAt": "2024-03-26T02:39:17.807Z", + "updatedAt": "2024-03-26T02:39:17.807Z" + } + }, + { + "id": 237, + "lat": -35.259167068471584, + "lng": 149.0651452, + "name": { + "id": 12, + "symbol": "W1", + "description": "West 1", + "label": "West 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.675Z", + "updatedAt": "2024-03-26T02:39:17.675Z" + } + }, + { + "id": 238, + "lat": -35.258987405414764, + "lng": 149.0651452, + "name": { + "id": 13, + "symbol": "W2", + "description": "West 2", + "label": "West 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.686Z", + "updatedAt": "2024-03-26T02:39:17.686Z" + } + }, + { + "id": 239, + "lat": -35.25880774235794, + "lng": 149.0651452, + "name": { + "id": 14, + "symbol": "W3", + "description": "West 3", + "label": "West 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.700Z", + "updatedAt": "2024-03-26T02:39:17.700Z" + } + }, + { + "id": 240, + "lat": -35.25862807930111, + "lng": 149.0651452, + "name": { + "id": 15, + "symbol": "W4", + "description": "West 4", + "label": "West 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.713Z", + "updatedAt": "2024-03-26T02:39:17.713Z" + } + }, + { + "id": 241, + "lat": -35.25844841624429, + "lng": 149.0651452, + "name": { + "id": 16, + "symbol": "W5", + "description": "West 5", + "label": "West 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.732Z", + "updatedAt": "2024-03-26T02:39:17.732Z" + } + }, + { + "id": 242, + "lat": -35.25835858471588, + "lng": 149.0651452, + "name": { + "id": 24, + "symbol": "NW", + "description": "North West", + "label": "North West", + "uri": "", + "createdAt": "2024-03-26T02:39:17.833Z", + "updatedAt": "2024-03-26T02:39:17.833Z" + } + }, + { + "id": 243, + "lat": -35.25835858471588, + "lng": 149.06525521373527, + "name": { + "id": 17, + "symbol": "N1", + "description": "North 1", + "label": "North 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.743Z", + "updatedAt": "2024-03-26T02:39:17.743Z" + } + }, + { + "id": 244, + "lat": -35.25835858471588, + "lng": 149.06547524120586, + "name": { + "id": 18, + "symbol": "N2", + "description": "North 2", + "label": "North 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.752Z", + "updatedAt": "2024-03-26T02:39:17.752Z" + } + }, + { + "id": 245, + "lat": -35.25835858471588, + "lng": 149.06569526867642, + "name": { + "id": 19, + "symbol": "N3", + "description": "North 3", + "label": "North 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.764Z", + "updatedAt": "2024-03-26T02:39:17.764Z" + } + }, + { + "id": 246, + "lat": -35.25835858471588, + "lng": 149.06591529614698, + "name": { + "id": 20, + "symbol": "N4", + "description": "North 4", + "label": "North 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.783Z", + "updatedAt": "2024-03-26T02:39:17.783Z" + } + }, + { + "id": 247, + "lat": -35.25835858471588, + "lng": 149.06613532361757, + "name": { + "id": 21, + "symbol": "N5", + "description": "North 5", + "label": "North 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.794Z", + "updatedAt": "2024-03-26T02:39:17.794Z" + } + }, + { + "id": 248, + "lat": -35.25835858471588, + "lng": 149.06624533735285, + "name": { + "id": 25, + "symbol": "NE", + "description": "North East", + "label": "North East", + "uri": "", + "createdAt": "2024-03-26T02:39:17.844Z", + "updatedAt": "2024-03-26T02:39:17.844Z" + } + }, + { + "id": 249, + "lat": -35.25844841624429, + "lng": 149.06624533735285, + "name": { + "id": 6, + "symbol": "E5", + "description": "East 5", + "label": "East 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.608Z", + "updatedAt": "2024-03-26T02:39:17.608Z" + } + }, + { + "id": 250, + "lat": -35.25862807930111, + "lng": 149.06624533735285, + "name": { + "id": 5, + "symbol": "E4", + "description": "East 4", + "label": "East 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.598Z", + "updatedAt": "2024-03-26T02:39:17.598Z" + } + }, + { + "id": 251, + "lat": -35.25880774235794, + "lng": 149.06624533735285, + "name": { + "id": 4, + "symbol": "E3", + "description": "East 3", + "label": "East 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.585Z", + "updatedAt": "2024-03-26T02:39:17.585Z" + } + }, + { + "id": 252, + "lat": -35.258987405414764, + "lng": 149.06624533735285, + "name": { + "id": 3, + "symbol": "E2", + "description": "East 2", + "label": "East 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.575Z", + "updatedAt": "2024-03-26T02:39:17.575Z" + } + }, + { + "id": 253, + "lat": -35.259167068471584, + "lng": 149.06624533735285, + "name": { + "id": 2, + "symbol": "E1", + "description": "East 1", + "label": "East 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.564Z", + "updatedAt": "2024-03-26T02:39:17.564Z" + } + }, + { + "id": 254, + "lat": -35.2592569, + "lng": 149.06624533735285, + "name": { + "id": 23, + "symbol": "SE", + "description": "South East", + "label": "South East", + "uri": "", + "createdAt": "2024-03-26T02:39:17.821Z", + "updatedAt": "2024-03-26T02:39:17.821Z" + } + }, + { + "id": 255, + "lat": -35.2592569, + "lng": 149.06613532361757, + "name": { + "id": 11, + "symbol": "S5", + "description": "South 5", + "label": "South 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.664Z", + "updatedAt": "2024-03-26T02:39:17.664Z" + } + }, + { + "id": 256, + "lat": -35.2592569, + "lng": 149.06591529614698, + "name": { + "id": 10, + "symbol": "S4", + "description": "South 4", + "label": "South 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.654Z", + "updatedAt": "2024-03-26T02:39:17.654Z" + } + }, + { + "id": 257, + "lat": -35.2592569, + "lng": 149.06569526867642, + "name": { + "id": 9, + "symbol": "S3", + "description": "South 3", + "label": "South 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.643Z", + "updatedAt": "2024-03-26T02:39:17.643Z" + } + }, + { + "id": 258, + "lat": -35.2592569, + "lng": 149.06547524120586, + "name": { + "id": 8, + "symbol": "S2", + "description": "South 2", + "label": "South 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.629Z", + "updatedAt": "2024-03-26T02:39:17.629Z" + } + }, + { + "id": 259, + "lat": -35.2592569, + "lng": 149.06525521373527, + "name": { + "id": 7, + "symbol": "S1", + "description": "South 1", + "label": "South 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.619Z", + "updatedAt": "2024-03-26T02:39:17.619Z" + } + }, + { + "id": 260, + "lat": -35.25880774235794, + "lng": 149.06569526867645, + "name": { + "id": 1, + "symbol": "C", + "description": "Centre", + "label": "Centre", + "uri": "", + "createdAt": "2024-03-26T02:39:17.548Z", + "updatedAt": "2024-03-26T02:39:17.548Z" + } + } + ], + "fauna_plot_point": [] + } + }, + "survey_metadata": { + "id": 51, + "orgMintedUUID": "9beeba20-374a-470a-b4f4-caed34e07572", + "survey_details": { + "id": 58, + "survey_model": "basal-area-dbh-measure-survey", + "time": "2024-03-28T03:17:01.727Z", + "uuid": "263f710d-9af5-456a-9c68-b2b675763220", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "5005b0af-4360-4a8c-a203-b2c9e440547e", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 58, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": "0.1.0-a2ba7a48", + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": "Monitor FDCP API-production", + "system_org": null + } + } + }, + "basal-area-dbh-measure-observation": [ + { + "id": 2, + "dead": false, + "multi_stemmed": false, + "ellipse": false, + "buttresses": false, + "date_time": "2024-03-28T03:17:01.764Z", + "createdAt": "2024-03-28T03:17:23.160Z", + "updatedAt": "2024-03-28T03:17:23.160Z", + "location": { + "id": 37, + "lat": -35.2592444, + "lng": 149.0651491 + }, + "floristics_voucher": { + "id": 17, + "voucher_full": null, + "voucher_lite": { + "id": 4, + "field_name": "Plantae [Regnum] (scientific: Plantae Haeckel)", + "voucher_barcode": "123456", + "host_species": null, + "unique_id": null, + "date_time": "2024-03-26T03:06:02.937Z", + "createdAt": "2024-03-26T03:07:16.452Z", + "updatedAt": "2024-03-26T03:07:16.452Z", + "growth_form_1": { + "id": 1, + "symbol": "V", + "label": "Sedge", + "description": "Herbaceous, usually perennial erect plant generally with a tufted habit and of the families Cyperaceae (true sedges) or Restionaceae (node sedges).", + "uri": "https://linked.data.gov.au/def/nrm/a7b8b3c8-a329-5a62-8741-e1d6efe9528b", + "createdAt": "2024-03-26T02:39:35.671Z", + "updatedAt": "2024-03-26T02:39:35.671Z" + }, + "growth_form_2": { + "id": 1, + "symbol": "V", + "label": "Sedge", + "description": "Herbaceous, usually perennial erect plant generally with a tufted habit and of the families Cyperaceae (true sedges) or Restionaceae (node sedges).", + "uri": "https://linked.data.gov.au/def/nrm/a7b8b3c8-a329-5a62-8741-e1d6efe9528b", + "createdAt": "2024-03-26T02:39:35.671Z", + "updatedAt": "2024-03-26T02:39:35.671Z" + }, + "habit": [ + { + "id": 1, + "lut": { + "id": 1, + "symbol": "ASC", + "label": "Ascending", + "description": "Weakly erect", + "uri": "", + "createdAt": "2024-03-26T02:39:07.482Z", + "updatedAt": "2024-03-26T02:39:07.482Z" + } + } + ], + "phenology": [ + { + "id": 1, + "lut": { + "id": 1, + "symbol": "FLO", + "label": "Flowers", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:07.681Z", + "updatedAt": "2024-03-26T02:39:07.681Z" + } + } + ], + "floristics_veg_survey_lite": { + "id": 4, + "collect_ptv_submodule": false, + "createdAt": "2024-03-26T03:07:16.263Z", + "updatedAt": "2024-03-26T03:07:16.263Z", + "plot_visit": { + "id": 12, + "start_date": "2024-03-26T03:03:26.025Z", + "end_date": null, + "visit_field_name": "Chris test 2024-03-26", + "createdAt": "2024-03-26T03:03:57.778Z", + "updatedAt": "2024-03-26T03:03:57.778Z", + "plot_layout": { + "id": 12, + "orientation": null, + "replicate": 1, + "is_the_plot_permanently_marked": false, + "createdAt": "2024-03-26T03:03:57.623Z", + "updatedAt": "2024-03-26T03:03:57.623Z", + "plot_selection": { + "id": 16, + "plot_label": "CTMSEH4221", + "uuid": "cf23aacf-1f02-43fb-8a78-559d3261838b", + "comments": "Test again 2024-03-26", + "date_time": "2024-03-26T03:01:18.936Z", + "createdAt": "2024-03-26T03:02:01.377Z", + "updatedAt": "2024-03-26T03:02:01.377Z", + "plot_name": { + "id": 16, + "unique_digits": "4221", + "state": { + "id": 1, + "symbol": "CT", + "label": "Australian Capital Territory", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/4253c9af-b390-5b69-aaf3-ce5cda0c36bf", + "createdAt": "2024-03-26T02:39:32.171Z", + "updatedAt": "2024-03-26T02:39:32.171Z", + "createdBy": null, + "updatedBy": null + }, + "program": { + "id": 9, + "symbol": "M", + "label": "MERIT", + "description": "A generic program for MERIT use, as they can have many 'programs' for a given plot", + "uri": "", + "createdAt": "2024-03-26T02:39:18.503Z", + "updatedAt": "2024-03-26T02:39:18.503Z", + "createdBy": null, + "updatedBy": null + }, + "bioregion": { + "id": 83, + "symbol": "SEH", + "label": "South Eastern Highlands", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:41.803Z", + "updatedAt": "2024-03-26T02:39:41.803Z", + "states": [ + { + "id": 2, + "symbol": "NS", + "label": "New South Wales", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/14b8c3fe-3a60-5f3a-bb7a-b9cf5df49b4e", + "createdAt": "2024-03-26T02:39:32.183Z", + "updatedAt": "2024-03-26T02:39:32.183Z" + }, + { + "id": 7, + "symbol": "VC", + "label": "Victoria", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/5e6382e6-d26c-5889-ba7e-f051d82d6173", + "createdAt": "2024-03-26T02:39:32.246Z", + "updatedAt": "2024-03-26T02:39:32.246Z" + }, + { + "id": 1, + "symbol": "CT", + "label": "Australian Capital Territory", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/4253c9af-b390-5b69-aaf3-ce5cda0c36bf", + "createdAt": "2024-03-26T02:39:32.171Z", + "updatedAt": "2024-03-26T02:39:32.171Z" + } + ], + "createdBy": null, + "updatedBy": null + } + }, + "recommended_location": { + "id": 34, + "lat": -35.2592561, + "lng": 149.0651502 + }, + "recommended_location_point": { + "id": 5, + "symbol": "C", + "label": "Centre", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:17.341Z", + "updatedAt": "2024-03-26T02:39:17.341Z" + }, + "plot_selection_survey": { + "id": 7, + "startdate": null, + "createdAt": "2024-03-26T03:02:00.063Z", + "updatedAt": "2024-03-26T03:02:00.063Z", + "survey_metadata": { + "id": 44, + "orgMintedUUID": null, + "survey_details": { + "id": 46, + "survey_model": "plot-selection-survey", + "time": "2024-03-26T03:01:57.093Z", + "uuid": "0d47c9c1-5842-4a02-b8f6-e95a92916b9f", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "a9cb9e38-690f-41c9-8151-06108caf539d", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 46, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": null, + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": null, + "system_org": null + } + } + } + }, + "plot_type": { + "id": 1, + "symbol": "C", + "label": "Control", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:18.129Z", + "updatedAt": "2024-03-26T02:39:18.129Z" + }, + "permanently_marked": null, + "plot_dimensions": { + "id": 1, + "symbol": "100m", + "label": "100 x 100", + "description": "100 m by 100 m (1 hectare)", + "uri": "", + "createdAt": "2024-03-26T02:39:17.254Z", + "updatedAt": "2024-03-26T02:39:17.254Z" + }, + "plot_points": [ + { + "id": 236, + "lat": -35.2592569, + "lng": 149.0651452, + "name": { + "id": 22, + "symbol": "SW", + "description": "South West", + "label": "South West", + "uri": "", + "createdAt": "2024-03-26T02:39:17.807Z", + "updatedAt": "2024-03-26T02:39:17.807Z" + } + }, + { + "id": 237, + "lat": -35.259167068471584, + "lng": 149.0651452, + "name": { + "id": 12, + "symbol": "W1", + "description": "West 1", + "label": "West 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.675Z", + "updatedAt": "2024-03-26T02:39:17.675Z" + } + }, + { + "id": 238, + "lat": -35.258987405414764, + "lng": 149.0651452, + "name": { + "id": 13, + "symbol": "W2", + "description": "West 2", + "label": "West 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.686Z", + "updatedAt": "2024-03-26T02:39:17.686Z" + } + }, + { + "id": 239, + "lat": -35.25880774235794, + "lng": 149.0651452, + "name": { + "id": 14, + "symbol": "W3", + "description": "West 3", + "label": "West 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.700Z", + "updatedAt": "2024-03-26T02:39:17.700Z" + } + }, + { + "id": 240, + "lat": -35.25862807930111, + "lng": 149.0651452, + "name": { + "id": 15, + "symbol": "W4", + "description": "West 4", + "label": "West 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.713Z", + "updatedAt": "2024-03-26T02:39:17.713Z" + } + }, + { + "id": 241, + "lat": -35.25844841624429, + "lng": 149.0651452, + "name": { + "id": 16, + "symbol": "W5", + "description": "West 5", + "label": "West 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.732Z", + "updatedAt": "2024-03-26T02:39:17.732Z" + } + }, + { + "id": 242, + "lat": -35.25835858471588, + "lng": 149.0651452, + "name": { + "id": 24, + "symbol": "NW", + "description": "North West", + "label": "North West", + "uri": "", + "createdAt": "2024-03-26T02:39:17.833Z", + "updatedAt": "2024-03-26T02:39:17.833Z" + } + }, + { + "id": 243, + "lat": -35.25835858471588, + "lng": 149.06525521373527, + "name": { + "id": 17, + "symbol": "N1", + "description": "North 1", + "label": "North 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.743Z", + "updatedAt": "2024-03-26T02:39:17.743Z" + } + }, + { + "id": 244, + "lat": -35.25835858471588, + "lng": 149.06547524120586, + "name": { + "id": 18, + "symbol": "N2", + "description": "North 2", + "label": "North 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.752Z", + "updatedAt": "2024-03-26T02:39:17.752Z" + } + }, + { + "id": 245, + "lat": -35.25835858471588, + "lng": 149.06569526867642, + "name": { + "id": 19, + "symbol": "N3", + "description": "North 3", + "label": "North 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.764Z", + "updatedAt": "2024-03-26T02:39:17.764Z" + } + }, + { + "id": 246, + "lat": -35.25835858471588, + "lng": 149.06591529614698, + "name": { + "id": 20, + "symbol": "N4", + "description": "North 4", + "label": "North 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.783Z", + "updatedAt": "2024-03-26T02:39:17.783Z" + } + }, + { + "id": 247, + "lat": -35.25835858471588, + "lng": 149.06613532361757, + "name": { + "id": 21, + "symbol": "N5", + "description": "North 5", + "label": "North 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.794Z", + "updatedAt": "2024-03-26T02:39:17.794Z" + } + }, + { + "id": 248, + "lat": -35.25835858471588, + "lng": 149.06624533735285, + "name": { + "id": 25, + "symbol": "NE", + "description": "North East", + "label": "North East", + "uri": "", + "createdAt": "2024-03-26T02:39:17.844Z", + "updatedAt": "2024-03-26T02:39:17.844Z" + } + }, + { + "id": 249, + "lat": -35.25844841624429, + "lng": 149.06624533735285, + "name": { + "id": 6, + "symbol": "E5", + "description": "East 5", + "label": "East 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.608Z", + "updatedAt": "2024-03-26T02:39:17.608Z" + } + }, + { + "id": 250, + "lat": -35.25862807930111, + "lng": 149.06624533735285, + "name": { + "id": 5, + "symbol": "E4", + "description": "East 4", + "label": "East 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.598Z", + "updatedAt": "2024-03-26T02:39:17.598Z" + } + }, + { + "id": 251, + "lat": -35.25880774235794, + "lng": 149.06624533735285, + "name": { + "id": 4, + "symbol": "E3", + "description": "East 3", + "label": "East 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.585Z", + "updatedAt": "2024-03-26T02:39:17.585Z" + } + }, + { + "id": 252, + "lat": -35.258987405414764, + "lng": 149.06624533735285, + "name": { + "id": 3, + "symbol": "E2", + "description": "East 2", + "label": "East 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.575Z", + "updatedAt": "2024-03-26T02:39:17.575Z" + } + }, + { + "id": 253, + "lat": -35.259167068471584, + "lng": 149.06624533735285, + "name": { + "id": 2, + "symbol": "E1", + "description": "East 1", + "label": "East 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.564Z", + "updatedAt": "2024-03-26T02:39:17.564Z" + } + }, + { + "id": 254, + "lat": -35.2592569, + "lng": 149.06624533735285, + "name": { + "id": 23, + "symbol": "SE", + "description": "South East", + "label": "South East", + "uri": "", + "createdAt": "2024-03-26T02:39:17.821Z", + "updatedAt": "2024-03-26T02:39:17.821Z" + } + }, + { + "id": 255, + "lat": -35.2592569, + "lng": 149.06613532361757, + "name": { + "id": 11, + "symbol": "S5", + "description": "South 5", + "label": "South 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.664Z", + "updatedAt": "2024-03-26T02:39:17.664Z" + } + }, + { + "id": 256, + "lat": -35.2592569, + "lng": 149.06591529614698, + "name": { + "id": 10, + "symbol": "S4", + "description": "South 4", + "label": "South 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.654Z", + "updatedAt": "2024-03-26T02:39:17.654Z" + } + }, + { + "id": 257, + "lat": -35.2592569, + "lng": 149.06569526867642, + "name": { + "id": 9, + "symbol": "S3", + "description": "South 3", + "label": "South 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.643Z", + "updatedAt": "2024-03-26T02:39:17.643Z" + } + }, + { + "id": 258, + "lat": -35.2592569, + "lng": 149.06547524120586, + "name": { + "id": 8, + "symbol": "S2", + "description": "South 2", + "label": "South 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.629Z", + "updatedAt": "2024-03-26T02:39:17.629Z" + } + }, + { + "id": 259, + "lat": -35.2592569, + "lng": 149.06525521373527, + "name": { + "id": 7, + "symbol": "S1", + "description": "South 1", + "label": "South 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.619Z", + "updatedAt": "2024-03-26T02:39:17.619Z" + } + }, + { + "id": 260, + "lat": -35.25880774235794, + "lng": 149.06569526867645, + "name": { + "id": 1, + "symbol": "C", + "description": "Centre", + "label": "Centre", + "uri": "", + "createdAt": "2024-03-26T02:39:17.548Z", + "updatedAt": "2024-03-26T02:39:17.548Z" + } + } + ], + "fauna_plot_point": [] + } + }, + "ptv_protocol_variant": null, + "survey_metadata": { + "id": 46, + "orgMintedUUID": "3eef6171-682f-49a4-ab03-6663f94e6564", + "survey_details": { + "id": 49, + "survey_model": "floristics-veg-survey-lite", + "time": "2024-03-26T03:06:02.904Z", + "uuid": "90df7817-1b5e-410c-95d1-09e223a13002", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "bbd550c0-04c5-4a8c-ae39-cc748e920fd4", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 49, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": "0.1.0-a2ba7a48", + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": "Monitor FDCP API-production", + "system_org": null + } + } + }, + "photo": [ + { + "id": 13, + "comment": null, + "single_photo": { + "id": 12, + "name": "1269ecd3-a2e4-49c2-b46a-547c528c108f.png", + "alternativeText": null, + "caption": null, + "width": 284, + "height": 58, + "formats": { + "thumbnail": { + "ext": ".png", + "url": "https://beta-monitor-binary.s3.ap-southeast-2.amazonaws.com/thumbnail_1269ecd3_a2e4_49c2_b46a_547c528c108f_477fc3ac7b.png", + "hash": "thumbnail_1269ecd3_a2e4_49c2_b46a_547c528c108f_477fc3ac7b", + "mime": "image/png", + "name": "thumbnail_1269ecd3-a2e4-49c2-b46a-547c528c108f.png", + "path": null, + "size": 7.38, + "width": 245, + "height": 50, + "sizeInBytes": 7375 + } + }, + "hash": "1269ecd3_a2e4_49c2_b46a_547c528c108f_477fc3ac7b", + "ext": ".png", + "mime": "image/png", + "size": 1.33, + "url": "https://beta-monitor-binary.s3.ap-southeast-2.amazonaws.com/1269ecd3_a2e4_49c2_b46a_547c528c108f_477fc3ac7b.png", + "previewUrl": null, + "provider": "aws-s3", + "provider_metadata": null, + "folderPath": "/1", + "createdAt": "2024-03-26T03:07:14.181Z", + "updatedAt": "2024-03-26T03:07:14.181Z" + } + } + ] + } + }, + "stem_1": { + "id": 2, + "DBH": 3, + "POM_single": 1.3, + "circumference_at_breast_height": null, + "ellipse_DBH_measurement": null + }, + "stem": [], + "buttressed_tree": null, + "basal_area_dbh_measure_survey_full": { + "id": 2, + "createdAt": "2024-03-28T03:17:22.980Z", + "updatedAt": "2024-03-28T03:17:22.980Z", + "plot_size": { + "id": 2, + "symbol": "P100", + "label": "100 x 100 m", + "description": "The 100 x 100 m (1 ha) core monitoring plot is the survey area for the Enhanced protocol.", + "uri": "", + "createdAt": "2024-03-26T02:39:43.499Z", + "updatedAt": "2024-03-26T02:39:43.499Z", + "survey_method": { + "id": 1, + "symbol": "T", + "label": "Transects", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:04.143Z", + "updatedAt": "2024-03-26T02:39:04.143Z" + }, + "transect_number": null + }, + "basal_dbh_instrument": { + "id": 1, + "symbol": "DIA", + "label": "Diameter Tape Measure", + "description": "The type of instrument used to measure the diameter of a stem/tree trunk, usually at 1.3 m height above ground as diameter at breast height (DBH).", + "uri": "https://linked.data.gov.au/def/nrm/237b799a-a1f3-5a1e-b88a-69429d05523a", + "createdAt": "2024-03-26T02:39:00.432Z", + "updatedAt": "2024-03-26T02:39:00.432Z" + }, + "plot_visit": { + "id": 12, + "start_date": "2024-03-26T03:03:26.025Z", + "end_date": null, + "visit_field_name": "Chris test 2024-03-26", + "createdAt": "2024-03-26T03:03:57.778Z", + "updatedAt": "2024-03-26T03:03:57.778Z", + "plot_layout": { + "id": 12, + "orientation": null, + "replicate": 1, + "is_the_plot_permanently_marked": false, + "createdAt": "2024-03-26T03:03:57.623Z", + "updatedAt": "2024-03-26T03:03:57.623Z", + "plot_selection": { + "id": 16, + "plot_label": "CTMSEH4221", + "uuid": "cf23aacf-1f02-43fb-8a78-559d3261838b", + "comments": "Test again 2024-03-26", + "date_time": "2024-03-26T03:01:18.936Z", + "createdAt": "2024-03-26T03:02:01.377Z", + "updatedAt": "2024-03-26T03:02:01.377Z", + "plot_name": { + "id": 16, + "unique_digits": "4221", + "state": { + "id": 1, + "symbol": "CT", + "label": "Australian Capital Territory", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/4253c9af-b390-5b69-aaf3-ce5cda0c36bf", + "createdAt": "2024-03-26T02:39:32.171Z", + "updatedAt": "2024-03-26T02:39:32.171Z" + }, + "program": { + "id": 9, + "symbol": "M", + "label": "MERIT", + "description": "A generic program for MERIT use, as they can have many 'programs' for a given plot", + "uri": "", + "createdAt": "2024-03-26T02:39:18.503Z", + "updatedAt": "2024-03-26T02:39:18.503Z" + }, + "bioregion": { + "id": 83, + "symbol": "SEH", + "label": "South Eastern Highlands", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:41.803Z", + "updatedAt": "2024-03-26T02:39:41.803Z", + "states": [ + { + "id": 2, + "symbol": "NS", + "label": "New South Wales", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/14b8c3fe-3a60-5f3a-bb7a-b9cf5df49b4e", + "createdAt": "2024-03-26T02:39:32.183Z", + "updatedAt": "2024-03-26T02:39:32.183Z" + }, + { + "id": 7, + "symbol": "VC", + "label": "Victoria", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/5e6382e6-d26c-5889-ba7e-f051d82d6173", + "createdAt": "2024-03-26T02:39:32.246Z", + "updatedAt": "2024-03-26T02:39:32.246Z" + }, + { + "id": 1, + "symbol": "CT", + "label": "Australian Capital Territory", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/4253c9af-b390-5b69-aaf3-ce5cda0c36bf", + "createdAt": "2024-03-26T02:39:32.171Z", + "updatedAt": "2024-03-26T02:39:32.171Z" + } + ] + } + }, + "recommended_location": { + "id": 34, + "lat": -35.2592561, + "lng": 149.0651502 + }, + "recommended_location_point": { + "id": 5, + "symbol": "C", + "label": "Centre", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:17.341Z", + "updatedAt": "2024-03-26T02:39:17.341Z" + }, + "plot_selection_survey": { + "id": 7, + "startdate": null, + "createdAt": "2024-03-26T03:02:00.063Z", + "updatedAt": "2024-03-26T03:02:00.063Z", + "survey_metadata": { + "id": 44, + "orgMintedUUID": null, + "survey_details": { + "id": 46, + "survey_model": "plot-selection-survey", + "time": "2024-03-26T03:01:57.093Z", + "uuid": "0d47c9c1-5842-4a02-b8f6-e95a92916b9f", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "a9cb9e38-690f-41c9-8151-06108caf539d", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 46, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": null, + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": null, + "system_org": null + } + } + } + }, + "plot_type": { + "id": 1, + "symbol": "C", + "label": "Control", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:18.129Z", + "updatedAt": "2024-03-26T02:39:18.129Z" + }, + "permanently_marked": null, + "plot_dimensions": { + "id": 1, + "symbol": "100m", + "label": "100 x 100", + "description": "100 m by 100 m (1 hectare)", + "uri": "", + "createdAt": "2024-03-26T02:39:17.254Z", + "updatedAt": "2024-03-26T02:39:17.254Z" + }, + "plot_points": [ + { + "id": 236, + "lat": -35.2592569, + "lng": 149.0651452, + "name": { + "id": 22, + "symbol": "SW", + "description": "South West", + "label": "South West", + "uri": "", + "createdAt": "2024-03-26T02:39:17.807Z", + "updatedAt": "2024-03-26T02:39:17.807Z" + } + }, + { + "id": 237, + "lat": -35.259167068471584, + "lng": 149.0651452, + "name": { + "id": 12, + "symbol": "W1", + "description": "West 1", + "label": "West 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.675Z", + "updatedAt": "2024-03-26T02:39:17.675Z" + } + }, + { + "id": 238, + "lat": -35.258987405414764, + "lng": 149.0651452, + "name": { + "id": 13, + "symbol": "W2", + "description": "West 2", + "label": "West 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.686Z", + "updatedAt": "2024-03-26T02:39:17.686Z" + } + }, + { + "id": 239, + "lat": -35.25880774235794, + "lng": 149.0651452, + "name": { + "id": 14, + "symbol": "W3", + "description": "West 3", + "label": "West 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.700Z", + "updatedAt": "2024-03-26T02:39:17.700Z" + } + }, + { + "id": 240, + "lat": -35.25862807930111, + "lng": 149.0651452, + "name": { + "id": 15, + "symbol": "W4", + "description": "West 4", + "label": "West 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.713Z", + "updatedAt": "2024-03-26T02:39:17.713Z" + } + }, + { + "id": 241, + "lat": -35.25844841624429, + "lng": 149.0651452, + "name": { + "id": 16, + "symbol": "W5", + "description": "West 5", + "label": "West 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.732Z", + "updatedAt": "2024-03-26T02:39:17.732Z" + } + }, + { + "id": 242, + "lat": -35.25835858471588, + "lng": 149.0651452, + "name": { + "id": 24, + "symbol": "NW", + "description": "North West", + "label": "North West", + "uri": "", + "createdAt": "2024-03-26T02:39:17.833Z", + "updatedAt": "2024-03-26T02:39:17.833Z" + } + }, + { + "id": 243, + "lat": -35.25835858471588, + "lng": 149.06525521373527, + "name": { + "id": 17, + "symbol": "N1", + "description": "North 1", + "label": "North 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.743Z", + "updatedAt": "2024-03-26T02:39:17.743Z" + } + }, + { + "id": 244, + "lat": -35.25835858471588, + "lng": 149.06547524120586, + "name": { + "id": 18, + "symbol": "N2", + "description": "North 2", + "label": "North 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.752Z", + "updatedAt": "2024-03-26T02:39:17.752Z" + } + }, + { + "id": 245, + "lat": -35.25835858471588, + "lng": 149.06569526867642, + "name": { + "id": 19, + "symbol": "N3", + "description": "North 3", + "label": "North 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.764Z", + "updatedAt": "2024-03-26T02:39:17.764Z" + } + }, + { + "id": 246, + "lat": -35.25835858471588, + "lng": 149.06591529614698, + "name": { + "id": 20, + "symbol": "N4", + "description": "North 4", + "label": "North 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.783Z", + "updatedAt": "2024-03-26T02:39:17.783Z" + } + }, + { + "id": 247, + "lat": -35.25835858471588, + "lng": 149.06613532361757, + "name": { + "id": 21, + "symbol": "N5", + "description": "North 5", + "label": "North 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.794Z", + "updatedAt": "2024-03-26T02:39:17.794Z" + } + }, + { + "id": 248, + "lat": -35.25835858471588, + "lng": 149.06624533735285, + "name": { + "id": 25, + "symbol": "NE", + "description": "North East", + "label": "North East", + "uri": "", + "createdAt": "2024-03-26T02:39:17.844Z", + "updatedAt": "2024-03-26T02:39:17.844Z" + } + }, + { + "id": 249, + "lat": -35.25844841624429, + "lng": 149.06624533735285, + "name": { + "id": 6, + "symbol": "E5", + "description": "East 5", + "label": "East 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.608Z", + "updatedAt": "2024-03-26T02:39:17.608Z" + } + }, + { + "id": 250, + "lat": -35.25862807930111, + "lng": 149.06624533735285, + "name": { + "id": 5, + "symbol": "E4", + "description": "East 4", + "label": "East 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.598Z", + "updatedAt": "2024-03-26T02:39:17.598Z" + } + }, + { + "id": 251, + "lat": -35.25880774235794, + "lng": 149.06624533735285, + "name": { + "id": 4, + "symbol": "E3", + "description": "East 3", + "label": "East 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.585Z", + "updatedAt": "2024-03-26T02:39:17.585Z" + } + }, + { + "id": 252, + "lat": -35.258987405414764, + "lng": 149.06624533735285, + "name": { + "id": 3, + "symbol": "E2", + "description": "East 2", + "label": "East 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.575Z", + "updatedAt": "2024-03-26T02:39:17.575Z" + } + }, + { + "id": 253, + "lat": -35.259167068471584, + "lng": 149.06624533735285, + "name": { + "id": 2, + "symbol": "E1", + "description": "East 1", + "label": "East 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.564Z", + "updatedAt": "2024-03-26T02:39:17.564Z" + } + }, + { + "id": 254, + "lat": -35.2592569, + "lng": 149.06624533735285, + "name": { + "id": 23, + "symbol": "SE", + "description": "South East", + "label": "South East", + "uri": "", + "createdAt": "2024-03-26T02:39:17.821Z", + "updatedAt": "2024-03-26T02:39:17.821Z" + } + }, + { + "id": 255, + "lat": -35.2592569, + "lng": 149.06613532361757, + "name": { + "id": 11, + "symbol": "S5", + "description": "South 5", + "label": "South 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.664Z", + "updatedAt": "2024-03-26T02:39:17.664Z" + } + }, + { + "id": 256, + "lat": -35.2592569, + "lng": 149.06591529614698, + "name": { + "id": 10, + "symbol": "S4", + "description": "South 4", + "label": "South 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.654Z", + "updatedAt": "2024-03-26T02:39:17.654Z" + } + }, + { + "id": 257, + "lat": -35.2592569, + "lng": 149.06569526867642, + "name": { + "id": 9, + "symbol": "S3", + "description": "South 3", + "label": "South 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.643Z", + "updatedAt": "2024-03-26T02:39:17.643Z" + } + }, + { + "id": 258, + "lat": -35.2592569, + "lng": 149.06547524120586, + "name": { + "id": 8, + "symbol": "S2", + "description": "South 2", + "label": "South 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.629Z", + "updatedAt": "2024-03-26T02:39:17.629Z" + } + }, + { + "id": 259, + "lat": -35.2592569, + "lng": 149.06525521373527, + "name": { + "id": 7, + "symbol": "S1", + "description": "South 1", + "label": "South 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.619Z", + "updatedAt": "2024-03-26T02:39:17.619Z" + } + }, + { + "id": 260, + "lat": -35.25880774235794, + "lng": 149.06569526867645, + "name": { + "id": 1, + "symbol": "C", + "description": "Centre", + "label": "Centre", + "uri": "", + "createdAt": "2024-03-26T02:39:17.548Z", + "updatedAt": "2024-03-26T02:39:17.548Z" + } + } + ], + "fauna_plot_point": [] + } + }, + "survey_metadata": { + "id": 51, + "orgMintedUUID": "9beeba20-374a-470a-b4f4-caed34e07572", + "survey_details": { + "id": 58, + "survey_model": "basal-area-dbh-measure-survey", + "time": "2024-03-28T03:17:01.727Z", + "uuid": "263f710d-9af5-456a-9c68-b2b675763220", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "5005b0af-4360-4a8c-a203-b2c9e440547e", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 58, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": "0.1.0-a2ba7a48", + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": "Monitor FDCP API-production", + "system_org": null + } + } + } + } + ] + } +} From 2977f0efbf0e36da89a6aa272b8befa93378e6d8 Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 3 Apr 2024 10:26:18 +1100 Subject: [PATCH 25/52] Get site/dates from reverse lookup fieldcapture#3049 --- .../paratoo/ParatooCollectionId.groovy | 2 +- .../paratoo/ParatooProtocolConfig.groovy | 21 +- .../ecodata/paratoo/ParatooProvenance.groovy | 16 +- .../paratoo/ParatooSurveyDetails.groovy | 11 + .../paratoo/ParatooSurveyMetadata.groovy | 8 + .../paratoo/ParatooProtocolConfigSpec.groovy | 33 +- .../floristicsStandardReverseLookup.json | 1192 +++++++++++++++++ 7 files changed, 1260 insertions(+), 23 deletions(-) create mode 100644 src/test/resources/paratoo/floristicsStandardReverseLookup.json diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollectionId.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollectionId.groovy index 89f55fae8..4f0f6d527 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollectionId.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooCollectionId.groovy @@ -54,7 +54,7 @@ class ParatooCollectionId implements Validateable { Date eventTime = map.eventTime ? DateUtil.parseWithMilliseconds(map.eventTime) : null new ParatooCollectionId( eventTime: eventTime, - survey_metadata: new ParatooSurveyMetadata(map.survey_metadata) + survey_metadata: ParatooSurveyMetadata.fromMap(map.survey_metadata), ) } } diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index b55d56f7d..91de2d5fe 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -24,12 +24,15 @@ class ParatooProtocolConfig { String endDatePath = 'end_date_time' String surveyIdPath = 'survey_metadata' String plotVisitPath = 'plot_visit' + String plotVisitStartDatePath = "${plotVisitPath}.start_date" + String plotVisitEndDatePath = "${plotVisitPath}.end_date" String plotLayoutPath = "${plotVisitPath}.plot_layout" String plotLayoutIdPath = "${plotLayoutPath}.id" String plotLayoutPointsPath = "${plotLayoutPath}.plot_points" String plotSelectionPath = "${plotLayoutPath}.plot_selection" String plotLayoutDimensionLabelPath = "${plotLayoutPath}.plot_dimensions.label" String plotLayoutTypeLabelPath = "${plotLayoutPath}.plot_type.label" + String getApiEndpoint(ParatooCollectionId surveyId) { apiEndpoint ?: defaultEndpoint(surveyId) } @@ -51,7 +54,13 @@ class ParatooProtocolConfig { String date = getProperty(surveyData, startDatePath) if (date == null) { - date = getPropertyFromSurvey(surveyData, startDatePath) + if (usesPlotLayout) { + date = getProperty(surveyData, plotVisitStartDatePath) + } + else { + date = getPropertyFromSurvey(surveyData, startDatePath) + } + } removeMilliseconds(date) @@ -70,7 +79,15 @@ class ParatooProtocolConfig { String date = getProperty(surveyData, endDatePath) if (date == null) { - date = getPropertyFromSurvey(surveyData, endDatePath) + if (usesPlotLayout) { + date = getProperty(surveyData, plotVisitEndDatePath) + if (!date) { + date = getProperty(surveyData, plotVisitStartDatePath) + } + } + else { + date = getPropertyFromSurvey(surveyData, endDatePath) + } } removeMilliseconds(date) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProvenance.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProvenance.groovy index e4c125633..5d33f64c0 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProvenance.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProvenance.groovy @@ -18,7 +18,21 @@ class ParatooProvenance { version_core_documentation: version_core_documentation, system_app: system_app, system_org: system_org, - version_org: version_org + version_org: version_org, + system_core: system_core, + version_core: version_core ] } + + static ParatooProvenance fromMap(Map data) { + new ParatooProvenance( + version_app: data.version_app, + version_core_documentation: data.version_core_documentation, + system_app: data.system_app, + system_org: data.system_org, + version_org: data.version_org, + system_core: data.system_core, + version_core: data.version_core + ) + } } diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyDetails.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyDetails.groovy index 8cca655a6..566c3ce8e 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyDetails.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyDetails.groovy @@ -19,4 +19,15 @@ class ParatooSurveyDetails { protocol_version: protocol_version ] } + + static ParatooSurveyDetails fromMap(Map data) { + new ParatooSurveyDetails( + survey_model: data.survey_model, + time: data.time, + uuid: data.uuid, + project_id: data.project_id, + protocol_id: data.protocol_id, + protocol_version: data.protocol_version + ) + } } diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyMetadata.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyMetadata.groovy index 99413d0b3..98e5dfafe 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyMetadata.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooSurveyMetadata.groovy @@ -22,4 +22,12 @@ class ParatooSurveyMetadata implements Validateable { ] } + static ParatooSurveyMetadata fromMap(Map data) { + ParatooSurveyMetadata surveyMetadata = new ParatooSurveyMetadata([ + survey_details: ParatooSurveyDetails.fromMap(data.survey_details), + provenance: ParatooProvenance.fromMap(data.provenance), + orgMintedUUID: data.orgMintedUUID + ]) + } + } diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index ae021cc7b..3a3610682 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -69,43 +69,38 @@ class ParatooProtocolConfigSpec extends Specification { def "The floristics-standard survey can be used with this config"() { setup: - Map surveyData = readSurveyData('floristicsStandard') + Map surveyData = readSurveyData('floristicsStandardReverseLookup') Map floristicsSurveyConfig = [ apiEndpoint:'floristics-veg-survey-lites', usesPlotLayout:true, - startDatePath: 'attributes.start_date_time', - endDatePath: 'attributes.end_date_time', - surveyIdPath: 'attributes.surveyId' + startDatePath: 'plot_visit.start_date', + endDatePath: 'plot_visit.end_date' ] ParatooProtocolConfig config = new ParatooProtocolConfig(floristicsSurveyConfig) + config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: surveyData.survey_metadata])) expect: - config.getStartDate(surveyData) == '2022-09-21T01:55:44Z' - config.getEndDate(surveyData) == "2022-09-21T01:55:44Z" - config.getGeoJson(surveyData) == [type:"Feature", geometry:[type:"Polygon", coordinates:[[[152.880694, -27.388252], [152.880651, -27.388336], [152.880518, -27.388483], [152.880389, -27.388611], [152.88028, -27.388749], [152.880154, -27.388903], [152.880835, -27.389463], [152.880644, -27.389366], [152.880525, -27.389248], [152.88035, -27.389158], [152.880195, -27.389021], [152.880195, -27.389373], [152.880797, -27.388316], [152.881448, -27.388909], [152.881503, -27.388821], [152.881422, -27.388766], [152.881263, -27.388644], [152.881107, -27.388549], [152.880939, -27.388445], [152.881314, -27.389035], [152.88122, -27.389208], [152.881089, -27.389343], [152.880973, -27.389472], [152.880916, -27.389553], [152.880694, -27.388252]]]], properties:[name:"QDASEQ0001 - Control (100 x 100)", externalId:1, description:"QDASEQ0001 - Control (100 x 100)", notes:"some comment"]] + config.getStartDate(surveyData.collections) == '2024-03-26T03:03:26Z' + config.getEndDate(surveyData.collections) == '2024-03-26T03:03:26Z' + config.getGeoJson(surveyData.collections) == [type:"Feature", geometry:[type:"Polygon", coordinates:[[[149.0651452, -35.2592569], [149.0651452, -35.259167068471584], [149.0651452, -35.258987405414764], [149.0651452, -35.25880774235794], [149.0651452, -35.25862807930111], [149.0651452, -35.25844841624429], [149.0651452, -35.25835858471588], [149.06525521373527, -35.25835858471588], [149.06547524120586, -35.25835858471588], [149.06569526867642, -35.25835858471588], [149.06591529614698, -35.25835858471588], [149.06613532361757, -35.25835858471588], [149.06624533735285, -35.25835858471588], [149.06624533735285, -35.25844841624429], [149.06624533735285, -35.25862807930111], [149.06624533735285, -35.25880774235794], [149.06624533735285, -35.258987405414764], [149.06624533735285, -35.259167068471584], [149.06624533735285, -35.2592569], [149.06613532361757, -35.2592569], [149.06591529614698, -35.2592569], [149.06569526867642, -35.2592569], [149.06547524120586, -35.2592569], [149.06525521373527, -35.2592569], [149.06569526867645, -35.25880774235794], [149.0651452, -35.2592569]]]], properties:[name:"CTMSEH4221 - Control (100 x 100)", externalId:12, description:"CTMSEH4221 - Control (100 x 100)", notes:"Test again 2024-03-26"]] } def "The basal-area-dbh-measure-survey can be used with this config"() { setup: - Map surveyData = readSurveyData('basalAreaDbh') + Map surveyData = readSurveyData('basalAreaDbhReverseLookup') Map basalAreaDbhMeasureSurveyConfig = [ apiEndpoint:'basal-area-dbh-measure-surveys', usesPlotLayout:true, - startDatePath: 'attributes.start_date', - endDatePath: 'attributes.start_date', + startDatePath: 'start_date', + endDatePath: 'start_date', ] ParatooProtocolConfig config = new ParatooProtocolConfig(basalAreaDbhMeasureSurveyConfig) + config.setSurveyId(ParatooCollectionId.fromMap([survey_metadata: surveyData.survey_metadata])) expect: - config.getStartDate(surveyData) == '2023-09-22T00:59:47Z' - config.getEndDate(surveyData) == "2023-09-22T00:59:47Z" - config.getGeoJson(surveyData) == [ - type:"Feature", - geometry:[ - type:"Polygon", - coordinates:[[[138.63720760798054, -34.97222197296049], [138.63720760798054, -34.97204230990367], [138.63720760798054, -34.971862646846844], [138.63720760798054, -34.97168298379002], [138.63720760798054, -34.9715033207332], [138.63720760798054, -34.971413489204785], [138.63731723494544, -34.971413489204785], [138.6375364888752, -34.971413489204785], [138.63775574280498, -34.971413489204785], [138.63797499673475, -34.971413489204785], [138.63819425066453, -34.971413489204785], [138.63830387762943, -34.971413489204785], [138.63830387762943, -34.9715033207332], [138.63830387762943, -34.97168298379002], [138.63830387762943, -34.971862646846844], [138.63830387762943, -34.97204230990367], [138.63830387762943, -34.97222197296049], [138.63830387762943, -34.9723118044889], [138.63819425066453, -34.9723118044889], [138.63797499673475, -34.9723118044889], [138.63775574280498, -34.9723118044889], [138.6375364888752, -34.9723118044889], [138.63731723494544, -34.9723118044889], [138.63720760798054, -34.9723118044889], [138.63720760798054, -34.97222197296049]]]], - properties:["name":"SATFLB0001 - Control (100 x 100)", externalId:4, description:"SATFLB0001 - Control (100 x 100)", notes:"some comment"]] - + config.getStartDate(surveyData.collections) == '2024-03-26T03:03:26Z' + config.getEndDate(surveyData.collections) == "2024-03-26T03:03:26Z" + config.getGeoJson(surveyData.collections) == [type:"Feature", geometry:[type:"Polygon", coordinates:[[[149.0651452, -35.2592569], [149.0651452, -35.259167068471584], [149.0651452, -35.258987405414764], [149.0651452, -35.25880774235794], [149.0651452, -35.25862807930111], [149.0651452, -35.25844841624429], [149.0651452, -35.25835858471588], [149.06525521373527, -35.25835858471588], [149.06547524120586, -35.25835858471588], [149.06569526867642, -35.25835858471588], [149.06591529614698, -35.25835858471588], [149.06613532361757, -35.25835858471588], [149.06624533735285, -35.25835858471588], [149.06624533735285, -35.25844841624429], [149.06624533735285, -35.25862807930111], [149.06624533735285, -35.25880774235794], [149.06624533735285, -35.258987405414764], [149.06624533735285, -35.259167068471584], [149.06624533735285, -35.2592569], [149.06613532361757, -35.2592569], [149.06591529614698, -35.2592569], [149.06569526867642, -35.2592569], [149.06547524120586, -35.2592569], [149.06525521373527, -35.2592569], [149.06569526867645, -35.25880774235794], [149.0651452, -35.2592569]]]], properties:[name:"CTMSEH4221 - Control (100 x 100)", externalId:12, description:"CTMSEH4221 - Control (100 x 100)", notes:"Test again 2024-03-26"]] } def "The observations from opportunistic-survey can be filtered" () { diff --git a/src/test/resources/paratoo/floristicsStandardReverseLookup.json b/src/test/resources/paratoo/floristicsStandardReverseLookup.json new file mode 100644 index 000000000..c623ee9dc --- /dev/null +++ b/src/test/resources/paratoo/floristicsStandardReverseLookup.json @@ -0,0 +1,1192 @@ +{ + "survey_metadata": { + "id": 4, + "org_minted_uuid": "3eef6171-682f-49a4-ab03-6663f94e6564", + "createdAt": "2024-03-26T03:07:18.025Z", + "updatedAt": "2024-03-26T03:07:18.025Z", + "survey_details": { + "id": 50, + "survey_model": "floristics-veg-survey-lite", + "time": "2024-03-26T03:06:02.904Z", + "uuid": "90df7817-1b5e-410c-95d1-09e223a13002", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "bbd550c0-04c5-4a8c-ae39-cc748e920fd4", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 50, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": "0.1.0-a2ba7a48", + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": "Monitor FDCP API-production", + "system_org": null + } + }, + "collections": { + "floristics-veg-survey-lite": { + "id": 4, + "collect_ptv_submodule": false, + "createdAt": "2024-03-26T03:07:16.263Z", + "updatedAt": "2024-03-26T03:07:16.263Z", + "plot_visit": { + "id": 12, + "start_date": "2024-03-26T03:03:26.025Z", + "end_date": null, + "visit_field_name": "Chris test 2024-03-26", + "createdAt": "2024-03-26T03:03:57.778Z", + "updatedAt": "2024-03-26T03:03:57.778Z", + "plot_layout": { + "id": 12, + "orientation": null, + "replicate": 1, + "is_the_plot_permanently_marked": false, + "createdAt": "2024-03-26T03:03:57.623Z", + "updatedAt": "2024-03-26T03:03:57.623Z", + "plot_selection": { + "id": 16, + "plot_label": "CTMSEH4221", + "uuid": "cf23aacf-1f02-43fb-8a78-559d3261838b", + "comments": "Test again 2024-03-26", + "date_time": "2024-03-26T03:01:18.936Z", + "createdAt": "2024-03-26T03:02:01.377Z", + "updatedAt": "2024-03-26T03:02:01.377Z", + "plot_name": { + "id": 16, + "unique_digits": "4221", + "state": { + "id": 1, + "symbol": "CT", + "label": "Australian Capital Territory", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/4253c9af-b390-5b69-aaf3-ce5cda0c36bf", + "createdAt": "2024-03-26T02:39:32.171Z", + "updatedAt": "2024-03-26T02:39:32.171Z" + }, + "program": { + "id": 9, + "symbol": "M", + "label": "MERIT", + "description": "A generic program for MERIT use, as they can have many 'programs' for a given plot", + "uri": "", + "createdAt": "2024-03-26T02:39:18.503Z", + "updatedAt": "2024-03-26T02:39:18.503Z" + }, + "bioregion": { + "id": 83, + "symbol": "SEH", + "label": "South Eastern Highlands", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:41.803Z", + "updatedAt": "2024-03-26T02:39:41.803Z", + "states": [ + { + "id": 2, + "symbol": "NS", + "label": "New South Wales", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/14b8c3fe-3a60-5f3a-bb7a-b9cf5df49b4e", + "createdAt": "2024-03-26T02:39:32.183Z", + "updatedAt": "2024-03-26T02:39:32.183Z" + }, + { + "id": 7, + "symbol": "VC", + "label": "Victoria", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/5e6382e6-d26c-5889-ba7e-f051d82d6173", + "createdAt": "2024-03-26T02:39:32.246Z", + "updatedAt": "2024-03-26T02:39:32.246Z" + }, + { + "id": 1, + "symbol": "CT", + "label": "Australian Capital Territory", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/4253c9af-b390-5b69-aaf3-ce5cda0c36bf", + "createdAt": "2024-03-26T02:39:32.171Z", + "updatedAt": "2024-03-26T02:39:32.171Z" + } + ] + } + }, + "recommended_location": { + "id": 34, + "lat": -35.2592561, + "lng": 149.0651502 + }, + "recommended_location_point": { + "id": 5, + "symbol": "C", + "label": "Centre", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:17.341Z", + "updatedAt": "2024-03-26T02:39:17.341Z" + }, + "plot_selection_survey": { + "id": 7, + "startdate": null, + "createdAt": "2024-03-26T03:02:00.063Z", + "updatedAt": "2024-03-26T03:02:00.063Z", + "survey_metadata": { + "id": 44, + "orgMintedUUID": null, + "survey_details": { + "id": 46, + "survey_model": "plot-selection-survey", + "time": "2024-03-26T03:01:57.093Z", + "uuid": "0d47c9c1-5842-4a02-b8f6-e95a92916b9f", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "a9cb9e38-690f-41c9-8151-06108caf539d", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 46, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": null, + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": null, + "system_org": null + } + } + } + }, + "plot_type": { + "id": 1, + "symbol": "C", + "label": "Control", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:18.129Z", + "updatedAt": "2024-03-26T02:39:18.129Z" + }, + "permanently_marked": null, + "plot_dimensions": { + "id": 1, + "symbol": "100m", + "label": "100 x 100", + "description": "100 m by 100 m (1 hectare)", + "uri": "", + "createdAt": "2024-03-26T02:39:17.254Z", + "updatedAt": "2024-03-26T02:39:17.254Z" + }, + "plot_points": [ + { + "id": 236, + "lat": -35.2592569, + "lng": 149.0651452, + "name": { + "id": 22, + "symbol": "SW", + "description": "South West", + "label": "South West", + "uri": "", + "createdAt": "2024-03-26T02:39:17.807Z", + "updatedAt": "2024-03-26T02:39:17.807Z" + } + }, + { + "id": 237, + "lat": -35.259167068471584, + "lng": 149.0651452, + "name": { + "id": 12, + "symbol": "W1", + "description": "West 1", + "label": "West 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.675Z", + "updatedAt": "2024-03-26T02:39:17.675Z" + } + }, + { + "id": 238, + "lat": -35.258987405414764, + "lng": 149.0651452, + "name": { + "id": 13, + "symbol": "W2", + "description": "West 2", + "label": "West 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.686Z", + "updatedAt": "2024-03-26T02:39:17.686Z" + } + }, + { + "id": 239, + "lat": -35.25880774235794, + "lng": 149.0651452, + "name": { + "id": 14, + "symbol": "W3", + "description": "West 3", + "label": "West 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.700Z", + "updatedAt": "2024-03-26T02:39:17.700Z" + } + }, + { + "id": 240, + "lat": -35.25862807930111, + "lng": 149.0651452, + "name": { + "id": 15, + "symbol": "W4", + "description": "West 4", + "label": "West 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.713Z", + "updatedAt": "2024-03-26T02:39:17.713Z" + } + }, + { + "id": 241, + "lat": -35.25844841624429, + "lng": 149.0651452, + "name": { + "id": 16, + "symbol": "W5", + "description": "West 5", + "label": "West 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.732Z", + "updatedAt": "2024-03-26T02:39:17.732Z" + } + }, + { + "id": 242, + "lat": -35.25835858471588, + "lng": 149.0651452, + "name": { + "id": 24, + "symbol": "NW", + "description": "North West", + "label": "North West", + "uri": "", + "createdAt": "2024-03-26T02:39:17.833Z", + "updatedAt": "2024-03-26T02:39:17.833Z" + } + }, + { + "id": 243, + "lat": -35.25835858471588, + "lng": 149.06525521373527, + "name": { + "id": 17, + "symbol": "N1", + "description": "North 1", + "label": "North 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.743Z", + "updatedAt": "2024-03-26T02:39:17.743Z" + } + }, + { + "id": 244, + "lat": -35.25835858471588, + "lng": 149.06547524120586, + "name": { + "id": 18, + "symbol": "N2", + "description": "North 2", + "label": "North 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.752Z", + "updatedAt": "2024-03-26T02:39:17.752Z" + } + }, + { + "id": 245, + "lat": -35.25835858471588, + "lng": 149.06569526867642, + "name": { + "id": 19, + "symbol": "N3", + "description": "North 3", + "label": "North 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.764Z", + "updatedAt": "2024-03-26T02:39:17.764Z" + } + }, + { + "id": 246, + "lat": -35.25835858471588, + "lng": 149.06591529614698, + "name": { + "id": 20, + "symbol": "N4", + "description": "North 4", + "label": "North 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.783Z", + "updatedAt": "2024-03-26T02:39:17.783Z" + } + }, + { + "id": 247, + "lat": -35.25835858471588, + "lng": 149.06613532361757, + "name": { + "id": 21, + "symbol": "N5", + "description": "North 5", + "label": "North 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.794Z", + "updatedAt": "2024-03-26T02:39:17.794Z" + } + }, + { + "id": 248, + "lat": -35.25835858471588, + "lng": 149.06624533735285, + "name": { + "id": 25, + "symbol": "NE", + "description": "North East", + "label": "North East", + "uri": "", + "createdAt": "2024-03-26T02:39:17.844Z", + "updatedAt": "2024-03-26T02:39:17.844Z" + } + }, + { + "id": 249, + "lat": -35.25844841624429, + "lng": 149.06624533735285, + "name": { + "id": 6, + "symbol": "E5", + "description": "East 5", + "label": "East 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.608Z", + "updatedAt": "2024-03-26T02:39:17.608Z" + } + }, + { + "id": 250, + "lat": -35.25862807930111, + "lng": 149.06624533735285, + "name": { + "id": 5, + "symbol": "E4", + "description": "East 4", + "label": "East 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.598Z", + "updatedAt": "2024-03-26T02:39:17.598Z" + } + }, + { + "id": 251, + "lat": -35.25880774235794, + "lng": 149.06624533735285, + "name": { + "id": 4, + "symbol": "E3", + "description": "East 3", + "label": "East 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.585Z", + "updatedAt": "2024-03-26T02:39:17.585Z" + } + }, + { + "id": 252, + "lat": -35.258987405414764, + "lng": 149.06624533735285, + "name": { + "id": 3, + "symbol": "E2", + "description": "East 2", + "label": "East 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.575Z", + "updatedAt": "2024-03-26T02:39:17.575Z" + } + }, + { + "id": 253, + "lat": -35.259167068471584, + "lng": 149.06624533735285, + "name": { + "id": 2, + "symbol": "E1", + "description": "East 1", + "label": "East 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.564Z", + "updatedAt": "2024-03-26T02:39:17.564Z" + } + }, + { + "id": 254, + "lat": -35.2592569, + "lng": 149.06624533735285, + "name": { + "id": 23, + "symbol": "SE", + "description": "South East", + "label": "South East", + "uri": "", + "createdAt": "2024-03-26T02:39:17.821Z", + "updatedAt": "2024-03-26T02:39:17.821Z" + } + }, + { + "id": 255, + "lat": -35.2592569, + "lng": 149.06613532361757, + "name": { + "id": 11, + "symbol": "S5", + "description": "South 5", + "label": "South 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.664Z", + "updatedAt": "2024-03-26T02:39:17.664Z" + } + }, + { + "id": 256, + "lat": -35.2592569, + "lng": 149.06591529614698, + "name": { + "id": 10, + "symbol": "S4", + "description": "South 4", + "label": "South 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.654Z", + "updatedAt": "2024-03-26T02:39:17.654Z" + } + }, + { + "id": 257, + "lat": -35.2592569, + "lng": 149.06569526867642, + "name": { + "id": 9, + "symbol": "S3", + "description": "South 3", + "label": "South 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.643Z", + "updatedAt": "2024-03-26T02:39:17.643Z" + } + }, + { + "id": 258, + "lat": -35.2592569, + "lng": 149.06547524120586, + "name": { + "id": 8, + "symbol": "S2", + "description": "South 2", + "label": "South 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.629Z", + "updatedAt": "2024-03-26T02:39:17.629Z" + } + }, + { + "id": 259, + "lat": -35.2592569, + "lng": 149.06525521373527, + "name": { + "id": 7, + "symbol": "S1", + "description": "South 1", + "label": "South 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.619Z", + "updatedAt": "2024-03-26T02:39:17.619Z" + } + }, + { + "id": 260, + "lat": -35.25880774235794, + "lng": 149.06569526867645, + "name": { + "id": 1, + "symbol": "C", + "description": "Centre", + "label": "Centre", + "uri": "", + "createdAt": "2024-03-26T02:39:17.548Z", + "updatedAt": "2024-03-26T02:39:17.548Z" + } + } + ], + "fauna_plot_point": [] + } + }, + "ptv_protocol_variant": null, + "survey_metadata": { + "id": 46, + "orgMintedUUID": "3eef6171-682f-49a4-ab03-6663f94e6564", + "survey_details": { + "id": 49, + "survey_model": "floristics-veg-survey-lite", + "time": "2024-03-26T03:06:02.904Z", + "uuid": "90df7817-1b5e-410c-95d1-09e223a13002", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "bbd550c0-04c5-4a8c-ae39-cc748e920fd4", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 49, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": "0.1.0-a2ba7a48", + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": "Monitor FDCP API-production", + "system_org": null + } + } + }, + "floristics-veg-voucher-lite": [ + { + "id": 4, + "field_name": "Plantae [Regnum] (scientific: Plantae Haeckel)", + "voucher_barcode": "123456", + "host_species": null, + "unique_id": null, + "date_time": "2024-03-26T03:06:02.937Z", + "createdAt": "2024-03-26T03:07:16.452Z", + "updatedAt": "2024-03-26T03:07:16.452Z", + "growth_form_1": { + "id": 1, + "symbol": "V", + "label": "Sedge", + "description": "Herbaceous, usually perennial erect plant generally with a tufted habit and of the families Cyperaceae (true sedges) or Restionaceae (node sedges).", + "uri": "https://linked.data.gov.au/def/nrm/a7b8b3c8-a329-5a62-8741-e1d6efe9528b", + "createdAt": "2024-03-26T02:39:35.671Z", + "updatedAt": "2024-03-26T02:39:35.671Z" + }, + "growth_form_2": { + "id": 1, + "symbol": "V", + "label": "Sedge", + "description": "Herbaceous, usually perennial erect plant generally with a tufted habit and of the families Cyperaceae (true sedges) or Restionaceae (node sedges).", + "uri": "https://linked.data.gov.au/def/nrm/a7b8b3c8-a329-5a62-8741-e1d6efe9528b", + "createdAt": "2024-03-26T02:39:35.671Z", + "updatedAt": "2024-03-26T02:39:35.671Z" + }, + "habit": [ + { + "id": 1, + "lut": { + "id": 1, + "symbol": "ASC", + "label": "Ascending", + "description": "Weakly erect", + "uri": "", + "createdAt": "2024-03-26T02:39:07.482Z", + "updatedAt": "2024-03-26T02:39:07.482Z" + } + } + ], + "phenology": [ + { + "id": 1, + "lut": { + "id": 1, + "symbol": "FLO", + "label": "Flowers", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:07.681Z", + "updatedAt": "2024-03-26T02:39:07.681Z" + } + } + ], + "floristics_veg_survey_lite": { + "id": 4, + "collect_ptv_submodule": false, + "createdAt": "2024-03-26T03:07:16.263Z", + "updatedAt": "2024-03-26T03:07:16.263Z", + "plot_visit": { + "id": 12, + "start_date": "2024-03-26T03:03:26.025Z", + "end_date": null, + "visit_field_name": "Chris test 2024-03-26", + "createdAt": "2024-03-26T03:03:57.778Z", + "updatedAt": "2024-03-26T03:03:57.778Z", + "plot_layout": { + "id": 12, + "orientation": null, + "replicate": 1, + "is_the_plot_permanently_marked": false, + "createdAt": "2024-03-26T03:03:57.623Z", + "updatedAt": "2024-03-26T03:03:57.623Z", + "plot_selection": { + "id": 16, + "plot_label": "CTMSEH4221", + "uuid": "cf23aacf-1f02-43fb-8a78-559d3261838b", + "comments": "Test again 2024-03-26", + "date_time": "2024-03-26T03:01:18.936Z", + "createdAt": "2024-03-26T03:02:01.377Z", + "updatedAt": "2024-03-26T03:02:01.377Z", + "plot_name": { + "id": 16, + "unique_digits": "4221", + "state": { + "id": 1, + "symbol": "CT", + "label": "Australian Capital Territory", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/4253c9af-b390-5b69-aaf3-ce5cda0c36bf", + "createdAt": "2024-03-26T02:39:32.171Z", + "updatedAt": "2024-03-26T02:39:32.171Z" + }, + "program": { + "id": 9, + "symbol": "M", + "label": "MERIT", + "description": "A generic program for MERIT use, as they can have many 'programs' for a given plot", + "uri": "", + "createdAt": "2024-03-26T02:39:18.503Z", + "updatedAt": "2024-03-26T02:39:18.503Z" + }, + "bioregion": { + "id": 83, + "symbol": "SEH", + "label": "South Eastern Highlands", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:41.803Z", + "updatedAt": "2024-03-26T02:39:41.803Z", + "states": [ + { + "id": 2, + "symbol": "NS", + "label": "New South Wales", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/14b8c3fe-3a60-5f3a-bb7a-b9cf5df49b4e", + "createdAt": "2024-03-26T02:39:32.183Z", + "updatedAt": "2024-03-26T02:39:32.183Z" + }, + { + "id": 7, + "symbol": "VC", + "label": "Victoria", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/5e6382e6-d26c-5889-ba7e-f051d82d6173", + "createdAt": "2024-03-26T02:39:32.246Z", + "updatedAt": "2024-03-26T02:39:32.246Z" + }, + { + "id": 1, + "symbol": "CT", + "label": "Australian Capital Territory", + "description": "State/Jurisdiction where the study is/was conducted.", + "uri": "https://linked.data.gov.au/def/nrm/4253c9af-b390-5b69-aaf3-ce5cda0c36bf", + "createdAt": "2024-03-26T02:39:32.171Z", + "updatedAt": "2024-03-26T02:39:32.171Z" + } + ] + } + }, + "recommended_location": { + "id": 34, + "lat": -35.2592561, + "lng": 149.0651502 + }, + "recommended_location_point": { + "id": 5, + "symbol": "C", + "label": "Centre", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:17.341Z", + "updatedAt": "2024-03-26T02:39:17.341Z" + }, + "plot_selection_survey": { + "id": 7, + "startdate": null, + "createdAt": "2024-03-26T03:02:00.063Z", + "updatedAt": "2024-03-26T03:02:00.063Z", + "survey_metadata": { + "id": 44, + "orgMintedUUID": null, + "survey_details": { + "id": 46, + "survey_model": "plot-selection-survey", + "time": "2024-03-26T03:01:57.093Z", + "uuid": "0d47c9c1-5842-4a02-b8f6-e95a92916b9f", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "a9cb9e38-690f-41c9-8151-06108caf539d", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 46, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": null, + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": null, + "system_org": null + } + } + } + }, + "plot_type": { + "id": 1, + "symbol": "C", + "label": "Control", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:18.129Z", + "updatedAt": "2024-03-26T02:39:18.129Z" + }, + "permanently_marked": null, + "plot_dimensions": { + "id": 1, + "symbol": "100m", + "label": "100 x 100", + "description": "100 m by 100 m (1 hectare)", + "uri": "", + "createdAt": "2024-03-26T02:39:17.254Z", + "updatedAt": "2024-03-26T02:39:17.254Z" + }, + "plot_points": [ + { + "id": 236, + "lat": -35.2592569, + "lng": 149.0651452, + "name": { + "id": 22, + "symbol": "SW", + "description": "South West", + "label": "South West", + "uri": "", + "createdAt": "2024-03-26T02:39:17.807Z", + "updatedAt": "2024-03-26T02:39:17.807Z" + } + }, + { + "id": 237, + "lat": -35.259167068471584, + "lng": 149.0651452, + "name": { + "id": 12, + "symbol": "W1", + "description": "West 1", + "label": "West 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.675Z", + "updatedAt": "2024-03-26T02:39:17.675Z" + } + }, + { + "id": 238, + "lat": -35.258987405414764, + "lng": 149.0651452, + "name": { + "id": 13, + "symbol": "W2", + "description": "West 2", + "label": "West 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.686Z", + "updatedAt": "2024-03-26T02:39:17.686Z" + } + }, + { + "id": 239, + "lat": -35.25880774235794, + "lng": 149.0651452, + "name": { + "id": 14, + "symbol": "W3", + "description": "West 3", + "label": "West 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.700Z", + "updatedAt": "2024-03-26T02:39:17.700Z" + } + }, + { + "id": 240, + "lat": -35.25862807930111, + "lng": 149.0651452, + "name": { + "id": 15, + "symbol": "W4", + "description": "West 4", + "label": "West 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.713Z", + "updatedAt": "2024-03-26T02:39:17.713Z" + } + }, + { + "id": 241, + "lat": -35.25844841624429, + "lng": 149.0651452, + "name": { + "id": 16, + "symbol": "W5", + "description": "West 5", + "label": "West 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.732Z", + "updatedAt": "2024-03-26T02:39:17.732Z" + } + }, + { + "id": 242, + "lat": -35.25835858471588, + "lng": 149.0651452, + "name": { + "id": 24, + "symbol": "NW", + "description": "North West", + "label": "North West", + "uri": "", + "createdAt": "2024-03-26T02:39:17.833Z", + "updatedAt": "2024-03-26T02:39:17.833Z" + } + }, + { + "id": 243, + "lat": -35.25835858471588, + "lng": 149.06525521373527, + "name": { + "id": 17, + "symbol": "N1", + "description": "North 1", + "label": "North 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.743Z", + "updatedAt": "2024-03-26T02:39:17.743Z" + } + }, + { + "id": 244, + "lat": -35.25835858471588, + "lng": 149.06547524120586, + "name": { + "id": 18, + "symbol": "N2", + "description": "North 2", + "label": "North 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.752Z", + "updatedAt": "2024-03-26T02:39:17.752Z" + } + }, + { + "id": 245, + "lat": -35.25835858471588, + "lng": 149.06569526867642, + "name": { + "id": 19, + "symbol": "N3", + "description": "North 3", + "label": "North 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.764Z", + "updatedAt": "2024-03-26T02:39:17.764Z" + } + }, + { + "id": 246, + "lat": -35.25835858471588, + "lng": 149.06591529614698, + "name": { + "id": 20, + "symbol": "N4", + "description": "North 4", + "label": "North 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.783Z", + "updatedAt": "2024-03-26T02:39:17.783Z" + } + }, + { + "id": 247, + "lat": -35.25835858471588, + "lng": 149.06613532361757, + "name": { + "id": 21, + "symbol": "N5", + "description": "North 5", + "label": "North 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.794Z", + "updatedAt": "2024-03-26T02:39:17.794Z" + } + }, + { + "id": 248, + "lat": -35.25835858471588, + "lng": 149.06624533735285, + "name": { + "id": 25, + "symbol": "NE", + "description": "North East", + "label": "North East", + "uri": "", + "createdAt": "2024-03-26T02:39:17.844Z", + "updatedAt": "2024-03-26T02:39:17.844Z" + } + }, + { + "id": 249, + "lat": -35.25844841624429, + "lng": 149.06624533735285, + "name": { + "id": 6, + "symbol": "E5", + "description": "East 5", + "label": "East 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.608Z", + "updatedAt": "2024-03-26T02:39:17.608Z" + } + }, + { + "id": 250, + "lat": -35.25862807930111, + "lng": 149.06624533735285, + "name": { + "id": 5, + "symbol": "E4", + "description": "East 4", + "label": "East 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.598Z", + "updatedAt": "2024-03-26T02:39:17.598Z" + } + }, + { + "id": 251, + "lat": -35.25880774235794, + "lng": 149.06624533735285, + "name": { + "id": 4, + "symbol": "E3", + "description": "East 3", + "label": "East 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.585Z", + "updatedAt": "2024-03-26T02:39:17.585Z" + } + }, + { + "id": 252, + "lat": -35.258987405414764, + "lng": 149.06624533735285, + "name": { + "id": 3, + "symbol": "E2", + "description": "East 2", + "label": "East 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.575Z", + "updatedAt": "2024-03-26T02:39:17.575Z" + } + }, + { + "id": 253, + "lat": -35.259167068471584, + "lng": 149.06624533735285, + "name": { + "id": 2, + "symbol": "E1", + "description": "East 1", + "label": "East 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.564Z", + "updatedAt": "2024-03-26T02:39:17.564Z" + } + }, + { + "id": 254, + "lat": -35.2592569, + "lng": 149.06624533735285, + "name": { + "id": 23, + "symbol": "SE", + "description": "South East", + "label": "South East", + "uri": "", + "createdAt": "2024-03-26T02:39:17.821Z", + "updatedAt": "2024-03-26T02:39:17.821Z" + } + }, + { + "id": 255, + "lat": -35.2592569, + "lng": 149.06613532361757, + "name": { + "id": 11, + "symbol": "S5", + "description": "South 5", + "label": "South 5", + "uri": "", + "createdAt": "2024-03-26T02:39:17.664Z", + "updatedAt": "2024-03-26T02:39:17.664Z" + } + }, + { + "id": 256, + "lat": -35.2592569, + "lng": 149.06591529614698, + "name": { + "id": 10, + "symbol": "S4", + "description": "South 4", + "label": "South 4", + "uri": "", + "createdAt": "2024-03-26T02:39:17.654Z", + "updatedAt": "2024-03-26T02:39:17.654Z" + } + }, + { + "id": 257, + "lat": -35.2592569, + "lng": 149.06569526867642, + "name": { + "id": 9, + "symbol": "S3", + "description": "South 3", + "label": "South 3", + "uri": "", + "createdAt": "2024-03-26T02:39:17.643Z", + "updatedAt": "2024-03-26T02:39:17.643Z" + } + }, + { + "id": 258, + "lat": -35.2592569, + "lng": 149.06547524120586, + "name": { + "id": 8, + "symbol": "S2", + "description": "South 2", + "label": "South 2", + "uri": "", + "createdAt": "2024-03-26T02:39:17.629Z", + "updatedAt": "2024-03-26T02:39:17.629Z" + } + }, + { + "id": 259, + "lat": -35.2592569, + "lng": 149.06525521373527, + "name": { + "id": 7, + "symbol": "S1", + "description": "South 1", + "label": "South 1", + "uri": "", + "createdAt": "2024-03-26T02:39:17.619Z", + "updatedAt": "2024-03-26T02:39:17.619Z" + } + }, + { + "id": 260, + "lat": -35.25880774235794, + "lng": 149.06569526867645, + "name": { + "id": 1, + "symbol": "C", + "description": "Centre", + "label": "Centre", + "uri": "", + "createdAt": "2024-03-26T02:39:17.548Z", + "updatedAt": "2024-03-26T02:39:17.548Z" + } + } + ], + "fauna_plot_point": [] + } + }, + "ptv_protocol_variant": null, + "survey_metadata": { + "id": 46, + "orgMintedUUID": "3eef6171-682f-49a4-ab03-6663f94e6564", + "survey_details": { + "id": 49, + "survey_model": "floristics-veg-survey-lite", + "time": "2024-03-26T03:06:02.904Z", + "uuid": "90df7817-1b5e-410c-95d1-09e223a13002", + "project_id": "a0f57791-e858-4f33-ae8e-7e3e3fffb447", + "protocol_id": "bbd550c0-04c5-4a8c-ae39-cc748e920fd4", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 49, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": "0.1.0-a2ba7a48", + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": "Monitor FDCP API-production", + "system_org": null + } + } + }, + "photo": [ + { + "id": 13, + "comment": null, + "single_photo": { + "id": 12, + "name": "1269ecd3-a2e4-49c2-b46a-547c528c108f.png", + "alternativeText": null, + "caption": null, + "width": 284, + "height": 58, + "formats": { + "thumbnail": { + "ext": ".png", + "url": "https://beta-monitor-binary.s3.ap-southeast-2.amazonaws.com/thumbnail_1269ecd3_a2e4_49c2_b46a_547c528c108f_477fc3ac7b.png", + "hash": "thumbnail_1269ecd3_a2e4_49c2_b46a_547c528c108f_477fc3ac7b", + "mime": "image/png", + "name": "thumbnail_1269ecd3-a2e4-49c2-b46a-547c528c108f.png", + "path": null, + "size": 7.38, + "width": 245, + "height": 50, + "sizeInBytes": 7375 + } + }, + "hash": "1269ecd3_a2e4_49c2_b46a_547c528c108f_477fc3ac7b", + "ext": ".png", + "mime": "image/png", + "size": 1.33, + "url": "https://beta-monitor-binary.s3.ap-southeast-2.amazonaws.com/1269ecd3_a2e4_49c2_b46a_547c528c108f_477fc3ac7b.png", + "previewUrl": null, + "provider": "aws-s3", + "provider_metadata": null, + "folderPath": "/1", + "createdAt": "2024-03-26T03:07:14.181Z", + "updatedAt": "2024-03-26T03:07:14.181Z" + } + } + ] + } + ], + "floristics-veg-virtual-voucher": [] + } +} \ No newline at end of file From 6d7355a63207783da529a74d40fc15b9baa5b3c8 Mon Sep 17 00:00:00 2001 From: temi Date: Sat, 6 Apr 2024 09:14:16 +1100 Subject: [PATCH 26/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixed several broken tests - new record will have default individual count of 1 - refactored code - refined logic of rearrangeSurveyData --- .../domain/au/org/ala/ecodata/Record.groovy | 4 +- .../ala/ecodata/ElasticSearchService.groovy | 2 +- .../au/org/ala/ecodata/ParatooService.groovy | 156 +- .../paratoo/ParatooProtocolConfig.groovy | 118 +- .../org/ala/ecodata/ParatooServiceSpec.groovy | 70 +- .../paratoo/ParatooProtocolConfigSpec.groovy | 256 ++- src/test/resources/paratoo/basalAreaDbh.json | 776 ------- .../paratoo/basalAreaDbhReverseLookup.json | 1910 ++++++----------- .../resources/paratoo/floristicsStandard.json | 620 ------ .../floristicsStandardReverseLookup.json | 1285 +++-------- .../opportunisticSurveyObservations.json | 340 --- ...nisticSurveyObservationsReverseLookup.json | 174 ++ .../photoPointSurveyReverseLookup.json | 574 +++++ .../plotDefinitionSurveyReverseLookup.json | 542 +++++ .../paratoo/vegetationMappingObservation.json | 150 -- ...tationMappingObservationReverseLookup.json | 148 ++ .../paratoo/vegetationMappingSurvey.json | 151 -- 17 files changed, 2782 insertions(+), 4494 deletions(-) delete mode 100644 src/test/resources/paratoo/basalAreaDbh.json delete mode 100644 src/test/resources/paratoo/floristicsStandard.json delete mode 100644 src/test/resources/paratoo/opportunisticSurveyObservations.json create mode 100644 src/test/resources/paratoo/opportunisticSurveyObservationsReverseLookup.json create mode 100644 src/test/resources/paratoo/photoPointSurveyReverseLookup.json create mode 100644 src/test/resources/paratoo/plotDefinitionSurveyReverseLookup.json delete mode 100644 src/test/resources/paratoo/vegetationMappingObservation.json create mode 100644 src/test/resources/paratoo/vegetationMappingObservationReverseLookup.json delete mode 100644 src/test/resources/paratoo/vegetationMappingSurvey.json diff --git a/grails-app/domain/au/org/ala/ecodata/Record.groovy b/grails-app/domain/au/org/ala/ecodata/Record.groovy index 800c1513c..a95454682 100644 --- a/grails-app/domain/au/org/ala/ecodata/Record.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Record.groovy @@ -35,14 +35,13 @@ class Record { Double generalizedDecimalLatitude Double generalizedDecimalLongitude Integer coordinateUncertaintyInMeters - Integer individualCount + Integer individualCount = 1 Integer numberOfOrganisms Date dateCreated Date lastUpdated String outputId String json Integer outputItemId - String individualsOrGroups = 'Individuals' String status = ACTIVE static transients = ['recordNumber'] @@ -76,7 +75,6 @@ class Record { name nullable: true vernacularName nullable: true scientificName nullable: true - individualsOrGroups nullable: true, inList: ['Individuals', 'Groups'] } String getRecordNumber(sightingsUrl){ diff --git a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy index 93a914eb7..0f34079b0 100644 --- a/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy @@ -1317,7 +1317,7 @@ class ElasticSearchService { * @param path * @return */ - List getDataFromPath(output, List path){ + static List getDataFromPath(output, List path){ def temp = output List result = [] List navigatedPath = [] diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 35964c921..920b4c40d 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -71,7 +71,6 @@ class ParatooService { RecordService recordService MetadataService metadataService UserService userService - ElasticSearchService elasticSearchService /** * The rules we use to find projects eligible for use by paratoo are: @@ -232,60 +231,64 @@ class ParatooService { } Map asyncFetchCollection(ParatooCollection collection, Map authHeader, String userId, ParatooProject project) { - int counter = 0 - boolean forceActivityCreation = false - Map surveyDataAndObservations = null - Map dataSet = project.project.custom?.dataSets?.find{it.dataSetId == collection.orgMintedUUID} + Activity.withSession { session -> + int counter = 0 + boolean forceActivityCreation = false + Map surveyDataAndObservations = null + Map response = null + Map dataSet = project.project.custom?.dataSets?.find{it.dataSetId == collection.orgMintedUUID} + + if (!dataSet) { + throw new RuntimeException("Unable to find data set with orgMintedUUID: "+collection.orgMintedUUID) + } - if (!dataSet) { - throw new RuntimeException("Unable to find data set with orgMintedUUID: "+collection.orgMintedUUID) - } + // wait for 5 seconds before fetching data + while(response == null && counter < PARATOO_MAX_RETRIES) { + sleep(5 * 1000) + try { + response = retrieveSurveyAndObservations(collection, authHeader) + surveyDataAndObservations = response?.collections + } catch (Exception e) { + log.error("Error fetching collection data for ${collection.orgMintedUUID}: ${e.message}") + } - // wait for 5 seconds before fetching data - while(surveyDataAndObservations == null && counter < PARATOO_MAX_RETRIES) { - sleep(5 * 1000) - try { - surveyDataAndObservations = retrieveSurveyAndObservations(collection, authHeader) - } catch (Exception e) { - log.error("Error fetching collection data for ${collection.orgMintedUUID}: ${e.message}") + counter++ } - counter++ - } + if (surveyDataAndObservations == null) { + log.error("Unable to fetch collection data for ${collection.orgMintedUUID}") + return + } else { + ParatooCollectionId surveyId = ParatooCollectionId.fromMap(dataSet.surveyId) + ParatooProtocolConfig config = getProtocolConfig(surveyId.protocolId) + config.surveyId = surveyId + ActivityForm form = ActivityForm.findByExternalId(surveyId.protocolId) + // get survey data from reverse lookup response + Map surveyData = config.getSurveyData(surveyDataAndObservations) + // add plot data to survey observations + addPlotDataToObservations(surveyDataAndObservations, config) + rearrangeSurveyData(surveyDataAndObservations, surveyDataAndObservations, form.sections[0].template.relationships.ecodata, form.sections[0].template.relationships.apiOutput) + // transform data to make it compatible with data model + surveyDataAndObservations = recursivelyTransformData(form.sections[0].template.dataModel, surveyDataAndObservations, form.name) + // If we are unable to create a site, null will be returned - assigning a null siteId is valid. + + if (!dataSet.siteId) { + dataSet.siteId = createSiteFromSurveyData(surveyDataAndObservations, collection, surveyId, project.project, config, form) + } - if (surveyDataAndObservations == null) { - log.error("Unable to fetch collection data for ${collection.orgMintedUUID}") - return - } else { - ParatooCollectionId surveyId = ParatooCollectionId.fromMap(dataSet.surveyId) - ParatooProtocolConfig config = getProtocolConfig(surveyId.protocolId) - config.surveyId = surveyId - ActivityForm form = ActivityForm.findByExternalId(surveyId.protocolId) - - // addPlotDataToObservations mutates the data at the top level so a shallow copy is OK - Map surveyData = new HashMap(surveyDataAndObservations) - addPlotDataToObservations(surveyData, surveyDataAndObservations, config) - rearrangeSurveyData(surveyDataAndObservations, surveyDataAndObservations, form.sections[0].template.relationships.ecodata, form.sections[0].template.relationships.apiOutput) - // transform data to make it compatible with data model - surveyDataAndObservations = recursivelyTransformData(form.sections[0].template.dataModel, surveyDataAndObservations) - // If we are unable to create a site, null will be returned - assigning a null siteId is valid. - if (!dataSet.siteId) { - dataSet.siteId = createSiteFromSurveyData(surveyData, surveyDataAndObservations, collection, surveyId, project.project, config, form) - } - // make sure activity has not been created for this data set - if (!dataSet.activityId || forceActivityCreation) { - Activity.withSession { + // make sure activity has not been created for this data set + if (!dataSet.activityId || forceActivityCreation) { String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet.siteId, userId) List records = recordService.getAllByActivity(activityId) dataSet.areSpeciesRecorded = records?.size() > 0 dataSet.activityId = activityId } - } - dataSet.startDate = config.getStartDate(surveyData) - dataSet.endDate = config.getEndDate(surveyData) + dataSet.startDate = config.getStartDate(surveyDataAndObservations) + dataSet.endDate = config.getEndDate(surveyDataAndObservations) - projectService.update([custom: project.project.custom], project.id, false) + projectService.update([custom: project.project.custom], project.id, false) + } } } @@ -308,17 +311,23 @@ class ParatooService { def nodeObject = rootProperties[nodeName] if (nodeObject != null) { properties[nodeName] = nodeObject + // don't add root properties to remove list + if (properties != rootProperties) { + nodesToRemove.add(nodeName) + } } else { - nodeObject = findObservationDataFromAPIOutput(nodeName, apiOutputRelationship, rootProperties) + def result = findObservationDataFromAPIOutput(nodeName, apiOutputRelationship, rootProperties) + if (result.data instanceof List && (result.data.size() > 0)) { + nodeObject = result.data.first() + } + if (nodeObject != null) { properties[nodeName] = nodeObject + nodesToRemove.add(result.path) } } - if (isChild) { - nodesToRemove.add(nodeName) - } if (children) { if (nodeObject instanceof Map) { @@ -330,13 +339,16 @@ class ParatooService { } } } + ancestors.removeLast() } } // remove nodes that have been rearranged. removing during iteration will cause exception. if (!isChild) { - nodesToRemove.each { String node -> - properties.remove(node) + // sort based on depth of nesting so that child nodes are removed first before ancestors. + nodesToRemove = nodesToRemove.sort { a, b -> b.split('\\.').size() <=> a.split('\\.').size() } + nodesToRemove.each { String path -> + removeProperty(properties, path) } nodesToRemove.clear() @@ -351,11 +363,11 @@ class ParatooService { * @param surveyDataAndObservations * @param config */ - static void addPlotDataToObservations(Map surveyData, Map surveyDataAndObservations, ParatooProtocolConfig config) { - if (surveyDataAndObservations && surveyData) { - Map plotSelection = config.getPlotSelection(surveyData) - Map plotLayout = config.getPlotLayout(surveyData) - Map plotVisit = config.getPlotVisit(surveyData) + static void addPlotDataToObservations(Map surveyDataAndObservations, ParatooProtocolConfig config) { + if (surveyDataAndObservations && config.usesPlotLayout) { + Map plotSelection = config.getPlotSelection(surveyDataAndObservations) + Map plotLayout = config.getPlotLayout(surveyDataAndObservations) + Map plotVisit = config.getPlotVisit(surveyDataAndObservations) if (plotSelection) { surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_SELECTION] = plotSelection @@ -438,7 +450,7 @@ class ParatooService { * @param path * @return */ - def recursivelyTransformData(List dataModel, Map output) { + def recursivelyTransformData(List dataModel, Map output, String formName = "", int featureId = 1) { dataModel?.each { Map model -> switch (model.dataType) { case "list": @@ -457,7 +469,7 @@ class ParatooService { rows?.each { row -> if (row != null) { - recursivelyTransformData(model.columns, row) + recursivelyTransformData(model.columns, row, formName, featureId) } } break @@ -475,9 +487,18 @@ class ParatooService { // bird survey plot def point = output[model.name] output[model.name] = [ - type : 'Point', - coordinates: [point.lng, point.lat] + type : 'Feature', + geometry : [ + type : 'Point', + coordinates: [point.lng, point.lat] + ], + properties: [ + name : "Point", + externalId: point.id, + id: "${formName}-${featureId}" + ] ] + featureId ++ break case "image": case "document": @@ -492,10 +513,10 @@ class ParatooService { output } - private String createSiteFromSurveyData(Map surveyData, Map observation, ParatooCollection collection, ParatooCollectionId surveyId, Project project, ParatooProtocolConfig config, ActivityForm form) { + private String createSiteFromSurveyData(Map observation, ParatooCollection collection, ParatooCollectionId surveyId, Project project, ParatooProtocolConfig config, ActivityForm form) { String siteId = null // Create a site representing the location of the collection - Map geoJson = config.getGeoJson(surveyData, observation, form) + Map geoJson = config.getGeoJson(observation, form) if (geoJson) { List features = geoJson?.features ?: [] geoJson.remove('features') @@ -517,7 +538,7 @@ class ParatooService { } if (result.error) { // Don't treat this as a fatal error for the purposes of responding to the paratoo request - log.error("Error creating a site for survey " + collection.orgMintedIdentifier + ", project " + project.projectId + ": " + result.error) + log.error("Error creating a site for survey " + collection.orgMintedUUID + ", project " + project.projectId + ": " + result.error) } siteId = result.siteId } @@ -708,7 +729,7 @@ class ParatooService { Map response = webService.doPost(url, payload, false, authHeader) log.debug((response as JSON).toString()) - response?.resp?.collections + response?.resp } Map addOrUpdatePlotSelections(String userId, ParatooPlotSelectionData plotSelectionData) { @@ -1553,20 +1574,21 @@ class ParatooService { path = path.replaceAll(".properties", "") // get model definition for the parent try { - return getProperty(data, path)?.first() + def result = getProperty(data, path) + return [path: path, data: result] } catch (Exception e) { log.info("Error getting property for path: ${path}") } } else { - return data[modelToFind] + return [path: modelToFind, data: [data[modelToFind]]] } } /** * Remove a property at a given path. - * i.e. if path is a.b.c and object is {a: {b: {c: [:]}}} then after removing the property, properties will be {a: {b: [:]}} + * i.e. if path is a.b.c and object is [a: [b: [c: [:]]]] then after removing the property, properties will be [a: [b: [:]]] * @param object * @param key * @param parts @@ -1866,13 +1888,13 @@ class ParatooService { modelName?.toLowerCase()?.replaceAll("[^a-zA-Z0-9]+", ' ')?.tokenize(' ')?.collect { it.capitalize() }?.join() } - private def getProperty(def surveyData, String path) { + def getProperty(def surveyData, String path) { if (!path || surveyData == null) { return null } List parts = path.split(/\./) - deepCopy(elasticSearchService.getDataFromPath(surveyData, parts)) + deepCopy(ElasticSearchService.getDataFromPath(surveyData, parts)) } /** diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 91de2d5fe..9d7368d61 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -52,7 +52,7 @@ class ParatooProtocolConfig { return null } - String date = getProperty(surveyData, startDatePath) + def date = getProperty(surveyData, startDatePath) if (date == null) { if (usesPlotLayout) { date = getProperty(surveyData, plotVisitStartDatePath) @@ -63,12 +63,12 @@ class ParatooProtocolConfig { } + date = getFirst(date) removeMilliseconds(date) } def getPropertyFromSurvey(Map surveyData, String path) { - surveyData = getSurveyDataFromObservation(surveyData) - path = path.replaceFirst("^attributes.", '') + surveyData = getSurveyData(surveyData) getProperty(surveyData, path) } @@ -77,7 +77,7 @@ class ParatooProtocolConfig { return null } - String date = getProperty(surveyData, endDatePath) + def date = getProperty(surveyData, endDatePath) if (date == null) { if (usesPlotLayout) { date = getProperty(surveyData, plotVisitEndDatePath) @@ -90,6 +90,7 @@ class ParatooProtocolConfig { } } + date = getFirst(date) removeMilliseconds(date) } @@ -98,16 +99,19 @@ class ParatooProtocolConfig { return null } - Map result = getProperty(surveyData, surveyIdPath) + def result = getProperty(surveyData, surveyIdPath) if (result == null) { result = getPropertyFromSurvey(surveyData, surveyIdPath) } + result = getFirst(result) result } - private Map extractSiteDataFromPath(Map surveyData) { + private Map extractSiteDataFromPath(Map survey) { + Map surveyData = getSurveyData(survey) def geometryData = getProperty(surveyData, geometryPath) + geometryData = getFirst(geometryData) extractGeometryFromSiteData(geometryData) } @@ -171,56 +175,73 @@ class ParatooProtocolConfig { } def getProperty(Map surveyData, String path) { - if (surveyId) { - path = surveyId.survey_metadata.survey_details.survey_model+'.'+path - } - if (!path) { return null } - new PropertyAccessor(path).get(surveyData) + + def result = new PropertyAccessor(path).get(surveyData) + if (result == null) { + if (surveyId) { + path = surveyId.survey_metadata.survey_details.survey_model+'.'+path + } + + result = new PropertyAccessor(path).get(surveyData) + } + + result } - Map getGeoJson(Map survey, Map observation = null, ActivityForm form = null) { - if (!survey) { + Map getGeoJson(Map output, ActivityForm form = null) { + if (!output) { return null } Map geoJson = null if (usesPlotLayout) { - geoJson = extractSiteDataFromPlotVisit(survey) + geoJson = extractSiteDataFromPlotVisit(output) // get list of all features associated with observation - if (geoJson && form && observation) { - geoJson.features = extractFeatures(observation, form) + if (geoJson && form && output) { + geoJson.features = extractFeatures(output, form) } } - else if (form && observation) { - List features = extractFeatures(observation, form) + else if (geometryPath) { + geoJson = extractSiteDataFromPath(output) + } + else if (form && output) { + List features = extractFeatures(output, form) if (features) { - Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(features) - geoJson = GeometryUtils.geometryToGeoJsonMap(geometry) - geoJson.features = features + List featureGeometries = features.collect { it.geometry } + Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(featureGeometries) + geoJson = [ + type: 'Feature', + geometry: GeometryUtils.geometryToGeoJsonMap(geometry), + properties: [ + name: "Convex Hull", + description: "Convex Hull of ${features?.size()} feature(s)" + ], + features: features + ] } } - else if (geometryPath) { - geoJson = extractSiteDataFromPath(survey) - } geoJson } Map getPlotVisit (Map surveyData) { - Map plotVisit = getProperty(surveyData, plotVisitPath) + def result = getProperty(surveyData, plotVisitPath) + Map plotVisit = getFirst(result) copyWithExcludedProperty(plotVisit, ParatooService.PARATOO_DATAMODEL_PLOT_LAYOUT) } Map getPlotLayout (Map surveyData) { - Map plotLayout = getProperty(surveyData, plotLayoutPath) + def result = getProperty(surveyData, plotLayoutPath) + Map plotLayout = getFirst(result) copyWithExcludedProperty(plotLayout, ParatooService.PARATOO_DATAMODEL_PLOT_SELECTION) } Map getPlotSelection (Map surveyData) { - Map plotSelection = getProperty(surveyData, plotSelectionPath) + def result = getProperty(surveyData, plotSelectionPath) + Map plotSelection = getFirst(result) copyWithExcludedProperty(plotSelection) } @@ -258,20 +279,35 @@ class ParatooProtocolConfig { tmpSurveyId?.survey_details?.uuid == collectionId.survey_metadata?.survey_details.uuid } - private Map extractSiteDataFromPlotVisit(Map survey) { + static def getFirst (def value) { + if (value instanceof List) { + value = value.first() + } + + value + } + + def getSurveyData (Map survey) { + if (surveyId) { + def path = surveyId.survey_metadata.survey_details.survey_model + getFirst(survey[path]) + } + } - def plotLayoutId = getProperty(survey, plotLayoutIdPath) // Currently an int, may become uuid? + private Map extractSiteDataFromPlotVisit(Map survey) { + Map surveyData = getSurveyData(survey) + def plotLayoutId = getProperty(surveyData, plotLayoutIdPath) // Currently an int, may become uuid? if (!plotLayoutId) { log.warn("No plot_layout found in survey at path ${plotLayoutIdPath}") return null } - List plotLayoutPoints = getProperty(survey, plotLayoutPointsPath) - Map plotSelection = getProperty(survey, plotSelectionPath) + List plotLayoutPoints = getProperty(surveyData, plotLayoutPointsPath) + Map plotSelection = getProperty(surveyData, plotSelectionPath) Map plotSelectionGeoJson = plotSelectionToGeoJson(plotSelection) - String plotLayoutDimensionLabel = getProperty(survey, plotLayoutDimensionLabelPath) - String plotLayoutTypeLabel = getProperty(survey, plotLayoutTypeLabelPath) + String plotLayoutDimensionLabel = getProperty(surveyData, plotLayoutDimensionLabelPath) + String plotLayoutTypeLabel = getProperty(surveyData, plotLayoutTypeLabelPath) String name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' (' + plotLayoutDimensionLabel + ')' @@ -307,20 +343,6 @@ class ParatooProtocolConfig { plotGeometry } - Map getSurveyDataFromObservation(Map observation) { - String surveyAttribute = apiEndpoint - if(surveyAttribute?.endsWith('s')) { - surveyAttribute = surveyAttribute.substring(0, surveyAttribute.length() - 1) - } - - def survey = observation[surveyAttribute] - if (survey instanceof List) { - return survey[0] - } - - survey - } - private static List closePolygonIfRequired(List points) { if (points[0][0] != points[-1][0] || points[0][1] != points[-1][1]) { points << points[0] @@ -329,7 +351,7 @@ class ParatooProtocolConfig { } private static boolean exclude(Map point) { - point.name?.data?.attributes?.symbol == "C" // The plot layout has a centre point that we don't want + point.name?.data?.attributes?.symbol == "C" || point.name?.symbol == "C"// The plot layout has a centre point that we don't want } // Accepts a Map or ParatooPlotSelectionData as this is used by two separate calls. diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 121e74d7c..9953f58a8 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -5,18 +5,19 @@ import au.org.ala.ws.tokens.TokenService import com.nimbusds.oauth2.sdk.token.AccessToken import grails.converters.JSON import grails.test.mongodb.MongoSpec +import grails.testing.gorm.DataTest import grails.testing.services.ServiceUnitTest import groovy.json.JsonSlurper import org.codehaus.jackson.map.ObjectMapper import org.grails.web.converters.marshaller.json.CollectionMarshaller import org.grails.web.converters.marshaller.json.MapMarshaller -import static grails.async.Promises.waitAll +import static grails.async.Promises.waitAll /** * Tests for the ParatooService. * The tests are incomplete as some of the behaviour needs to be specified. */ -class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest { +class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest, DataTest { String userId = 'u1' SiteService siteService = Mock(SiteService) @@ -48,7 +49,6 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [resp: [collections: ["coarse-woody-debris-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z"]]]] + 1 * webService.doPost(*_) >> [resp: [collections: ["coarse-woody-debris-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z", start_date_time: "2023-09-01T00:00:00.123Z", end_date_time: "2023-09-01T00:00:00.123Z"]]]] 1 * tokenService.getAuthToken(true) >> Mock(AccessToken) 1 * projectService.update([custom: [dataSets: [expectedDataSetAsync]]], 'p1', false) >> [status: 'ok'] 1 * projectService.update([custom: [dataSets: [expectedDataSetSync]]], 'p1', false) >> [status: 'ok'] @@ -285,7 +285,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest" ] ) - ParatooCollectionId paratooCollectionId = buildCollectionId("mintCollectionIdBasalAreaPayload") + ParatooCollectionId paratooCollectionId = buildCollectionId("mintCollectionIdBasalAreaPayload","guid-3") Map dataSet = [dataSetId:'d1', grantId:'g1', surveyId:paratooCollectionId.toMap()] ParatooProject project = new ParatooProject(id: projectId, project: new Project(projectId: projectId, custom: [dataSets: [dataSet]])) Map surveyData = readSurveyData('basalAreaDbhReverseLookup') @@ -303,7 +303,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [activityId: '123'] 1 * recordService.getAllByActivity('123') >> [] 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { - (["guid-2": [ + (["guid-3": [ "name" : "Basal Area - DBH", "usesPlotLayout": true, "tags" : ["survey"], @@ -317,12 +317,12 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [userId: userId] and: - site.name == "CTMSEH4221 - Control (100 x 100)" - site.description == "CTMSEH4221 - Control (100 x 100)" - site.notes == "Test again 2024-03-26" + site.name == "SATFLB0001 - Control (100 x 100)" + site.description == "SATFLB0001 - Control (100 x 100)" + site.notes == "some comment" site.type == "surveyArea" site.publicationStatus == "published" - site.externalIds[0].externalId == "12" + site.externalIds[0].externalId == "2" site.externalIds[0].idType == ExternalId.IdType.MONITOR_PLOT_GUID result.updateResult == [status: 'ok'] @@ -371,7 +371,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Mon, 8 Apr 2024 10:45:10 +1000 Subject: [PATCH 27/52] Added an index on outputId in the Record collection. #923 --- grails-app/domain/au/org/ala/ecodata/Record.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/grails-app/domain/au/org/ala/ecodata/Record.groovy b/grails-app/domain/au/org/ala/ecodata/Record.groovy index a95454682..0dfe00481 100644 --- a/grails-app/domain/au/org/ala/ecodata/Record.groovy +++ b/grails-app/domain/au/org/ala/ecodata/Record.groovy @@ -15,6 +15,7 @@ class Record { projectActivityId index: true lastUpdated index: true dataSetId index: true + outputId index: true version false } From 844361ec3f1ecc8ba31fa86cb80e6c5d0e876f8e Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 8 Apr 2024 13:01:57 +1000 Subject: [PATCH 28/52] AtlasOfLivingAustralia/fieldcapture#3049 - do not add plot_visit, plot_layout and plot_selection to data model - refactored code --- .../services/au/org/ala/ecodata/ParatooService.groovy | 3 ++- .../org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 920b4c40d..1921eb650 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -44,7 +44,8 @@ class ParatooService { static final String PARATOO_FIELD_LABEL = "x-paratoo-rename" static final List PARATOO_IGNORE_MODEL_LIST = [ 'created_at', 'createdAt', 'updated_at', 'updatedAt', 'created_by', 'createdBy', 'updated_by', 'updatedBy', - 'published_at', 'publishedAt', 'x-paratoo-file-type' + 'published_at', 'publishedAt', 'x-paratoo-file-type', PARATOO_DATAMODEL_PLOT_LAYOUT, PARATOO_DATAMODEL_PLOT_SELECTION, + PARATOO_DATAMODEL_PLOT_VISIT ] static final List PARATOO_IGNORE_X_MODEL_REF_LIST_MINIMUM = [ 'file', 'admin::user' diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 9d7368d61..d81bb5e57 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -281,7 +281,13 @@ class ParatooProtocolConfig { static def getFirst (def value) { if (value instanceof List) { - value = value.first() + try { + value = value.first() + } + catch (NoSuchElementException e) { + log.warn("List is empty", e) + value = null + } } value From 52c991e61656cb95df8f16ed97056bd236bf4a6e Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 8 Apr 2024 16:08:41 +1000 Subject: [PATCH 29/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixes NullPointerException when creating species records --- .../groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy index a11f5a929..5cf5b43f5 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy @@ -6,7 +6,7 @@ class SpeciesConverter implements RecordFieldConverter { ] List convert(Map data, Map metadata = [:]) { - if (data == null) { + if ((data == null) || (data[metadata.name] == null) ) { return [] } From 39bec3c9b6757b45c87538edba2981ee877a6a07 Mon Sep 17 00:00:00 2001 From: temi Date: Tue, 9 Apr 2024 07:42:36 +1000 Subject: [PATCH 30/52] AtlasOfLivingAustralia/fieldcapture#3049 - improved parsing of taxon information - making sure lists are not reduced --- .../au/org/ala/ecodata/ParatooService.groovy | 28 +++++++++++++------ .../paratoo/ParatooProtocolConfig.groovy | 7 ++--- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 1921eb650..b599c8023 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -1,6 +1,6 @@ package au.org.ala.ecodata - +import au.org.ala.ecodata.metadata.PropertyAccessor import au.org.ala.ecodata.paratoo.* import au.org.ala.ws.tokens.TokenService import grails.async.Promise @@ -458,7 +458,7 @@ class ParatooService { String updatedPath = model.name def rows =[] try { - rows = getProperty(output, updatedPath)?.first() + rows = getProperty(output, updatedPath, true, false) } catch (Exception e) { log.info("Error getting list for ${model.name}: ${e.message}") @@ -1889,13 +1889,21 @@ class ParatooService { modelName?.toLowerCase()?.replaceAll("[^a-zA-Z0-9]+", ' ')?.tokenize(' ')?.collect { it.capitalize() }?.join() } - def getProperty(def surveyData, String path) { + def getProperty(def surveyData, String path, boolean useAccessor = false, boolean isDeepCopy = true) { if (!path || surveyData == null) { return null } - List parts = path.split(/\./) - deepCopy(ElasticSearchService.getDataFromPath(surveyData, parts)) + def result + if (useAccessor) { + result = new PropertyAccessor(path).get(surveyData) + } + else { + List parts = path.split(/\./) + result = ElasticSearchService.getDataFromPath(surveyData, parts) + } + + return isDeepCopy ? deepCopy(result) : result } /** @@ -1911,14 +1919,16 @@ class ParatooService { return null } - String regex = "\\(scientific:\\s*(.*?)\\)" + String regex = "([^\\[]*)\\[(.*)\\]\\s*\\(scientific:\\s*(.*?)\\)" Pattern pattern = Pattern.compile(regex) Matcher matcher = pattern.matcher(name) - Map result = [name: name, scientificName: name, outputSpeciesId: UUID.randomUUID().toString()] + Map result = [name: name, scientificName: name, commonName: name, outputSpeciesId: UUID.randomUUID().toString()] if (matcher.find()) { - name = matcher.group(1) - result.scientificName = result.name = name + result.commonName = matcher.group(1) + result.taxonRank = matcher.group(2) + result.scientificName = result.name = matcher.group(3) + result.name = result.scientificName ?: result.commonName ?: name } metadataService.autoPopulateSpeciesData(result) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index d81bb5e57..e19c27a37 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -180,11 +180,8 @@ class ParatooProtocolConfig { } def result = new PropertyAccessor(path).get(surveyData) - if (result == null) { - if (surveyId) { - path = surveyId.survey_metadata.survey_details.survey_model+'.'+path - } - + if ((result == null) && (surveyId != null)) { + path = surveyId.survey_metadata.survey_details.survey_model+'.'+path result = new PropertyAccessor(path).get(surveyData) } From 0a91555e528c8b0ce2e817cca9fdfed266f06785 Mon Sep 17 00:00:00 2001 From: temi Date: Tue, 9 Apr 2024 09:38:49 +1000 Subject: [PATCH 31/52] AtlasOfLivingAustralia/fieldcapture#3049 - resolve scientific name if common name is present - fixed broken test --- .../au/org/ala/ecodata/MetadataService.groovy | 5 +++-- .../services/au/org/ala/ecodata/ParatooService.groovy | 8 ++++---- .../au/org/ala/ecodata/ParatooServiceSpec.groovy | 11 ++++++++++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy index 423b86642..0f88754ee 100644 --- a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy +++ b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy @@ -845,12 +845,13 @@ class MetadataService { } Map autoPopulateSpeciesData(Map data){ - if (!data?.guid && data?.scientificName) { - def result = speciesReMatchService.searchBie(data.scientificName, 10) + if (!data?.guid && (data?.scientificName ?: data?.commonName)) { + def result = speciesReMatchService.searchBie(data.scientificName?: data.commonName, 10) // only if there is a single match if (result?.autoCompleteList?.size() == 1) { data.guid = result?.autoCompleteList[0]?.guid data.commonName = data.commonName ?: result?.autoCompleteList[0]?.commonName + data.scientificName = data.scientificName ?: result?.autoCompleteList[0]?.name } } diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index b599c8023..26f9e734c 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -1925,10 +1925,10 @@ class ParatooService { Map result = [name: name, scientificName: name, commonName: name, outputSpeciesId: UUID.randomUUID().toString()] if (matcher.find()) { - result.commonName = matcher.group(1) - result.taxonRank = matcher.group(2) - result.scientificName = result.name = matcher.group(3) - result.name = result.scientificName ?: result.commonName ?: name + result.commonName = matcher.group(1)?.trim() + result.taxonRank = matcher.group(2)?.trim() + result.scientificName = result.name = matcher.group(3)?.trim() + result.name = (result.scientificName ?: result.commonName ?: name)?.trim() } metadataService.autoPopulateSpeciesData(result) diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 9953f58a8..02cd453d7 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -486,7 +486,16 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null + + when: // no scientific name + result = service.transformSpeciesName("Frogs [Class] (scientific: )") + outputSpeciesId = result.remove("outputSpeciesId") + + then: + outputSpeciesId != null + result == [name: "Frogs", scientificName: "", guid: "A_GUID", commonName: "Frogs", taxonRank: "Class"] 1 * metadataService.autoPopulateSpeciesData(_) >> null } From 5c5e786272eb94b6345c37551f226d1a2f2429e8 Mon Sep 17 00:00:00 2001 From: temi Date: Tue, 9 Apr 2024 12:05:11 +1000 Subject: [PATCH 32/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixed getting start and end dates - corrected data format of section view model --- .../au/org/ala/ecodata/ParatooService.groovy | 16 ++++++++-------- .../ecodata/paratoo/ParatooProtocolConfig.groovy | 7 +++++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 26f9e734c..1528cd0d2 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -1672,22 +1672,22 @@ class ParatooService { type : "section", title: getLabel(component, name), boxed: true, - items: [ + items: [[ type : "repeat", source: dataModel.name, - items : [ + items : [[ type : "row", "class": "output-section", - items : [ + items : [[ type : "col", items: columns - ] - ] - ] + ]] + ]] + ]] ], component, name) - if (dataModel.isObject) { - viewModel.items.userAddedRows = false + if (dataModel.isObject && viewModel.items) { + viewModel.items.first().userAddedRows = false } viewModel diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index e19c27a37..11fb4a650 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -53,6 +53,7 @@ class ParatooProtocolConfig { } def date = getProperty(surveyData, startDatePath) + date = getFirst(date) if (date == null) { if (usesPlotLayout) { date = getProperty(surveyData, plotVisitStartDatePath) @@ -61,9 +62,9 @@ class ParatooProtocolConfig { date = getPropertyFromSurvey(surveyData, startDatePath) } + date = getFirst(date) } - date = getFirst(date) removeMilliseconds(date) } @@ -78,6 +79,7 @@ class ParatooProtocolConfig { } def date = getProperty(surveyData, endDatePath) + date = getFirst(date) if (date == null) { if (usesPlotLayout) { date = getProperty(surveyData, plotVisitEndDatePath) @@ -88,9 +90,10 @@ class ParatooProtocolConfig { else { date = getPropertyFromSurvey(surveyData, endDatePath) } + + date = getFirst(date) } - date = getFirst(date) removeMilliseconds(date) } From e68ecec474ab1e3df31f23e7175a8bfe0cc56b37 Mon Sep 17 00:00:00 2001 From: temi Date: Tue, 9 Apr 2024 16:52:59 +1000 Subject: [PATCH 33/52] AtlasOfLivingAustralia/fieldcapture#3049 - better species matching against bie --- .../au/org/ala/ecodata/MetadataService.groovy | 18 ++++++++++-------- .../au/org/ala/ecodata/ParatooService.groovy | 8 ++++++++ .../org/ala/ecodata/ParatooServiceSpec.groovy | 4 ++-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy index 0f88754ee..1ab861471 100644 --- a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy +++ b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy @@ -844,14 +844,16 @@ class MetadataService { data } - Map autoPopulateSpeciesData(Map data){ - if (!data?.guid && (data?.scientificName ?: data?.commonName)) { - def result = speciesReMatchService.searchBie(data.scientificName?: data.commonName, 10) - // only if there is a single match - if (result?.autoCompleteList?.size() == 1) { - data.guid = result?.autoCompleteList[0]?.guid - data.commonName = data.commonName ?: result?.autoCompleteList[0]?.commonName - data.scientificName = data.scientificName ?: result?.autoCompleteList[0]?.name + Map autoPopulateSpeciesData (Map data, int limit = 10) { + String searchName = (data?.scientificName)?.trim() + if (!data?.guid && (searchName)) { + def result = speciesReMatchService.searchBie(searchName, limit) + // find the name that exactly matches the search name + def bestMatch = result?.autoCompleteList?.find { it.matchedNames?.findResult { String name -> name.equalsIgnoreCase(searchName) } } + if (bestMatch) { + data.guid = bestMatch?.guid + data.commonName = data.commonName ?: bestMatch?.commonName + data.scientificName = data.scientificName ?: bestMatch?.name } } diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 1528cd0d2..16a6fa2b2 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -1932,6 +1932,14 @@ class ParatooService { } metadataService.autoPopulateSpeciesData(result) + // try again with common name + if ((result.guid == null) && result.commonName) { + def speciesObject = [scientificName: result.commonName] + metadataService.autoPopulateSpeciesData(speciesObject) + result.guid = speciesObject.guid + result.scientificName = result.scientificName ?: speciesObject.scientificName + } + // record is only created if guid is present result.guid = result.guid ?: "A_GUID" result diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 02cd453d7..d41443ed5 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -487,7 +487,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null + 2 * metadataService.autoPopulateSpeciesData(_) >> null when: // no scientific name result = service.transformSpeciesName("Frogs [Class] (scientific: )") @@ -496,7 +496,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null + 2 * metadataService.autoPopulateSpeciesData(_) >> null } void "buildRelationshipTree should build relationship tree correctly"() { From 2e5270303112e3c74c0af653aff4c90b5ab5d336 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 10 Apr 2024 05:57:47 +1000 Subject: [PATCH 34/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixed failing test --- .../services/au/org/ala/ecodata/ParatooService.groovy | 8 +++++--- .../groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 16a6fa2b2..8226a4d63 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -1925,10 +1925,12 @@ class ParatooService { Map result = [name: name, scientificName: name, commonName: name, outputSpeciesId: UUID.randomUUID().toString()] if (matcher.find()) { - result.commonName = matcher.group(1)?.trim() + String commonName = matcher.group(1)?.trim() + String scientificName = matcher.group(3)?.trim() + result.commonName = commonName ?: result.commonName result.taxonRank = matcher.group(2)?.trim() - result.scientificName = result.name = matcher.group(3)?.trim() - result.name = (result.scientificName ?: result.commonName ?: name)?.trim() + result.scientificName = scientificName ?: commonName ?: result.scientificName + result.name = scientificName ?: commonName ?: result.name } metadataService.autoPopulateSpeciesData(result) diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index d41443ed5..0f7c8fc32 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -495,7 +495,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> null } From 0e9806ee21c51259c54840dc1832aacc25c855d5 Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 10 Apr 2024 12:25:59 +1000 Subject: [PATCH 35/52] AtlasOfLivingAustralia/fieldcapture#3049 - improved species regex and finding match from bie output --- .../services/au/org/ala/ecodata/MetadataService.groovy | 9 ++++++++- .../services/au/org/ala/ecodata/ParatooService.groovy | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy index 1ab861471..ff5ff114c 100644 --- a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy +++ b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy @@ -849,7 +849,14 @@ class MetadataService { if (!data?.guid && (searchName)) { def result = speciesReMatchService.searchBie(searchName, limit) // find the name that exactly matches the search name - def bestMatch = result?.autoCompleteList?.find { it.matchedNames?.findResult { String name -> name.equalsIgnoreCase(searchName) } } + def bestMatch = result?.autoCompleteList?.find { + it.matchedNames?.findResult { String name -> + name.equalsIgnoreCase(searchName) + || name.equalsIgnoreCase(data.name) + || name.equalsIgnoreCase(data.commonName) + } + } + if (bestMatch) { data.guid = bestMatch?.guid data.commonName = data.commonName ?: bestMatch?.commonName diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 8226a4d63..4c720147b 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -1919,7 +1919,7 @@ class ParatooService { return null } - String regex = "([^\\[]*)\\[(.*)\\]\\s*\\(scientific:\\s*(.*?)\\)" + String regex = "([^\\[\\(]*)(?:\\[(.*)\\])?\\s*(?:\\(scientific:\\s*(.*?)\\))?" Pattern pattern = Pattern.compile(regex) Matcher matcher = pattern.matcher(name) Map result = [name: name, scientificName: name, commonName: name, outputSpeciesId: UUID.randomUUID().toString()] From f81d1c5fdad963643cbdb09573524a64e444832a Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 11 Apr 2024 06:41:03 +1000 Subject: [PATCH 36/52] AtlasOfLivingAustralia/fieldcapture#3049 - removing feature to rearrange data model according to relationship --- .../au/org/ala/ecodata/ParatooService.groovy | 36 ++++++++++++++----- .../paratoo/ParatooProtocolConfig.groovy | 20 +++++++++++ 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 4c720147b..15183deb5 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -7,6 +7,7 @@ import grails.async.Promise import grails.converters.JSON import grails.core.GrailsApplication import groovy.util.logging.Slf4j +import javassist.NotFoundException import java.util.regex.Matcher import java.util.regex.Pattern @@ -585,13 +586,26 @@ class ParatooService { } - mapProtocolToActivityForm(protocol, form, protocolConfig) - form.save() + try { + mapProtocolToActivityForm(protocol, form, protocolConfig) + form.save() - if (form.hasErrors()) { - result.errors << form.errors - log.warn "Error saving form with id: " + id + ", name: " + name + if (form.hasErrors()) { + result.errors << form.errors + log.warn "Error saving form with id: " + id + ", name: " + name + } + } + catch (NotFoundException e) { + String error = "Error: No protocol definition found in swagger documentation for protocol: " + name + result.errors << error + log.error error + } + catch (Exception e) { + String error = "Error: Unable to save form for protocol: " + name + result.errors << error + log.error error } + } else { String error = "Error: No valid guid found for protocol: " + name result.errors << error @@ -1110,15 +1124,19 @@ class ParatooService { Map template = [dataModel: [], viewModel: [], modelName: capitalizeModelName(protocol.attributes.name), record: true, relationships: [ecodata: [:], apiOutput: [:]]] Map properties = deepCopy(findProtocolEndpointDefinition(protocol, documentation)) + if (properties == null) { + throw new NotFoundException("No protocol endpoint found for ${protocol.attributes.endpointPrefix}/bulk") + } + resolveReferences(properties, components) Map cleanedProperties = cleanSwaggerDefinition(properties) cleanedProperties = deepCopy(cleanedProperties) - template.relationships.ecodata = buildTreeRelationshipOfModels(cleanedProperties) - template.relationships.apiOutput = buildPathToModel(cleanedProperties) +// template.relationships.ecodata = buildTreeRelationshipOfModels(cleanedProperties) +// template.relationships.apiOutput = buildPathToModel(cleanedProperties) println((template.relationships.apiOutput as JSON).toString(true)) println((template.relationships.ecodata as JSON).toString(true)) resolveModelReferences(cleanedProperties, components) - cleanedProperties = rearrangePropertiesAccordingToModelRelationship(cleanedProperties, template.relationships.apiOutput, template.relationships.ecodata) +// cleanedProperties = rearrangePropertiesAccordingToModelRelationship(cleanedProperties, template.relationships.apiOutput, template.relationships.ecodata) cleanedProperties = deepCopy(cleanedProperties) log.debug((properties as JSON).toString()) @@ -1126,7 +1144,7 @@ class ParatooService { template.dataModel.addAll(grailsApplication.config.getProperty("paratoo.defaultPlotLayoutDataModels", List)) template.viewModel.addAll(grailsApplication.config.getProperty("paratoo.defaultPlotLayoutViewModels", List)) } - cleanedProperties.properties.each { String name, def definition -> + cleanedProperties.each { String name, def definition -> if (definition instanceof Map) { modelVisitStack.push(name) Map result = convertToDataModelAndViewModel(definition, documentation, name, null, modelVisitStack, 0, name, config) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 11fb4a650..d134e9b34 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -118,6 +118,26 @@ class ParatooProtocolConfig { extractGeometryFromSiteData(geometryData) } + String getSurveyAttributeName() { + String surveyAttribute = apiEndpoint + if(surveyAttribute?.endsWith('s')) { + surveyAttribute = surveyAttribute.substring(0, surveyAttribute.length() - 1) + } + + surveyAttribute + } + + Map getSurveyDataFromObservation (Map observation) { + String surveyAttribute = getSurveyAttributeName() + + def survey = observation[surveyAttribute] + if (survey instanceof List) { + return survey[0] + } + + survey + } + private List extractFeatures (Map observation, ActivityForm form) { List features = [] form.sections.each { FormSection section -> From 0f29d4381fd4e1ca98129fa31475d86a89b1e7e9 Mon Sep 17 00:00:00 2001 From: temi Date: Thu, 11 Apr 2024 11:32:38 +1000 Subject: [PATCH 37/52] AtlasOfLivingAustralia/fieldcapture#3049 - added site id to plot-layout - remove dwcAttribute from plot-layout - added more data models to ignore list --- grails-app/conf/application.groovy | 1 - .../au/org/ala/ecodata/ParatooService.groovy | 13 +++++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index d410ed7ca..20488362a 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -1399,7 +1399,6 @@ paratoo.defaultPlotLayoutDataModels = [ [ dataType: "geoMap", name: "plot_layout", - dwcAttribute: "verbatimCoordinates", validate: "required", columns: [ [ diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 15183deb5..5574ee5e9 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -46,7 +46,7 @@ class ParatooService { static final List PARATOO_IGNORE_MODEL_LIST = [ 'created_at', 'createdAt', 'updated_at', 'updatedAt', 'created_by', 'createdBy', 'updated_by', 'updatedBy', 'published_at', 'publishedAt', 'x-paratoo-file-type', PARATOO_DATAMODEL_PLOT_LAYOUT, PARATOO_DATAMODEL_PLOT_SELECTION, - PARATOO_DATAMODEL_PLOT_VISIT + PARATOO_DATAMODEL_PLOT_VISIT, 'plot-visit', 'plot-selection', 'plot-layout' ] static final List PARATOO_IGNORE_X_MODEL_REF_LIST_MINIMUM = [ 'file', 'admin::user' @@ -280,6 +280,10 @@ class ParatooService { // make sure activity has not been created for this data set if (!dataSet.activityId || forceActivityCreation) { + // plot layout is of type geoMap. Therefore, expects a site id. + if (surveyDataAndObservations.containsKey(PARATOO_DATAMODEL_PLOT_LAYOUT) && dataSet.siteId) { + surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_LAYOUT] = dataSet.siteId + } String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet.siteId, userId) List records = recordService.getAllByActivity(activityId) dataSet.areSpeciesRecorded = records?.size() > 0 @@ -384,7 +388,7 @@ class ParatooService { * @param protocolId * @return */ - private ParatooProtocolConfig getProtocolConfig(String protocolId) { + ParatooProtocolConfig getProtocolConfig(String protocolId) { String result = settingService.getSetting(PARATOO_PROTOCOL_DATA_MAPPING_KEY) Map protocolDataConfig = JSON.parse(result ?: '{}') Map config = protocolDataConfig[protocolId] @@ -1131,10 +1135,11 @@ class ParatooService { resolveReferences(properties, components) Map cleanedProperties = cleanSwaggerDefinition(properties) cleanedProperties = deepCopy(cleanedProperties) +// rearrange models not working for protocols multiple relationship between models. Disabling it for now. // template.relationships.ecodata = buildTreeRelationshipOfModels(cleanedProperties) // template.relationships.apiOutput = buildPathToModel(cleanedProperties) - println((template.relationships.apiOutput as JSON).toString(true)) - println((template.relationships.ecodata as JSON).toString(true)) +// println((template.relationships.apiOutput as JSON).toString(true)) +// println((template.relationships.ecodata as JSON).toString(true)) resolveModelReferences(cleanedProperties, components) // cleanedProperties = rearrangePropertiesAccordingToModelRelationship(cleanedProperties, template.relationships.apiOutput, template.relationships.ecodata) cleanedProperties = deepCopy(cleanedProperties) From c77b987f986bb0afddb392d25a507434a9d4c62b Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 12 Apr 2024 05:54:36 +1000 Subject: [PATCH 38/52] AtlasOfLivingAustralia/fieldcapture#3049 - create new site for non-plot protocols - changed name of non-plot site to include protocol name and start date --- .../services/au/org/ala/ecodata/ParatooService.groovy | 9 ++++++--- src/main/groovy/au/org/ala/ecodata/DateUtil.groovy | 7 +++++++ .../org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy | 8 ++++++-- src/test/groovy/au/org/ala/ecodata/DateUtilSpec.groovy | 8 ++++++++ .../ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy | 7 +++++-- 5 files changed, 32 insertions(+), 7 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 5574ee5e9..b42762dec 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -265,8 +265,6 @@ class ParatooService { ParatooProtocolConfig config = getProtocolConfig(surveyId.protocolId) config.surveyId = surveyId ActivityForm form = ActivityForm.findByExternalId(surveyId.protocolId) - // get survey data from reverse lookup response - Map surveyData = config.getSurveyData(surveyDataAndObservations) // add plot data to survey observations addPlotDataToObservations(surveyDataAndObservations, config) rearrangeSurveyData(surveyDataAndObservations, surveyDataAndObservations, form.sections[0].template.relationships.ecodata, form.sections[0].template.relationships.apiOutput) @@ -535,7 +533,12 @@ class ParatooService { if (externalId) { siteProps.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: externalId)] } - Site site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, externalId) + Site site + // create new site for every non-plot submission + if (config.usesPlotLayout) { + site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, externalId) + } + Map result if (!site) { result = siteService.create(siteProps) diff --git a/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy b/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy index 4e8ad3499..97fa67bc4 100644 --- a/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy +++ b/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy @@ -1,5 +1,7 @@ package au.org.ala.ecodata +import au.org.ala.ecodata.converter.ISODateBindingConverter + import java.math.MathContext import java.math.RoundingMode import java.text.DecimalFormat @@ -127,4 +129,9 @@ class DateUtil { } return timeZone } + + static String convertUTCDateToStringInTimeZone (String dateStr, TimeZone clientTimezone = TimeZone.default, String format = "dd/MM/yyyy HH:mm:ss Z z") { + Date date = new ISODateBindingConverter().convert(dateStr, ISODateBindingConverter.FORMAT) + date.format(format, clientTimezone) + } } diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index d134e9b34..eeb070f20 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -39,6 +39,7 @@ class ParatooProtocolConfig { Map overrides = [dataModel: [:], viewModel: [:]] ParatooCollectionId surveyId + TimeZone clientTimeZone private static String removeMilliseconds(String isoDateWithMillis) { if (!isoDateWithMillis) { @@ -232,12 +233,15 @@ class ParatooProtocolConfig { if (features) { List featureGeometries = features.collect { it.geometry } Geometry geometry = GeometryUtils.getFeatureCollectionConvexHull(featureGeometries) + String startDateInString = getStartDate(output) + startDateInString = DateUtil.convertUTCDateToStringInTimeZone(startDateInString, clientTimeZone?:TimeZone.default) + String name = "${form.name} site - ${startDateInString}" geoJson = [ type: 'Feature', geometry: GeometryUtils.geometryToGeoJsonMap(geometry), properties: [ - name: "Convex Hull", - description: "Convex Hull of ${features?.size()} feature(s)" + name: name, + description: "${name} (convex hull of all features)", ], features: features ] diff --git a/src/test/groovy/au/org/ala/ecodata/DateUtilSpec.groovy b/src/test/groovy/au/org/ala/ecodata/DateUtilSpec.groovy index 4378f7d72..00887738d 100644 --- a/src/test/groovy/au/org/ala/ecodata/DateUtilSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/DateUtilSpec.groovy @@ -37,4 +37,12 @@ class DateUtilSpec extends Specification { expect: DateUtil.formatWithMilliseconds(DateUtil.parseWithMilliseconds(date)) == date } + + def "A date can be parsed and displayed in provided timezone"(){ + given: + String date = "2021-06-30T00:12:33Z" + + expect: + DateUtil.convertUTCDateToStringInTimeZone(date, TimeZone.getTimeZone("Australia/Sydney"), "dd/MM/yyyy") == "30/06/2021" + } } diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index af5d73f00..430da6570 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -1,6 +1,7 @@ package au.org.ala.ecodata.paratoo import au.org.ala.ecodata.ActivityForm +import au.org.ala.ecodata.DateUtil import au.org.ala.ecodata.ExternalId import au.org.ala.ecodata.FormSection import au.org.ala.ecodata.ParatooService @@ -250,9 +251,11 @@ class ParatooProtocolConfigSpec extends Specification { when: ParatooProtocolConfig config = new ParatooProtocolConfig(opportunisticSurveyConfig) + config.clientTimeZone = TimeZone.getTimeZone("Australia/Sydney") config.surveyId = ParatooCollectionId.fromMap([survey_metadata: response.survey_metadata]) transformData(surveyObservations, activityForm, config) String start_date = config.getStartDate(surveyObservations) + String startDateInDefaultTimeZone = DateUtil.convertUTCDateToStringInTimeZone(start_date, config.clientTimeZone) String end_date = config.getEndDate(surveyObservations) Map geoJson = config.getGeoJson(surveyObservations, activityForm) @@ -264,8 +267,8 @@ class ParatooProtocolConfigSpec extends Specification { geometry : [type: "Point", coordinates: [138.63, -35.0005]], features : [[type: "Feature", geometry: [type: "Point", coordinates: [138.63, -35.0005]], properties:[name:"Point", externalId:40, id:"aParatooForm 1-1"]]], properties: [ - name : "Convex Hull", - description: "Convex Hull of 1 feature(s)" + name : "aParatooForm 1 site - ${startDateInDefaultTimeZone}", + description: "aParatooForm 1 site - ${startDateInDefaultTimeZone} (convex hull of all features)" ] ] } From 8a3e40172be8052aa9c80345e2709a2f6c6b3494 Mon Sep 17 00:00:00 2001 From: temi Date: Mon, 15 Apr 2024 17:08:14 +1000 Subject: [PATCH 39/52] AtlasOfLivingAustralia/fieldcapture#3049 - helper functions to resubmit dataSet & to see autogenerated protocol activity form - added ability to create polygon feature - added capability to recognise more places to add feature data type --- grails-app/conf/application.groovy | 15 +--- .../au/org/ala/ecodata/AdminController.groovy | 57 +++++++++++++ .../org/ala/ecodata/ParatooController.groovy | 1 - .../au/org/ala/ecodata/ParatooService.groovy | 80 ++++++++++++++----- .../paratoo/ParatooProtocolConfig.groovy | 26 +++--- .../org/ala/ecodata/ParatooServiceSpec.groovy | 65 +++++++++++++++ 6 files changed, 200 insertions(+), 44 deletions(-) diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy index 20488362a..3a77d5315 100644 --- a/grails-app/conf/application.groovy +++ b/grails-app/conf/application.groovy @@ -106,6 +106,9 @@ if (!google.geocode.url) { if (!temp.file.cleanup.days) { temp.file.cleanup.days = 1 } +if(!paratoo.location.excluded) { + paratoo.location.excluded = ['location.vegetation-association-nvis'] +} access.expiry.maxEmails=500 @@ -1399,17 +1402,7 @@ paratoo.defaultPlotLayoutDataModels = [ [ dataType: "geoMap", name: "plot_layout", - validate: "required", - columns: [ - [ - dwcAttribute: "verbatimLatitude", - source: "plot_layoutLatitude" - ], - [ - dwcAttribute: "verbatimLongitude", - source: "plot_layoutLongitude" - ] - ] + validate: "required" ], [ dataType: "list", diff --git a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy index 99c9a8dd8..82fcd699a 100644 --- a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy @@ -1,5 +1,8 @@ package au.org.ala.ecodata +import au.org.ala.ecodata.paratoo.ParatooCollection +import au.org.ala.ecodata.paratoo.ParatooProject +import au.org.ala.ecodata.paratoo.ParatooProtocolConfig import au.org.ala.web.AlaSecured import grails.converters.JSON import grails.util.Environment @@ -725,4 +728,58 @@ class AdminController { render errors as JSON } + /** + * Re-fetch data from Paratoo. Helpful when data could not be parsed correctly the first time. + * + * @return + */ + @AlaSecured(["ROLE_ADMIN"]) + def reSubmitDataSet() { + String projectId = params.id + String dataSetId = params.dataSetId + String userId = params.userId ?: userService.getCurrentUser().userId + if (!projectId || !dataSetId || !userId) { + render text: [message: "Bad request"] as JSON, status: HttpStatus.SC_BAD_REQUEST + return + } + + ParatooCollection collection = new ParatooCollection(orgMintedUUID: dataSetId, coreProvenance: [:]) + List projects = paratooService.userProjects(userId) + ParatooProject project = projects.find {it.project.projectId == projectId } + if (project) { + paratooService.submitCollection(collection, project, userId) + render text: [message: "Submitted request to fetch data for dataSet $dataSetId in project $projectId by user $userId"] as JSON, status: HttpStatus.SC_OK, contentType: 'application/json' + } + else { + render text: [message: "Project not found"] as JSON, status: HttpStatus.SC_NOT_FOUND + } + } + + + /** + * Helper function to check the form generated for a protocol during the sync operation. + * Usual step is to update Paratoo config in DB. Use this function to check the form generated. + * @return + */ + @AlaSecured(["ROLE_ADMIN"]) + def checkActivityFormForProtocol() { + String protocolId = params.id + List protocols = paratooService.getProtocolsFromParatoo() + Map protocol = protocols.find { it.attributes.identifier == protocolId } + if (!protocol) { + render text: [message: "Protocol not found"] as JSON, status: HttpStatus.SC_NOT_FOUND, contentType: 'application/json' + return + } + + Map documentation = paratooService.getParatooSwaggerDocumentation() + ParatooProtocolConfig config = paratooService.getProtocolConfig(protocolId) + if (!config) { + render text: [message: "Protocol config not found"] as JSON, status: HttpStatus.SC_NOT_FOUND, contentType: 'application/json' + return + } + + Map template = paratooService.buildTemplateForProtocol(protocol, documentation, config) + render text: template as JSON, status: HttpStatus.SC_OK, contentType: 'application/json' + } + } diff --git a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy index 0521d8cec..577908d99 100644 --- a/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy @@ -26,7 +26,6 @@ import javax.ws.rs.GET import javax.ws.rs.POST import javax.ws.rs.PUT import javax.ws.rs.Path - // Requiring these scopes will guarantee we can get a valid userId out of the process. @Slf4j @au.ala.org.ws.security.RequireApiKey(scopes = ["profile", "openid"]) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index b42762dec..809fbcac2 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -46,7 +46,7 @@ class ParatooService { static final List PARATOO_IGNORE_MODEL_LIST = [ 'created_at', 'createdAt', 'updated_at', 'updatedAt', 'created_by', 'createdBy', 'updated_by', 'updatedBy', 'published_at', 'publishedAt', 'x-paratoo-file-type', PARATOO_DATAMODEL_PLOT_LAYOUT, PARATOO_DATAMODEL_PLOT_SELECTION, - PARATOO_DATAMODEL_PLOT_VISIT, 'plot-visit', 'plot-selection', 'plot-layout' + PARATOO_DATAMODEL_PLOT_VISIT, 'plot-visit', 'plot-selection', 'plot-layout', 'survey_metadata' ] static final List PARATOO_IGNORE_X_MODEL_REF_LIST_MINIMUM = [ 'file', 'admin::user' @@ -56,7 +56,9 @@ class ParatooService { ] + PARATOO_IGNORE_X_MODEL_REF_LIST_MINIMUM static final String PARATOO_WORKFLOW_PLOT_LAYOUT = "plot-layout" static final String PARATOO_COMPONENT = "x-paratoo-component" + static final String PARATOO_TYPE_ARRAY = "array" static final String PARATOO_LOCATION_COMPONENT = "location.location" + static final String PARATOO_LOCATION_COMPONENT_STARTS_WITH = "location." static final String PARATOO_DATAMODEL_PLOT_LAYOUT = "plot_layout" static final String PARATOO_DATAMODEL_PLOT_SELECTION = "plot_selection" static final String PARATOO_DATAMODEL_PLOT_VISIT = "plot_visit" @@ -209,9 +211,11 @@ class ParatooService { * If species are recorded, record of it are automatically generated. * @param collection * @param project + * @param userId - user making the data submission * @return */ - Map submitCollection(ParatooCollection collection, ParatooProject project) { + Map submitCollection(ParatooCollection collection, ParatooProject project, String userId = null) { + userId = userId ?: userService.currentUserDetails?.userId Map dataSet = project.project.custom?.dataSets?.find{it.dataSetId == collection.orgMintedUUID} if (!dataSet) { @@ -220,7 +224,7 @@ class ParatooService { dataSet.progress = Activity.STARTED dataSet.surveyId.coreSubmitTime = new Date() dataSet.surveyId.survey_metadata.provenance.putAll(collection.coreProvenance) - String userId = userService.currentUserDetails?.userId + Map authHeader = getAuthHeader() Promise promise = task { asyncFetchCollection(collection, authHeader, userId, project) @@ -235,7 +239,7 @@ class ParatooService { Map asyncFetchCollection(ParatooCollection collection, Map authHeader, String userId, ParatooProject project) { Activity.withSession { session -> int counter = 0 - boolean forceActivityCreation = false + boolean forceActivityCreation = grailsApplication.config.getProperty('paratoo.forceActivityCreation', Boolean, false) Map surveyDataAndObservations = null Map response = null Map dataSet = project.project.custom?.dataSets?.find{it.dataSetId == collection.orgMintedUUID} @@ -278,6 +282,8 @@ class ParatooService { // make sure activity has not been created for this data set if (!dataSet.activityId || forceActivityCreation) { + // delete previously created activity if forceActivityCreation is true? + // plot layout is of type geoMap. Therefore, expects a site id. if (surveyDataAndObservations.containsKey(PARATOO_DATAMODEL_PLOT_LAYOUT) && dataSet.siteId) { surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_LAYOUT] = dataSet.siteId @@ -489,19 +495,26 @@ class ParatooService { case "feature": // used by protocols like bird survey where a point represents a sight a bird has been observed in a // bird survey plot - def point = output[model.name] - output[model.name] = [ - type : 'Feature', - geometry : [ - type : 'Point', - coordinates: [point.lng, point.lat] - ], - properties: [ - name : "Point", - externalId: point.id, - id: "${formName}-${featureId}" - ] - ] + def location = output[model.name] + if (location instanceof Map) { + output[model.name] = [ + type : 'Feature', + geometry : [ + type : 'Point', + coordinates: [location.lng, location.lat] + ], + properties: [ + name : "Point ${formName}-${featureId}", + externalId: location.id, + id: "${formName}-${featureId}" + ] + ] + } + else if (location instanceof List) { + String name = "Polygon ${formName}-${featureId}" + output[model.name] = ParatooProtocolConfig.createFeatureFromGeoJSON (location, name, name) + } + featureId ++ break case "image": @@ -522,13 +535,13 @@ class ParatooService { // Create a site representing the location of the collection Map geoJson = config.getGeoJson(observation, form) if (geoJson) { + Map siteProps = siteService.propertiesFromGeoJson(geoJson, 'upload') List features = geoJson?.features ?: [] geoJson.remove('features') - Map siteProps = siteService.propertiesFromGeoJson(geoJson, 'upload') + siteProps.features = features siteProps.type = Site.TYPE_SURVEY_AREA siteProps.publicationStatus = PublicationStatus.PUBLISHED siteProps.projects = [project.projectId] - siteProps.features = features String externalId = geoJson.properties?.externalId if (externalId) { siteProps.externalIds = [new ExternalId(idType: ExternalId.IdType.MONITOR_PLOT_GUID, externalId: externalId)] @@ -537,6 +550,9 @@ class ParatooService { // create new site for every non-plot submission if (config.usesPlotLayout) { site = Site.findByExternalId(ExternalId.IdType.MONITOR_PLOT_GUID, externalId) + if (site?.features) { + siteProps.features?.addAll(site.features) + } } Map result @@ -886,6 +902,10 @@ class ParatooService { Map simplifyModelStructure(Map definition) { Map simplifiedDefinition = [:] + if (definition.type == PARATOO_TYPE_ARRAY) { + definition = definition.items + } + Map properties = getModelStructureFromDefinition(definition) List required = getRequiredModels(definition) String componentName = definition[PARATOO_COMPONENT] @@ -1295,6 +1315,22 @@ class ParatooService { modelVisitStack = modelVisitStack ?: new ArrayDeque() String componentName, modelName = component[PARATOO_MODEL_REF] + /** + * Some time component definition can be like + { + "type": "array", + "items": { + "type": "integer", + "x-paratoo-file-type": ["images"], + "x-model-ref": "file" + } + } + */ + if (!modelName && (component.properties?.getAt(PARATOO_MODEL_REF) == PARATOO_FILE_MODEL_NAME)) { + component = component.properties + modelName = component[PARATOO_MODEL_REF] + } + if (PARATOO_IGNORE_MODEL_LIST.contains(name)) return else if (component[PARATOO_SPECIES_TYPE]) { @@ -1392,8 +1428,10 @@ class ParatooService { } } - static boolean isLocationObject(Map input) { - input[PARATOO_COMPONENT] == PARATOO_LOCATION_COMPONENT + boolean isLocationObject(Map input) { + ((input[PARATOO_COMPONENT] == PARATOO_LOCATION_COMPONENT) || + input[PARATOO_COMPONENT]?.startsWith(PARATOO_LOCATION_COMPONENT_STARTS_WITH)) && + !grailsApplication.config.getProperty("paratoo.location.excluded", List)?.contains(input[PARATOO_COMPONENT]) } static Map addRequiredFlag(Map component, Map dataType, List required) { diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index eeb070f20..260c8ab30 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -341,17 +341,7 @@ class ParatooProtocolConfig { String name = plotSelectionGeoJson.properties.name + ' - ' + plotLayoutTypeLabel + ' (' + plotLayoutDimensionLabel + ')' - Map plotGeometory = toGeometry(plotLayoutPoints) - Map plotGeoJson = [ - type: 'Feature', - geometry: plotGeometory, - properties: [ - name: name, - externalId: plotLayoutId, - description: name, - notes: plotSelectionGeoJson?.properties?.notes - ] - ] + Map plotGeoJson = createFeatureFromGeoJSON(plotLayoutPoints, name, plotLayoutId, plotSelectionGeoJson?.properties?.notes) //Map faunaPlotGeoJson = toGeometry(plotLayout.fauna_plot_point) @@ -361,6 +351,20 @@ class ParatooProtocolConfig { plotGeoJson } + static Map createFeatureFromGeoJSON(List plotLayoutPoints, String name, String plotLayoutId, String notes = "") { + Map plotGeometory = toGeometry(plotLayoutPoints) + [ + type : 'Feature', + geometry : plotGeometory, + properties: [ + name : name, + externalId : plotLayoutId, + description: name, + notes : notes + ] + ] + } + static Map toGeometry(List points) { List coords = points?.findAll { !exclude(it) }.collect { [it.lng, it.lat] diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 0f7c8fc32..cb9925674 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -38,6 +38,8 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Tue, 16 Apr 2024 06:43:52 +1000 Subject: [PATCH 40/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixed broken tests --- .../au/org/ala/ecodata/ParatooService.groovy | 4 +- .../paratoo/ParatooProtocolConfig.groovy | 2 +- .../org/ala/ecodata/ParatooServiceSpec.groovy | 44 +++++++++---------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 809fbcac2..30df744cf 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -902,8 +902,8 @@ class ParatooService { Map simplifyModelStructure(Map definition) { Map simplifiedDefinition = [:] - if (definition.type == PARATOO_TYPE_ARRAY) { - definition = definition.items + if ((definition.type == PARATOO_TYPE_ARRAY) && definition.items) { + definition << definition.items } Map properties = getModelStructureFromDefinition(definition) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 260c8ab30..532ca1ff3 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -351,7 +351,7 @@ class ParatooProtocolConfig { plotGeoJson } - static Map createFeatureFromGeoJSON(List plotLayoutPoints, String name, String plotLayoutId, String notes = "") { + static Map createFeatureFromGeoJSON(List plotLayoutPoints, String name, def plotLayoutId, String notes = "") { Map plotGeometory = toGeometry(plotLayoutPoints) [ type : 'Feature', diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index cb9925674..9a7fd6539 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -1203,28 +1203,28 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Tue, 16 Apr 2024 07:33:54 +1000 Subject: [PATCH 41/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixed broken tests --- .../ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index 430da6570..2e7a079ac 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -85,7 +85,7 @@ class ParatooProtocolConfigSpec extends Specification { type : "Feature", geometry : [type: 'Point', coordinates: [149.0651536, -35.2592398]], properties: [ - name : "Point", + name : "Point aParatooForm 1-1", externalId: 44, id : "aParatooForm 1-1" ] @@ -198,7 +198,7 @@ class ParatooProtocolConfigSpec extends Specification { type : "Point", coordinates: [149.0651491, -35.2592444] ], - properties: ["name": "Point", externalId: 37, id: "aParatooForm 2-1"] + properties: ["name": "Point aParatooForm 2-1", externalId: 37, id: "aParatooForm 2-1"] ]] } @@ -265,7 +265,7 @@ class ParatooProtocolConfigSpec extends Specification { geoJson == [ type : "Feature", geometry : [type: "Point", coordinates: [138.63, -35.0005]], - features : [[type: "Feature", geometry: [type: "Point", coordinates: [138.63, -35.0005]], properties:[name:"Point", externalId:40, id:"aParatooForm 1-1"]]], + features : [[type: "Feature", geometry: [type: "Point", coordinates: [138.63, -35.0005]], properties:[name:"Point aParatooForm 1-1", externalId:40, id:"aParatooForm 1-1"]]], properties: [ name : "aParatooForm 1 site - ${startDateInDefaultTimeZone}", description: "aParatooForm 1 site - ${startDateInDefaultTimeZone} (convex hull of all features)" From 0a07c9596a3fd174b7c18e153ea7b84472e72e8d Mon Sep 17 00:00:00 2001 From: temi Date: Tue, 16 Apr 2024 11:14:00 +1000 Subject: [PATCH 42/52] AtlasOfLivingAustralia/fieldcapture#3049 - added default values for format, sizeUnknown --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 30df744cf..f99196c6e 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -18,6 +18,7 @@ import static grails.async.Promises.task */ @Slf4j class ParatooService { + static final String DATASET_DATABASE_TABLE = 'Database Table' static final int PARATOO_MAX_RETRIES = 3 static final String PARATOO_PROTOCOL_PATH = '/protocols' static final String PARATOO_DATA_PATH = '/protocols/reverse-lookup' @@ -296,6 +297,8 @@ class ParatooService { dataSet.startDate = config.getStartDate(surveyDataAndObservations) dataSet.endDate = config.getEndDate(surveyDataAndObservations) + dataSet.format = DATASET_DATABASE_TABLE + dataSet.sizeUnknown = true projectService.update([custom: project.project.custom], project.id, false) } From 5d006d9a711c86b4d80d1ca035ecdfc3436dcb44 Mon Sep 17 00:00:00 2001 From: temi Date: Tue, 16 Apr 2024 11:31:44 +1000 Subject: [PATCH 43/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixed failed test --- src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 9a7fd6539..0f7c1aa5d 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -182,7 +182,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Wed, 17 Apr 2024 11:58:00 +1000 Subject: [PATCH 44/52] AtlasOfLivingAustralia/fieldcapture#3049 - added capability to create line geojson --- .../au/org/ala/ecodata/ParatooService.groovy | 19 ++++-- .../paratoo/ParatooProtocolConfig.groovy | 25 +++++++- .../org/ala/ecodata/ParatooServiceSpec.groovy | 59 +++++++++++++++++++ .../paratoo/ParatooProtocolConfigSpec.groovy | 24 ++++++++ 4 files changed, 120 insertions(+), 7 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index f99196c6e..51448cbe4 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -274,7 +274,7 @@ class ParatooService { addPlotDataToObservations(surveyDataAndObservations, config) rearrangeSurveyData(surveyDataAndObservations, surveyDataAndObservations, form.sections[0].template.relationships.ecodata, form.sections[0].template.relationships.apiOutput) // transform data to make it compatible with data model - surveyDataAndObservations = recursivelyTransformData(form.sections[0].template.dataModel, surveyDataAndObservations, form.name) + surveyDataAndObservations = recursivelyTransformData(form.sections[0].template.dataModel, surveyDataAndObservations, form.name, config) // If we are unable to create a site, null will be returned - assigning a null siteId is valid. if (!dataSet.siteId) { @@ -463,7 +463,7 @@ class ParatooService { * @param path * @return */ - def recursivelyTransformData(List dataModel, Map output, String formName = "", int featureId = 1) { + def recursivelyTransformData(List dataModel, Map output, String formName = "", int featureId = 1, ParatooProtocolConfig config = null) { dataModel?.each { Map model -> switch (model.dataType) { case "list": @@ -482,7 +482,7 @@ class ParatooService { rows?.each { row -> if (row != null) { - recursivelyTransformData(model.columns, row, formName, featureId) + recursivelyTransformData(model.columns, row, formName, featureId, config) } } break @@ -514,8 +514,17 @@ class ParatooService { ] } else if (location instanceof List) { - String name = "Polygon ${formName}-${featureId}" - output[model.name] = ParatooProtocolConfig.createFeatureFromGeoJSON (location, name, name) + String name + switch (config?.geometryType) { + case "LineString": + name = "LineString ${formName}-${featureId}" + output[model.name] = ParatooProtocolConfig.createLineStringFeatureFromGeoJSON (location, name, null, name) + break + default: + name = "Polygon ${formName}-${featureId}" + output[model.name] = ParatooProtocolConfig.createFeatureFromGeoJSON (location, name, null, name) + break + } } featureId ++ diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 532ca1ff3..3a0e9f64b 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -352,10 +352,14 @@ class ParatooProtocolConfig { } static Map createFeatureFromGeoJSON(List plotLayoutPoints, String name, def plotLayoutId, String notes = "") { - Map plotGeometory = toGeometry(plotLayoutPoints) + Map plotGeometry = toGeometry(plotLayoutPoints) + createFeatureObject(plotGeometry, name, plotLayoutId, notes) + } + + static Map createFeatureObject(Map plotGeometry, String name, plotLayoutId, String notes = "") { [ type : 'Feature', - geometry : plotGeometory, + geometry : plotGeometry, properties: [ name : name, externalId : plotLayoutId, @@ -377,6 +381,23 @@ class ParatooProtocolConfig { plotGeometry } + static Map createLineStringFeatureFromGeoJSON (List plotLayoutPoints, String name, def plotLayoutId, String notes = "") { + Map plotGeometry = toLineStringGeometry(plotLayoutPoints) + createFeatureObject(plotGeometry, name, plotLayoutId, notes) + } + + static Map toLineStringGeometry(List points) { + List coords = points?.collect { + [it.lng, it.lat] + } + Map plotGeometry = coords ? [ + type : 'LineString', + coordinates: coords + ] : null + + plotGeometry + } + private static List closePolygonIfRequired(List points) { if (points[0][0] != points[-1][0] || points[0][1] != points[-1][1]) { points << points[0] diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 0f7c1aa5d..4dbb1dfce 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -1200,6 +1200,65 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Wed, 17 Apr 2024 14:08:52 +1000 Subject: [PATCH 45/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixes broken test --- .../paratoo/ParatooProtocolConfigSpec.groovy | 39 +++++++++++-------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index 1bdd21203..81a5ed8e5 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -274,27 +274,32 @@ class ParatooProtocolConfigSpec extends Specification { } def "Should create line GeoJSON objects" () { - expect: - ParatooProtocolConfig.toLineStringGeometry([[lat: 1, lng: 2], [lat: 3, lng: 4]]) == [ + when: + def result = ParatooProtocolConfig.toLineStringGeometry([[lat: 1, lng: 2], [lat: 3, lng: 4]]) + + then: + result == [ type : 'LineString', coordinates: [[2, 1], [4, 3]] ] - expect: - ParatooProtocolConfig.createLineStringFeatureFromGeoJSON([[lat: 1, lng: 2], [lat: 3, lng: 4], [lat: 5, lng: 6]], "test name", 1, "test notes") == - [ - "type" : "Feature", - "geometry" : [ - type : 'LineString', - coordinates: [[2, 1], [4, 3], [6, 5]] - ], - "properties": [ - "name" : "test name", - "externalId" : 1, - "description": "test name", - "notes" : "test notes" - ] - ] + when: + result = ParatooProtocolConfig.createLineStringFeatureFromGeoJSON([[lat: 1, lng: 2], [lat: 3, lng: 4], [lat: 5, lng: 6]], "test name", 1, "test notes") + + then: + result == [ + "type" : "Feature", + "geometry" : [ + type : 'LineString', + coordinates: [[2, 1], [4, 3], [6, 5]] + ], + "properties": [ + "name" : "test name", + "externalId" : 1, + "description": "test name", + "notes" : "test notes" + ] + ] } def transformData(Map surveyDataAndObservations, ActivityForm form, ParatooProtocolConfig config) { From 3dcb1badb059c05c51fc2c045c3bfbb62382376c Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 17 Apr 2024 14:20:45 +1000 Subject: [PATCH 46/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixes broken test --- grails-app/services/au/org/ala/ecodata/ParatooService.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 51448cbe4..699c72c5f 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -274,7 +274,7 @@ class ParatooService { addPlotDataToObservations(surveyDataAndObservations, config) rearrangeSurveyData(surveyDataAndObservations, surveyDataAndObservations, form.sections[0].template.relationships.ecodata, form.sections[0].template.relationships.apiOutput) // transform data to make it compatible with data model - surveyDataAndObservations = recursivelyTransformData(form.sections[0].template.dataModel, surveyDataAndObservations, form.name, config) + surveyDataAndObservations = recursivelyTransformData(form.sections[0].template.dataModel, surveyDataAndObservations, form.name, 1, config) // If we are unable to create a site, null will be returned - assigning a null siteId is valid. if (!dataSet.siteId) { From 1beb3b2a1bc01bc3a38cc86b4d82291f53fc3d2d Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 17 Apr 2024 14:48:20 +1000 Subject: [PATCH 47/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixes broken test --- .../groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index 4dbb1dfce..f17761fd9 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -1215,7 +1215,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Thu, 18 Apr 2024 06:47:44 +1000 Subject: [PATCH 48/52] AtlasOfLivingAustralia/fieldcapture#3049 - changed how start and end dates are calculated for plot based protocols - refactored code --- .../paratoo/ParatooProtocolConfig.groovy | 68 +- .../paratoo/ParatooProtocolConfigSpec.groovy | 77 +- .../floristicsStandardReverseLookup.json | 1666 ++++++++++++++++- 3 files changed, 1752 insertions(+), 59 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy index 3a0e9f64b..e6b0be669 100644 --- a/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy +++ b/src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy @@ -1,6 +1,7 @@ package au.org.ala.ecodata.paratoo import au.org.ala.ecodata.* +import au.org.ala.ecodata.converter.ISODateBindingConverter import au.org.ala.ecodata.metadata.OutputMetadata import au.org.ala.ecodata.metadata.PropertyAccessor import com.fasterxml.jackson.annotation.JsonIgnoreProperties @@ -24,6 +25,7 @@ class ParatooProtocolConfig { String endDatePath = 'end_date_time' String surveyIdPath = 'survey_metadata' String plotVisitPath = 'plot_visit' + String plotProtocolObservationDatePath = "date_time" String plotVisitStartDatePath = "${plotVisitPath}.start_date" String plotVisitEndDatePath = "${plotVisitPath}.end_date" String plotLayoutPath = "${plotVisitPath}.plot_layout" @@ -53,20 +55,46 @@ class ParatooProtocolConfig { return null } - def date = getProperty(surveyData, startDatePath) - date = getFirst(date) - if (date == null) { - if (usesPlotLayout) { - date = getProperty(surveyData, plotVisitStartDatePath) - } - else { + def date + if (usesPlotLayout) { + List dates = getDatesFromObservation(surveyData) + date = dates ? DateUtil.format(dates.first()) : null + return date + } + else { + date = getProperty(surveyData, startDatePath) + if (!date) { date = getPropertyFromSurvey(surveyData, startDatePath) } date = getFirst(date) + return removeMilliseconds(date) + } + } + + /** + * Get date from plotProtocolObservationDatePath and sort them. + * @param surveyData - reverse lookup output which includes survey and observation data + * @return + */ + List getDatesFromObservation(Map surveyData) { + Map surveysData = surveyData.findAll { key, value -> + ![ getSurveyAttributeName(), ParatooService.PARATOO_DATAMODEL_PLOT_SELECTION, + ParatooService.PARATOO_DATAMODEL_PLOT_VISIT, ParatooService.PARATOO_DATAMODEL_PLOT_LAYOUT].contains(key) + } + List result = [] + ISODateBindingConverter converter = new ISODateBindingConverter() + surveysData.each { key, value -> + def dates = getProperty(value, plotProtocolObservationDatePath) + dates = dates instanceof List ? dates : [dates] + + result.addAll(dates.collect { String date -> + date ? converter.convert(date, ISODateBindingConverter.FORMAT) : null + }) } - removeMilliseconds(date) + result = result.findAll { it != null } + result.sort() } def getPropertyFromSurvey(Map surveyData, String path) { @@ -79,23 +107,21 @@ class ParatooProtocolConfig { return null } - def date = getProperty(surveyData, endDatePath) - date = getFirst(date) - if (date == null) { - if (usesPlotLayout) { - date = getProperty(surveyData, plotVisitEndDatePath) - if (!date) { - date = getProperty(surveyData, plotVisitStartDatePath) - } - } - else { + def date + if (usesPlotLayout) { + def dates = getDatesFromObservation(surveyData) + date = dates ? DateUtil.format(dates.last()) : null + return date + } + else { + date = getProperty(surveyData, endDatePath) + if (!date) { date = getPropertyFromSurvey(surveyData, endDatePath) } date = getFirst(date) + return removeMilliseconds(date) } - - removeMilliseconds(date) } Map getSurveyId(Map surveyData) { @@ -198,7 +224,7 @@ class ParatooProtocolConfig { apiEndpoint } - def getProperty(Map surveyData, String path) { + def getProperty(def surveyData, String path) { if (!path) { return null } diff --git a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy index 81a5ed8e5..dbfbf87fa 100644 --- a/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfigSpec.groovy @@ -50,23 +50,24 @@ class ParatooProtocolConfigSpec extends Specification { [ name : "vegetation-mapping-survey", dataType: "list", + columns : [ + + ] + ], + [ + name : "vegetation-mapping-observation", + dataType: "list", columns : [ [ - name : "vegetation-mapping-observation", - dataType: "list", - columns : [ - [ - dataType: "feature", - name : "position" - ] - ] + dataType: "feature", + name : "position" ] ] ] ], relationships: [ - ecodata : ["vegetation-mapping-survey": ["vegetation-mapping-observation": [:]]], - apiOutput: ["vegetation-mapping-observation": ["vegetation-mapping-observation": [:]]] + ecodata : [:], + apiOutput: [:] ] ] ] @@ -133,8 +134,8 @@ class ParatooProtocolConfigSpec extends Specification { transformData(observation, activityForm, config) then: - config.getStartDate(observation) == "2021-08-26T11:26:54Z" - config.getEndDate(observation) == "2021-08-26T13:26:54Z" + config.getStartDate(observation) == "2024-04-08T01:23:28Z" + config.getEndDate(observation) == "2024-04-10T01:23:28Z" config.getGeoJson(observation) == [type: "Feature", geometry: [type: "Polygon", coordinates: [[[152.880694, -27.388252], [152.880651, -27.388336], [152.880518, -27.388483], [152.880389, -27.388611], [152.88028, -27.388749], [152.880154, -27.388903], [152.880835, -27.389463], [152.880644, -27.389366], [152.880525, -27.389248], [152.88035, -27.389158], [152.880195, -27.389021], [152.880195, -27.389373], [152.880797, -27.388316], [152.881448, -27.388909], [152.881503, -27.388821], [152.881422, -27.388766], [152.881263, -27.388644], [152.881107, -27.388549], [152.880939, -27.388445], [152.881314, -27.389035], [152.88122, -27.389208], [152.881089, -27.389343], [152.880973, -27.389472], [152.880916, -27.389553], [152.880694, -27.388252]]]], properties: [name: "QDASEQ0001 - Control (100 x 100)", externalId: 1, description: "QDASEQ0001 - Control (100 x 100)", notes: "some comment"]] } @@ -157,23 +158,24 @@ class ParatooProtocolConfigSpec extends Specification { [ dataType: "list", name : "basal-area-dbh-measure-survey", + columns : [ + + ] + ], + [ + dataType: "list", + name : "basal-area-dbh-measure-observation", columns : [ [ - dataType: "list", - name : "basal-area-dbh-measure-observation", - columns : [ - [ - dataType: "feature", - name : "location" - ] - ] + dataType: "feature", + name : "location" ] ] ] ], viewModel : [], relationships: [ - ecodata : ["basal-area-dbh-measure-survey":["basal-area-dbh-measure-observation": [:]]], + ecodata : [:], apiOutput: [:] ] ] @@ -183,8 +185,8 @@ class ParatooProtocolConfigSpec extends Specification { transformData(observation, activityForm, config) expect: - config.getStartDate(observation) == '2023-09-22T00:59:47Z' - config.getEndDate(observation) == "2023-09-22T00:59:47Z" + config.getStartDate(observation) == "2024-03-28T03:17:01Z" + config.getEndDate(observation) == "2024-03-28T03:17:01Z" config.getGeoJson(observation) == [ type : "Feature", geometry : [ @@ -226,23 +228,24 @@ class ParatooProtocolConfigSpec extends Specification { [ name : "opportunistic-survey", dataType: "list", - columns: [[ - name : "opportunistic-observation", - dataType: "list", - columns : [ - [ - name : "location", - dataType: "feature", - required: true, - external: true - ] - ] - ]] + columns: [] + ], + [ + name : "opportunistic-observation", + dataType: "list", + columns : [ + [ + name : "location", + dataType: "feature", + required: true, + external: true + ] + ] ] ], relationships: [ - ecodata : ["opportunistic-survey": ["opportunistic-observation": [:]]], - apiOutput: ["opportunistic-observation.opportunistic_survey": ["opportunistic-survey": [:]], "opportunistic-observation": ["opportunistic-observation": [:]]] + ecodata : [:], + apiOutput: [:] ] ] ] diff --git a/src/test/resources/paratoo/floristicsStandardReverseLookup.json b/src/test/resources/paratoo/floristicsStandardReverseLookup.json index 7c5391fdb..f89b84506 100644 --- a/src/test/resources/paratoo/floristicsStandardReverseLookup.json +++ b/src/test/resources/paratoo/floristicsStandardReverseLookup.json @@ -484,6 +484,1670 @@ "fauna_plot_point": [] } } - } + }, + "floristics-veg-voucher-lite": [ + { + "id": 8, + "field_name": "Anthocerotidae [Subclassis] (scientific: Anthocerotidae Rosenv.)", + "voucher_barcode": "1233445", + "host_species": null, + "unique_id": null, + "date_time": null, + "createdAt": "2024-04-09T01:24:21.943Z", + "updatedAt": "2024-04-09T01:24:21.943Z", + "growth_form_1": { + "id": 1, + "symbol": "V", + "label": "Sedge", + "description": "Herbaceous, usually perennial erect plant generally with a tufted habit and of the families Cyperaceae (true sedges) or Restionaceae (node sedges).", + "uri": "https://linked.data.gov.au/def/nrm/a7b8b3c8-a329-5a62-8741-e1d6efe9528b", + "createdAt": "2024-03-26T02:39:35.671Z", + "updatedAt": "2024-03-26T02:39:35.671Z" + }, + "growth_form_2": { + "id": 2, + "symbol": "X", + "label": "Grass-tree", + "description": "The 'Growth-form' represents Australian grass-trees. E.g., members of the Xanthorrhoeaceae.", + "uri": "https://linked.data.gov.au/def/nrm/1be47880-4ee6-5df9-8eda-551c58078771", + "createdAt": "2024-03-26T02:39:35.687Z", + "updatedAt": "2024-03-26T02:39:35.687Z" + }, + "habit": [ + { + "id": 5, + "lut": { + "id": 11, + "symbol": "SUC", + "label": "Succulent", + "description": "Fleshy, juicy, soft in texture and usually thickened", + "uri": "", + "createdAt": "2024-03-26T02:39:07.633Z", + "updatedAt": "2024-03-26T02:39:07.633Z" + } + } + ], + "phenology": [ + { + "id": 5, + "lut": { + "id": 9, + "symbol": "SEE", + "label": "Seedling", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:07.771Z", + "updatedAt": "2024-03-26T02:39:07.771Z" + } + } + ], + "floristics_veg_survey_lite": { + "survey_metadata": { + "id": 8, + "org_minted_uuid": "9beeba20-374a-470a-b4f4-caed34e07572", + "createdAt": "2024-03-28T03:17:25.302Z", + "updatedAt": "2024-03-28T03:17:25.302Z", + "survey_details": { + "id": 59, + "survey_model": "floristics-veg-survey-lite", + "time": "2024-03-28T03:17:01.727Z", + "uuid": "263f710d-9af5-456a-9c68-b2b675763220", + "project_id": "0d02b422-5bf7-495f-b9f2-fa0a3046937f", + "protocol_id": "cd2cbbc7-2f17-4b0f-91b4-06f46e9c90f2", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 59, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": "0.1.0-a2ba7a48", + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": "Monitor FDCP API-production", + "system_org": null + } + }, + "start_date_time": "2022-09-21T01:55:44.186Z", + "end_date_time": "2022-09-21T01:55:44.186Z", + "createdAt": "2023-09-14T06:00:17.099Z", + "updatedAt": "2023-09-14T06:00:17.099Z", + "plot_visit": { + "start_date": "2021-08-26T11:26:54.317Z", + "end_date": "2021-08-26T13:26:54.317Z", + "visit_field_name": "QLD winter survey", + "createdAt": "2023-09-14T06:00:12.120Z", + "updatedAt": "2023-09-14T06:00:17.882Z", + "plot_layout": { + "id": 1, + "replicate": 1, + "is_the_plot_permanently_marked": false, + "orientation": 1, + "createdAt": "2023-09-14T06:00:11.473Z", + "updatedAt": "2023-09-14T06:00:11.473Z", + "plot_selection": { + "plot_label": "QDASEQ0001", + "uuid": "a6f84a0e-2239-4bfa-aac6-6d2c99e252f1", + "comments": "some comment", + "createdAt": "2023-09-14T05:59:34.851Z", + "updatedAt": "2023-09-14T05:59:34.851Z", + "plot_name": { + "id": 1, + "unique_digits": "0001", + "state": { + "symbol": "QD", + "label": "Queensland", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/d9d48a48-f74e-55b7-a360-82975c6be412", + "createdAt": "2023-09-14T05:59:33.978Z", + "updatedAt": "2023-09-14T05:59:33.978Z" + }, + "program": { + "symbol": "NLP", + "label": "National Landcare Program", + "description": "", + "uri": "", + "createdAt": "2023-09-14T05:59:34.006Z", + "updatedAt": "2023-09-14T05:59:34.006Z" + }, + "bioregion": { + "symbol": "SEQ", + "label": "South Eastern Queensland", + "description": "", + "uri": "", + "createdAt": "2023-09-14T05:59:34.695Z", + "updatedAt": "2023-09-14T05:59:39.355Z", + "states": [ + { + "symbol": "NS", + "label": "New South Wales", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/14b8c3fe-3a60-5f3a-bb7a-b9cf5df49b4e", + "createdAt": "2023-09-14T05:59:33.966Z", + "updatedAt": "2023-09-14T05:59:33.966Z" + }, + { + "symbol": "QD", + "label": "Queensland", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/d9d48a48-f74e-55b7-a360-82975c6be412", + "createdAt": "2023-09-14T05:59:33.978Z", + "updatedAt": "2023-09-14T05:59:33.978Z" + } + ] + } + }, + "recommended_location": { + "id": 1, + "lat": -35.0004723472134, + "lng": 138.66067886352542 + }, + "recommended_location_point": { + "symbol": "C", + "description": "Centre", + "label": "Centre", + "uri": "", + "createdAt": "2023-09-14T05:59:34.718Z", + "updatedAt": "2023-09-14T05:59:34.718Z" + }, + "plot_selection_survey": null + }, + "plot_type": { + "symbol": "C", + "label": "Control", + "description": "", + "uri": "", + "createdAt": "2023-09-14T05:59:53.672Z", + "updatedAt": "2023-09-14T05:59:53.672Z" + }, + "permanently_marked": null, + "plot_dimensions": { + "symbol": "100m", + "label": "100 x 100", + "description": "100 m by 100 m (1 hectare)", + "uri": "", + "createdAt": "2023-09-14T05:59:53.412Z", + "updatedAt": "2023-09-14T05:59:53.412Z" + }, + "plot_points": [ + { + "id": 2, + "lat": -27.388252, + "lng": 152.880694, + "name": { + "symbol": "W5", + "description": "West 5", + "label": "West 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.787Z", + "updatedAt": "2023-09-14T05:59:34.787Z" + } + }, + { + "id": 1, + "lat": -27.388336, + "lng": 152.880651, + "name": { + "symbol": "W4", + "description": "West 4", + "label": "West 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.783Z", + "updatedAt": "2023-09-14T05:59:34.783Z" + } + }, + { + "id": 3, + "lat": -27.388483, + "lng": 152.880518, + "name": { + "symbol": "W3", + "description": "West 3", + "label": "West 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.779Z", + "updatedAt": "2023-09-14T05:59:34.779Z" + } + }, + { + "id": 4, + "lat": -27.388611, + "lng": 152.880389, + "name": { + "symbol": "W2", + "description": "West 2", + "label": "West 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.775Z", + "updatedAt": "2023-09-14T05:59:34.775Z" + } + }, + { + "id": 5, + "lat": -27.388749, + "lng": 152.88028, + "name": { + "symbol": "W1", + "description": "West 1", + "label": "West 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.771Z", + "updatedAt": "2023-09-14T05:59:34.771Z" + } + }, + { + "id": 6, + "lat": -27.388903, + "lng": 152.880154, + "name": { + "symbol": "SW", + "description": "South West", + "label": "South West", + "uri": "", + "createdAt": "2023-09-14T05:59:34.815Z", + "updatedAt": "2023-09-14T05:59:34.815Z" + } + }, + { + "id": 7, + "lat": -27.389463, + "lng": 152.880835, + "name": { + "symbol": "SE", + "description": "South East", + "label": "South East", + "uri": "", + "createdAt": "2023-09-14T05:59:34.823Z", + "updatedAt": "2023-09-14T05:59:34.823Z" + } + }, + { + "id": 8, + "lat": -27.389366, + "lng": 152.880644, + "name": { + "symbol": "S5", + "description": "South 5", + "label": "South 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.767Z", + "updatedAt": "2023-09-14T05:59:34.767Z" + } + }, + { + "id": 9, + "lat": -27.389248, + "lng": 152.880525, + "name": { + "symbol": "S4", + "description": "South 4", + "label": "South 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.763Z", + "updatedAt": "2023-09-14T05:59:34.763Z" + } + }, + { + "id": 10, + "lat": -27.389158, + "lng": 152.88035, + "name": { + "symbol": "S3", + "description": "South 3", + "label": "South 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.760Z", + "updatedAt": "2023-09-14T05:59:34.760Z" + } + }, + { + "id": 11, + "lat": -27.389021, + "lng": 152.880195, + "name": { + "symbol": "S2", + "description": "South 2", + "label": "South 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.756Z", + "updatedAt": "2023-09-14T05:59:34.756Z" + } + }, + { + "id": 12, + "lat": -27.389373, + "lng": 152.880195, + "name": { + "symbol": "S1", + "description": "South 1", + "label": "South 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.751Z", + "updatedAt": "2023-09-14T05:59:34.751Z" + } + }, + { + "id": 13, + "lat": -27.388316, + "lng": 152.880797, + "name": { + "symbol": "NW", + "description": "North West", + "label": "North West", + "uri": "", + "createdAt": "2023-09-14T05:59:34.827Z", + "updatedAt": "2023-09-14T05:59:34.827Z" + } + }, + { + "id": 14, + "lat": -27.388909, + "lng": 152.881448, + "name": { + "symbol": "NE", + "description": "North East", + "label": "North East", + "uri": "", + "createdAt": "2023-09-14T05:59:34.831Z", + "updatedAt": "2023-09-14T05:59:34.831Z" + } + }, + { + "id": 15, + "lat": -27.388821, + "lng": 152.881503, + "name": { + "symbol": "N5", + "description": "North 5", + "label": "North 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.811Z", + "updatedAt": "2023-09-14T05:59:34.811Z" + } + }, + { + "id": 16, + "lat": -27.388766, + "lng": 152.881422, + "name": { + "symbol": "N4", + "description": "North 4", + "label": "North 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.808Z", + "updatedAt": "2023-09-14T05:59:34.808Z" + } + }, + { + "id": 17, + "lat": -27.388644, + "lng": 152.881263, + "name": { + "symbol": "N3", + "description": "North 3", + "label": "North 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.803Z", + "updatedAt": "2023-09-14T05:59:34.803Z" + } + }, + { + "id": 18, + "lat": -27.388549, + "lng": 152.881107, + "name": { + "symbol": "N2", + "description": "North 2", + "label": "North 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.799Z", + "updatedAt": "2023-09-14T05:59:34.799Z" + } + }, + { + "id": 19, + "lat": -27.388445, + "lng": 152.880939, + "name": { + "symbol": "N1", + "description": "North 1", + "label": "North 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.795Z", + "updatedAt": "2023-09-14T05:59:34.795Z" + } + }, + { + "id": 20, + "lat": -27.389035, + "lng": 152.881314, + "name": { + "symbol": "E5", + "description": "East 5", + "label": "East 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.744Z", + "updatedAt": "2023-09-14T05:59:34.744Z" + } + }, + { + "id": 21, + "lat": -27.389208, + "lng": 152.88122, + "name": { + "symbol": "E4", + "description": "East 4", + "label": "East 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.740Z", + "updatedAt": "2023-09-14T05:59:34.740Z" + } + }, + { + "id": 22, + "lat": -27.388974, + "lng": 152.880106, + "name": { + "symbol": "C", + "description": "Centre", + "label": "Centre", + "uri": "", + "createdAt": "2023-09-14T05:59:34.718Z", + "updatedAt": "2023-09-14T05:59:34.718Z" + } + }, + { + "id": 23, + "lat": -27.389343, + "lng": 152.881089, + "name": { + "symbol": "E3", + "description": "East 3", + "label": "East 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.736Z", + "updatedAt": "2023-09-14T05:59:34.736Z" + } + }, + { + "id": 24, + "lat": -27.389472, + "lng": 152.880973, + "name": { + "symbol": "E2", + "description": "East 2", + "label": "East 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.731Z", + "updatedAt": "2023-09-14T05:59:34.731Z" + } + }, + { + "id": 25, + "lat": -27.389553, + "lng": 152.880916, + "name": { + "symbol": "E1", + "description": "East 1", + "label": "East 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.726Z", + "updatedAt": "2023-09-14T05:59:34.726Z" + } + } + ], + "fauna_plot_point": [] + } + } + }, + "photo": [ + { + "id": 14, + "comment": null, + "single_photo": { + "id": 14, + "name": "debbccdd-1234-4a09-a1e5-3cb442788931.png", + "alternativeText": null, + "caption": null, + "width": 284, + "height": 58, + "formats": { + "thumbnail": { + "ext": ".png", + "url": "https://beta-monitor-binary.s3.ap-southeast-2.amazonaws.com/thumbnail_debbccdd_1234_4a09_a1e5_3cb442788931_8979970800.png", + "hash": "thumbnail_debbccdd_1234_4a09_a1e5_3cb442788931_8979970800", + "mime": "image/png", + "name": "thumbnail_debbccdd-1234-4a09-a1e5-3cb442788931.png", + "path": null, + "size": 7.38, + "width": 245, + "height": 50, + "sizeInBytes": 7375 + } + }, + "hash": "debbccdd_1234_4a09_a1e5_3cb442788931_8979970800", + "ext": ".png", + "mime": "image/png", + "size": 1.33, + "url": "https://beta-monitor-binary.s3.ap-southeast-2.amazonaws.com/debbccdd_1234_4a09_a1e5_3cb442788931_8979970800.png", + "previewUrl": null, + "provider": "aws-s3", + "provider_metadata": null, + "folderPath": "/1", + "createdAt": "2024-04-09T01:24:19.489Z", + "updatedAt": "2024-04-09T01:24:19.489Z" + } + } + ] + }, + { + "id": 8, + "field_name": "Anthocerotidae [Subclassis] (scientific: Anthocerotidae Rosenv.)", + "voucher_barcode": "1233445", + "host_species": null, + "unique_id": null, + "date_time": "2024-04-10T01:23:28.089Z", + "createdAt": "2024-04-09T01:24:21.943Z", + "updatedAt": "2024-04-09T01:24:21.943Z", + "growth_form_1": { + "id": 1, + "symbol": "V", + "label": "Sedge", + "description": "Herbaceous, usually perennial erect plant generally with a tufted habit and of the families Cyperaceae (true sedges) or Restionaceae (node sedges).", + "uri": "https://linked.data.gov.au/def/nrm/a7b8b3c8-a329-5a62-8741-e1d6efe9528b", + "createdAt": "2024-03-26T02:39:35.671Z", + "updatedAt": "2024-03-26T02:39:35.671Z" + }, + "growth_form_2": { + "id": 2, + "symbol": "X", + "label": "Grass-tree", + "description": "The 'Growth-form' represents Australian grass-trees. E.g., members of the Xanthorrhoeaceae.", + "uri": "https://linked.data.gov.au/def/nrm/1be47880-4ee6-5df9-8eda-551c58078771", + "createdAt": "2024-03-26T02:39:35.687Z", + "updatedAt": "2024-03-26T02:39:35.687Z" + }, + "habit": [ + { + "id": 5, + "lut": { + "id": 11, + "symbol": "SUC", + "label": "Succulent", + "description": "Fleshy, juicy, soft in texture and usually thickened", + "uri": "", + "createdAt": "2024-03-26T02:39:07.633Z", + "updatedAt": "2024-03-26T02:39:07.633Z" + } + } + ], + "phenology": [ + { + "id": 5, + "lut": { + "id": 9, + "symbol": "SEE", + "label": "Seedling", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:07.771Z", + "updatedAt": "2024-03-26T02:39:07.771Z" + } + } + ], + "floristics_veg_survey_lite": { + "survey_metadata": { + "id": 8, + "org_minted_uuid": "9beeba20-374a-470a-b4f4-caed34e07572", + "createdAt": "2024-03-28T03:17:25.302Z", + "updatedAt": "2024-03-28T03:17:25.302Z", + "survey_details": { + "id": 59, + "survey_model": "floristics-veg-survey-lite", + "time": "2024-03-28T03:17:01.727Z", + "uuid": "263f710d-9af5-456a-9c68-b2b675763220", + "project_id": "0d02b422-5bf7-495f-b9f2-fa0a3046937f", + "protocol_id": "cd2cbbc7-2f17-4b0f-91b4-06f46e9c90f2", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 59, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": "0.1.0-a2ba7a48", + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": "Monitor FDCP API-production", + "system_org": null + } + }, + "start_date_time": "2022-09-21T01:55:44.186Z", + "end_date_time": "2022-09-21T01:55:44.186Z", + "createdAt": "2023-09-14T06:00:17.099Z", + "updatedAt": "2023-09-14T06:00:17.099Z", + "plot_visit": { + "start_date": "2021-08-26T11:26:54.317Z", + "end_date": "2021-08-26T13:26:54.317Z", + "visit_field_name": "QLD winter survey", + "createdAt": "2023-09-14T06:00:12.120Z", + "updatedAt": "2023-09-14T06:00:17.882Z", + "plot_layout": { + "id": 1, + "replicate": 1, + "is_the_plot_permanently_marked": false, + "orientation": 1, + "createdAt": "2023-09-14T06:00:11.473Z", + "updatedAt": "2023-09-14T06:00:11.473Z", + "plot_selection": { + "plot_label": "QDASEQ0001", + "uuid": "a6f84a0e-2239-4bfa-aac6-6d2c99e252f1", + "comments": "some comment", + "createdAt": "2023-09-14T05:59:34.851Z", + "updatedAt": "2023-09-14T05:59:34.851Z", + "plot_name": { + "id": 1, + "unique_digits": "0001", + "state": { + "symbol": "QD", + "label": "Queensland", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/d9d48a48-f74e-55b7-a360-82975c6be412", + "createdAt": "2023-09-14T05:59:33.978Z", + "updatedAt": "2023-09-14T05:59:33.978Z" + }, + "program": { + "symbol": "NLP", + "label": "National Landcare Program", + "description": "", + "uri": "", + "createdAt": "2023-09-14T05:59:34.006Z", + "updatedAt": "2023-09-14T05:59:34.006Z" + }, + "bioregion": { + "symbol": "SEQ", + "label": "South Eastern Queensland", + "description": "", + "uri": "", + "createdAt": "2023-09-14T05:59:34.695Z", + "updatedAt": "2023-09-14T05:59:39.355Z", + "states": [ + { + "symbol": "NS", + "label": "New South Wales", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/14b8c3fe-3a60-5f3a-bb7a-b9cf5df49b4e", + "createdAt": "2023-09-14T05:59:33.966Z", + "updatedAt": "2023-09-14T05:59:33.966Z" + }, + { + "symbol": "QD", + "label": "Queensland", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/d9d48a48-f74e-55b7-a360-82975c6be412", + "createdAt": "2023-09-14T05:59:33.978Z", + "updatedAt": "2023-09-14T05:59:33.978Z" + } + ] + } + }, + "recommended_location": { + "id": 1, + "lat": -35.0004723472134, + "lng": 138.66067886352542 + }, + "recommended_location_point": { + "symbol": "C", + "description": "Centre", + "label": "Centre", + "uri": "", + "createdAt": "2023-09-14T05:59:34.718Z", + "updatedAt": "2023-09-14T05:59:34.718Z" + }, + "plot_selection_survey": null + }, + "plot_type": { + "symbol": "C", + "label": "Control", + "description": "", + "uri": "", + "createdAt": "2023-09-14T05:59:53.672Z", + "updatedAt": "2023-09-14T05:59:53.672Z" + }, + "permanently_marked": null, + "plot_dimensions": { + "symbol": "100m", + "label": "100 x 100", + "description": "100 m by 100 m (1 hectare)", + "uri": "", + "createdAt": "2023-09-14T05:59:53.412Z", + "updatedAt": "2023-09-14T05:59:53.412Z" + }, + "plot_points": [ + { + "id": 2, + "lat": -27.388252, + "lng": 152.880694, + "name": { + "symbol": "W5", + "description": "West 5", + "label": "West 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.787Z", + "updatedAt": "2023-09-14T05:59:34.787Z" + } + }, + { + "id": 1, + "lat": -27.388336, + "lng": 152.880651, + "name": { + "symbol": "W4", + "description": "West 4", + "label": "West 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.783Z", + "updatedAt": "2023-09-14T05:59:34.783Z" + } + }, + { + "id": 3, + "lat": -27.388483, + "lng": 152.880518, + "name": { + "symbol": "W3", + "description": "West 3", + "label": "West 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.779Z", + "updatedAt": "2023-09-14T05:59:34.779Z" + } + }, + { + "id": 4, + "lat": -27.388611, + "lng": 152.880389, + "name": { + "symbol": "W2", + "description": "West 2", + "label": "West 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.775Z", + "updatedAt": "2023-09-14T05:59:34.775Z" + } + }, + { + "id": 5, + "lat": -27.388749, + "lng": 152.88028, + "name": { + "symbol": "W1", + "description": "West 1", + "label": "West 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.771Z", + "updatedAt": "2023-09-14T05:59:34.771Z" + } + }, + { + "id": 6, + "lat": -27.388903, + "lng": 152.880154, + "name": { + "symbol": "SW", + "description": "South West", + "label": "South West", + "uri": "", + "createdAt": "2023-09-14T05:59:34.815Z", + "updatedAt": "2023-09-14T05:59:34.815Z" + } + }, + { + "id": 7, + "lat": -27.389463, + "lng": 152.880835, + "name": { + "symbol": "SE", + "description": "South East", + "label": "South East", + "uri": "", + "createdAt": "2023-09-14T05:59:34.823Z", + "updatedAt": "2023-09-14T05:59:34.823Z" + } + }, + { + "id": 8, + "lat": -27.389366, + "lng": 152.880644, + "name": { + "symbol": "S5", + "description": "South 5", + "label": "South 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.767Z", + "updatedAt": "2023-09-14T05:59:34.767Z" + } + }, + { + "id": 9, + "lat": -27.389248, + "lng": 152.880525, + "name": { + "symbol": "S4", + "description": "South 4", + "label": "South 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.763Z", + "updatedAt": "2023-09-14T05:59:34.763Z" + } + }, + { + "id": 10, + "lat": -27.389158, + "lng": 152.88035, + "name": { + "symbol": "S3", + "description": "South 3", + "label": "South 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.760Z", + "updatedAt": "2023-09-14T05:59:34.760Z" + } + }, + { + "id": 11, + "lat": -27.389021, + "lng": 152.880195, + "name": { + "symbol": "S2", + "description": "South 2", + "label": "South 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.756Z", + "updatedAt": "2023-09-14T05:59:34.756Z" + } + }, + { + "id": 12, + "lat": -27.389373, + "lng": 152.880195, + "name": { + "symbol": "S1", + "description": "South 1", + "label": "South 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.751Z", + "updatedAt": "2023-09-14T05:59:34.751Z" + } + }, + { + "id": 13, + "lat": -27.388316, + "lng": 152.880797, + "name": { + "symbol": "NW", + "description": "North West", + "label": "North West", + "uri": "", + "createdAt": "2023-09-14T05:59:34.827Z", + "updatedAt": "2023-09-14T05:59:34.827Z" + } + }, + { + "id": 14, + "lat": -27.388909, + "lng": 152.881448, + "name": { + "symbol": "NE", + "description": "North East", + "label": "North East", + "uri": "", + "createdAt": "2023-09-14T05:59:34.831Z", + "updatedAt": "2023-09-14T05:59:34.831Z" + } + }, + { + "id": 15, + "lat": -27.388821, + "lng": 152.881503, + "name": { + "symbol": "N5", + "description": "North 5", + "label": "North 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.811Z", + "updatedAt": "2023-09-14T05:59:34.811Z" + } + }, + { + "id": 16, + "lat": -27.388766, + "lng": 152.881422, + "name": { + "symbol": "N4", + "description": "North 4", + "label": "North 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.808Z", + "updatedAt": "2023-09-14T05:59:34.808Z" + } + }, + { + "id": 17, + "lat": -27.388644, + "lng": 152.881263, + "name": { + "symbol": "N3", + "description": "North 3", + "label": "North 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.803Z", + "updatedAt": "2023-09-14T05:59:34.803Z" + } + }, + { + "id": 18, + "lat": -27.388549, + "lng": 152.881107, + "name": { + "symbol": "N2", + "description": "North 2", + "label": "North 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.799Z", + "updatedAt": "2023-09-14T05:59:34.799Z" + } + }, + { + "id": 19, + "lat": -27.388445, + "lng": 152.880939, + "name": { + "symbol": "N1", + "description": "North 1", + "label": "North 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.795Z", + "updatedAt": "2023-09-14T05:59:34.795Z" + } + }, + { + "id": 20, + "lat": -27.389035, + "lng": 152.881314, + "name": { + "symbol": "E5", + "description": "East 5", + "label": "East 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.744Z", + "updatedAt": "2023-09-14T05:59:34.744Z" + } + }, + { + "id": 21, + "lat": -27.389208, + "lng": 152.88122, + "name": { + "symbol": "E4", + "description": "East 4", + "label": "East 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.740Z", + "updatedAt": "2023-09-14T05:59:34.740Z" + } + }, + { + "id": 22, + "lat": -27.388974, + "lng": 152.880106, + "name": { + "symbol": "C", + "description": "Centre", + "label": "Centre", + "uri": "", + "createdAt": "2023-09-14T05:59:34.718Z", + "updatedAt": "2023-09-14T05:59:34.718Z" + } + }, + { + "id": 23, + "lat": -27.389343, + "lng": 152.881089, + "name": { + "symbol": "E3", + "description": "East 3", + "label": "East 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.736Z", + "updatedAt": "2023-09-14T05:59:34.736Z" + } + }, + { + "id": 24, + "lat": -27.389472, + "lng": 152.880973, + "name": { + "symbol": "E2", + "description": "East 2", + "label": "East 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.731Z", + "updatedAt": "2023-09-14T05:59:34.731Z" + } + }, + { + "id": 25, + "lat": -27.389553, + "lng": 152.880916, + "name": { + "symbol": "E1", + "description": "East 1", + "label": "East 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.726Z", + "updatedAt": "2023-09-14T05:59:34.726Z" + } + } + ], + "fauna_plot_point": [] + } + } + }, + "photo": [ + { + "id": 14, + "comment": null, + "single_photo": { + "id": 14, + "name": "debbccdd-1234-4a09-a1e5-3cb442788931.png", + "alternativeText": null, + "caption": null, + "width": 284, + "height": 58, + "formats": { + "thumbnail": { + "ext": ".png", + "url": "https://beta-monitor-binary.s3.ap-southeast-2.amazonaws.com/thumbnail_debbccdd_1234_4a09_a1e5_3cb442788931_8979970800.png", + "hash": "thumbnail_debbccdd_1234_4a09_a1e5_3cb442788931_8979970800", + "mime": "image/png", + "name": "thumbnail_debbccdd-1234-4a09-a1e5-3cb442788931.png", + "path": null, + "size": 7.38, + "width": 245, + "height": 50, + "sizeInBytes": 7375 + } + }, + "hash": "debbccdd_1234_4a09_a1e5_3cb442788931_8979970800", + "ext": ".png", + "mime": "image/png", + "size": 1.33, + "url": "https://beta-monitor-binary.s3.ap-southeast-2.amazonaws.com/debbccdd_1234_4a09_a1e5_3cb442788931_8979970800.png", + "previewUrl": null, + "provider": "aws-s3", + "provider_metadata": null, + "folderPath": "/1", + "createdAt": "2024-04-09T01:24:19.489Z", + "updatedAt": "2024-04-09T01:24:19.489Z" + } + } + ] + },{ + "id": 8, + "field_name": "Anthocerotidae [Subclassis] (scientific: Anthocerotidae Rosenv.)", + "voucher_barcode": "1233445", + "host_species": null, + "unique_id": null, + "date_time": "2024-04-08T01:23:28.089Z", + "createdAt": "2024-04-09T01:24:21.943Z", + "updatedAt": "2024-04-09T01:24:21.943Z", + "growth_form_1": { + "id": 1, + "symbol": "V", + "label": "Sedge", + "description": "Herbaceous, usually perennial erect plant generally with a tufted habit and of the families Cyperaceae (true sedges) or Restionaceae (node sedges).", + "uri": "https://linked.data.gov.au/def/nrm/a7b8b3c8-a329-5a62-8741-e1d6efe9528b", + "createdAt": "2024-03-26T02:39:35.671Z", + "updatedAt": "2024-03-26T02:39:35.671Z" + }, + "growth_form_2": { + "id": 2, + "symbol": "X", + "label": "Grass-tree", + "description": "The 'Growth-form' represents Australian grass-trees. E.g., members of the Xanthorrhoeaceae.", + "uri": "https://linked.data.gov.au/def/nrm/1be47880-4ee6-5df9-8eda-551c58078771", + "createdAt": "2024-03-26T02:39:35.687Z", + "updatedAt": "2024-03-26T02:39:35.687Z" + }, + "habit": [ + { + "id": 5, + "lut": { + "id": 11, + "symbol": "SUC", + "label": "Succulent", + "description": "Fleshy, juicy, soft in texture and usually thickened", + "uri": "", + "createdAt": "2024-03-26T02:39:07.633Z", + "updatedAt": "2024-03-26T02:39:07.633Z" + } + } + ], + "phenology": [ + { + "id": 5, + "lut": { + "id": 9, + "symbol": "SEE", + "label": "Seedling", + "description": "", + "uri": "", + "createdAt": "2024-03-26T02:39:07.771Z", + "updatedAt": "2024-03-26T02:39:07.771Z" + } + } + ], + "floristics_veg_survey_lite": { + "survey_metadata": { + "id": 8, + "org_minted_uuid": "9beeba20-374a-470a-b4f4-caed34e07572", + "createdAt": "2024-03-28T03:17:25.302Z", + "updatedAt": "2024-03-28T03:17:25.302Z", + "survey_details": { + "id": 59, + "survey_model": "floristics-veg-survey-lite", + "time": "2024-03-28T03:17:01.727Z", + "uuid": "263f710d-9af5-456a-9c68-b2b675763220", + "project_id": "0d02b422-5bf7-495f-b9f2-fa0a3046937f", + "protocol_id": "cd2cbbc7-2f17-4b0f-91b4-06f46e9c90f2", + "protocol_version": "1", + "submodule_protocol_id": "" + }, + "provenance": { + "id": 59, + "version_app": "1.0.0-alpha.7-a2ba7a48", + "version_core": "0.1.0-a2ba7a48", + "version_core_documentation": "0.1.0-a2ba7a48", + "version_org": null, + "system_app": "Atlas of Living Australia-beta-testing-0.0.0.0-", + "system_core": "Monitor FDCP API-production", + "system_org": null + } + }, + "start_date_time": "2022-09-21T01:55:44.186Z", + "end_date_time": "2022-09-21T01:55:44.186Z", + "createdAt": "2023-09-14T06:00:17.099Z", + "updatedAt": "2023-09-14T06:00:17.099Z", + "plot_visit": { + "start_date": "2021-08-26T11:26:54.317Z", + "end_date": "2021-08-26T13:26:54.317Z", + "visit_field_name": "QLD winter survey", + "createdAt": "2023-09-14T06:00:12.120Z", + "updatedAt": "2023-09-14T06:00:17.882Z", + "plot_layout": { + "id": 1, + "replicate": 1, + "is_the_plot_permanently_marked": false, + "orientation": 1, + "createdAt": "2023-09-14T06:00:11.473Z", + "updatedAt": "2023-09-14T06:00:11.473Z", + "plot_selection": { + "plot_label": "QDASEQ0001", + "uuid": "a6f84a0e-2239-4bfa-aac6-6d2c99e252f1", + "comments": "some comment", + "createdAt": "2023-09-14T05:59:34.851Z", + "updatedAt": "2023-09-14T05:59:34.851Z", + "plot_name": { + "id": 1, + "unique_digits": "0001", + "state": { + "symbol": "QD", + "label": "Queensland", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/d9d48a48-f74e-55b7-a360-82975c6be412", + "createdAt": "2023-09-14T05:59:33.978Z", + "updatedAt": "2023-09-14T05:59:33.978Z" + }, + "program": { + "symbol": "NLP", + "label": "National Landcare Program", + "description": "", + "uri": "", + "createdAt": "2023-09-14T05:59:34.006Z", + "updatedAt": "2023-09-14T05:59:34.006Z" + }, + "bioregion": { + "symbol": "SEQ", + "label": "South Eastern Queensland", + "description": "", + "uri": "", + "createdAt": "2023-09-14T05:59:34.695Z", + "updatedAt": "2023-09-14T05:59:39.355Z", + "states": [ + { + "symbol": "NS", + "label": "New South Wales", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/14b8c3fe-3a60-5f3a-bb7a-b9cf5df49b4e", + "createdAt": "2023-09-14T05:59:33.966Z", + "updatedAt": "2023-09-14T05:59:33.966Z" + }, + { + "symbol": "QD", + "label": "Queensland", + "description": "", + "uri": "https://linked.data.gov.au/def/nrm/d9d48a48-f74e-55b7-a360-82975c6be412", + "createdAt": "2023-09-14T05:59:33.978Z", + "updatedAt": "2023-09-14T05:59:33.978Z" + } + ] + } + }, + "recommended_location": { + "id": 1, + "lat": -35.0004723472134, + "lng": 138.66067886352542 + }, + "recommended_location_point": { + "symbol": "C", + "description": "Centre", + "label": "Centre", + "uri": "", + "createdAt": "2023-09-14T05:59:34.718Z", + "updatedAt": "2023-09-14T05:59:34.718Z" + }, + "plot_selection_survey": null + }, + "plot_type": { + "symbol": "C", + "label": "Control", + "description": "", + "uri": "", + "createdAt": "2023-09-14T05:59:53.672Z", + "updatedAt": "2023-09-14T05:59:53.672Z" + }, + "permanently_marked": null, + "plot_dimensions": { + "symbol": "100m", + "label": "100 x 100", + "description": "100 m by 100 m (1 hectare)", + "uri": "", + "createdAt": "2023-09-14T05:59:53.412Z", + "updatedAt": "2023-09-14T05:59:53.412Z" + }, + "plot_points": [ + { + "id": 2, + "lat": -27.388252, + "lng": 152.880694, + "name": { + "symbol": "W5", + "description": "West 5", + "label": "West 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.787Z", + "updatedAt": "2023-09-14T05:59:34.787Z" + } + }, + { + "id": 1, + "lat": -27.388336, + "lng": 152.880651, + "name": { + "symbol": "W4", + "description": "West 4", + "label": "West 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.783Z", + "updatedAt": "2023-09-14T05:59:34.783Z" + } + }, + { + "id": 3, + "lat": -27.388483, + "lng": 152.880518, + "name": { + "symbol": "W3", + "description": "West 3", + "label": "West 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.779Z", + "updatedAt": "2023-09-14T05:59:34.779Z" + } + }, + { + "id": 4, + "lat": -27.388611, + "lng": 152.880389, + "name": { + "symbol": "W2", + "description": "West 2", + "label": "West 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.775Z", + "updatedAt": "2023-09-14T05:59:34.775Z" + } + }, + { + "id": 5, + "lat": -27.388749, + "lng": 152.88028, + "name": { + "symbol": "W1", + "description": "West 1", + "label": "West 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.771Z", + "updatedAt": "2023-09-14T05:59:34.771Z" + } + }, + { + "id": 6, + "lat": -27.388903, + "lng": 152.880154, + "name": { + "symbol": "SW", + "description": "South West", + "label": "South West", + "uri": "", + "createdAt": "2023-09-14T05:59:34.815Z", + "updatedAt": "2023-09-14T05:59:34.815Z" + } + }, + { + "id": 7, + "lat": -27.389463, + "lng": 152.880835, + "name": { + "symbol": "SE", + "description": "South East", + "label": "South East", + "uri": "", + "createdAt": "2023-09-14T05:59:34.823Z", + "updatedAt": "2023-09-14T05:59:34.823Z" + } + }, + { + "id": 8, + "lat": -27.389366, + "lng": 152.880644, + "name": { + "symbol": "S5", + "description": "South 5", + "label": "South 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.767Z", + "updatedAt": "2023-09-14T05:59:34.767Z" + } + }, + { + "id": 9, + "lat": -27.389248, + "lng": 152.880525, + "name": { + "symbol": "S4", + "description": "South 4", + "label": "South 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.763Z", + "updatedAt": "2023-09-14T05:59:34.763Z" + } + }, + { + "id": 10, + "lat": -27.389158, + "lng": 152.88035, + "name": { + "symbol": "S3", + "description": "South 3", + "label": "South 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.760Z", + "updatedAt": "2023-09-14T05:59:34.760Z" + } + }, + { + "id": 11, + "lat": -27.389021, + "lng": 152.880195, + "name": { + "symbol": "S2", + "description": "South 2", + "label": "South 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.756Z", + "updatedAt": "2023-09-14T05:59:34.756Z" + } + }, + { + "id": 12, + "lat": -27.389373, + "lng": 152.880195, + "name": { + "symbol": "S1", + "description": "South 1", + "label": "South 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.751Z", + "updatedAt": "2023-09-14T05:59:34.751Z" + } + }, + { + "id": 13, + "lat": -27.388316, + "lng": 152.880797, + "name": { + "symbol": "NW", + "description": "North West", + "label": "North West", + "uri": "", + "createdAt": "2023-09-14T05:59:34.827Z", + "updatedAt": "2023-09-14T05:59:34.827Z" + } + }, + { + "id": 14, + "lat": -27.388909, + "lng": 152.881448, + "name": { + "symbol": "NE", + "description": "North East", + "label": "North East", + "uri": "", + "createdAt": "2023-09-14T05:59:34.831Z", + "updatedAt": "2023-09-14T05:59:34.831Z" + } + }, + { + "id": 15, + "lat": -27.388821, + "lng": 152.881503, + "name": { + "symbol": "N5", + "description": "North 5", + "label": "North 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.811Z", + "updatedAt": "2023-09-14T05:59:34.811Z" + } + }, + { + "id": 16, + "lat": -27.388766, + "lng": 152.881422, + "name": { + "symbol": "N4", + "description": "North 4", + "label": "North 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.808Z", + "updatedAt": "2023-09-14T05:59:34.808Z" + } + }, + { + "id": 17, + "lat": -27.388644, + "lng": 152.881263, + "name": { + "symbol": "N3", + "description": "North 3", + "label": "North 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.803Z", + "updatedAt": "2023-09-14T05:59:34.803Z" + } + }, + { + "id": 18, + "lat": -27.388549, + "lng": 152.881107, + "name": { + "symbol": "N2", + "description": "North 2", + "label": "North 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.799Z", + "updatedAt": "2023-09-14T05:59:34.799Z" + } + }, + { + "id": 19, + "lat": -27.388445, + "lng": 152.880939, + "name": { + "symbol": "N1", + "description": "North 1", + "label": "North 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.795Z", + "updatedAt": "2023-09-14T05:59:34.795Z" + } + }, + { + "id": 20, + "lat": -27.389035, + "lng": 152.881314, + "name": { + "symbol": "E5", + "description": "East 5", + "label": "East 5", + "uri": "", + "createdAt": "2023-09-14T05:59:34.744Z", + "updatedAt": "2023-09-14T05:59:34.744Z" + } + }, + { + "id": 21, + "lat": -27.389208, + "lng": 152.88122, + "name": { + "symbol": "E4", + "description": "East 4", + "label": "East 4", + "uri": "", + "createdAt": "2023-09-14T05:59:34.740Z", + "updatedAt": "2023-09-14T05:59:34.740Z" + } + }, + { + "id": 22, + "lat": -27.388974, + "lng": 152.880106, + "name": { + "symbol": "C", + "description": "Centre", + "label": "Centre", + "uri": "", + "createdAt": "2023-09-14T05:59:34.718Z", + "updatedAt": "2023-09-14T05:59:34.718Z" + } + }, + { + "id": 23, + "lat": -27.389343, + "lng": 152.881089, + "name": { + "symbol": "E3", + "description": "East 3", + "label": "East 3", + "uri": "", + "createdAt": "2023-09-14T05:59:34.736Z", + "updatedAt": "2023-09-14T05:59:34.736Z" + } + }, + { + "id": 24, + "lat": -27.389472, + "lng": 152.880973, + "name": { + "symbol": "E2", + "description": "East 2", + "label": "East 2", + "uri": "", + "createdAt": "2023-09-14T05:59:34.731Z", + "updatedAt": "2023-09-14T05:59:34.731Z" + } + }, + { + "id": 25, + "lat": -27.389553, + "lng": 152.880916, + "name": { + "symbol": "E1", + "description": "East 1", + "label": "East 1", + "uri": "", + "createdAt": "2023-09-14T05:59:34.726Z", + "updatedAt": "2023-09-14T05:59:34.726Z" + } + } + ], + "fauna_plot_point": [] + } + } + }, + "photo": [ + { + "id": 14, + "comment": null, + "single_photo": { + "id": 14, + "name": "debbccdd-1234-4a09-a1e5-3cb442788931.png", + "alternativeText": null, + "caption": null, + "width": 284, + "height": 58, + "formats": { + "thumbnail": { + "ext": ".png", + "url": "https://beta-monitor-binary.s3.ap-southeast-2.amazonaws.com/thumbnail_debbccdd_1234_4a09_a1e5_3cb442788931_8979970800.png", + "hash": "thumbnail_debbccdd_1234_4a09_a1e5_3cb442788931_8979970800", + "mime": "image/png", + "name": "thumbnail_debbccdd-1234-4a09-a1e5-3cb442788931.png", + "path": null, + "size": 7.38, + "width": 245, + "height": 50, + "sizeInBytes": 7375 + } + }, + "hash": "debbccdd_1234_4a09_a1e5_3cb442788931_8979970800", + "ext": ".png", + "mime": "image/png", + "size": 1.33, + "url": "https://beta-monitor-binary.s3.ap-southeast-2.amazonaws.com/debbccdd_1234_4a09_a1e5_3cb442788931_8979970800.png", + "previewUrl": null, + "provider": "aws-s3", + "provider_metadata": null, + "folderPath": "/1", + "createdAt": "2024-04-09T01:24:19.489Z", + "updatedAt": "2024-04-09T01:24:19.489Z" + } + } + ] + } + ], + "floristics-veg-virtual-voucher": [] } } \ No newline at end of file From 2f253c59bbbc8551ebd1a3aca7fdbaf763ad592c Mon Sep 17 00:00:00 2001 From: temi Date: Wed, 24 Apr 2024 07:15:04 +1000 Subject: [PATCH 49/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixes null pointer exception --- .../au/org/ala/ecodata/converter/RecordConverter.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy index be1764602..c0482bd08 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/RecordConverter.groovy @@ -151,8 +151,8 @@ class RecordConverter { public static Map overrideFieldValues(Map source, Map additional) { Map result = [:] - result << source - result << additional + result << (source ?: [:]) + result << (additional ?: [:]) result } From ab022cced4b44eeeff197425e4c4dfbf3f2ea0cb Mon Sep 17 00:00:00 2001 From: chrisala Date: Wed, 24 Apr 2024 16:03:27 +1000 Subject: [PATCH 50/52] Added time to dataset name #924 --- .../services/au/org/ala/ecodata/ParatooService.groovy | 2 +- src/main/groovy/au/org/ala/ecodata/DateUtil.groovy | 6 ++++++ .../groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 699c72c5f..0491b6ac1 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -755,7 +755,7 @@ class ParatooService { dataSet.progress = Activity.PLANNED String dataSetName = buildName( paratooCollectionId.protocolId, - DateUtil.formatAsDisplayDate(paratooCollectionId.eventTime), project) + DateUtil.formatAsDisplayDateTime(paratooCollectionId.eventTime), project) dataSet.name = dataSetName dataSet diff --git a/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy b/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy index 97fa67bc4..8c5831423 100644 --- a/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy +++ b/src/main/groovy/au/org/ala/ecodata/DateUtil.groovy @@ -23,6 +23,7 @@ class DateUtil { static DateTimeFormatter ISO_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'") static DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd") + static DateTimeFormatter DISPLAY_DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd h:mm a") static Date parse(String dateStr) { SimpleDateFormat format = new SimpleDateFormat(dateFormat) return format.parse(dateStr.replace("Z", "+0000")) @@ -48,6 +49,11 @@ class DateUtil { dateTime.format(DATE_FORMATTER) } + static String formatAsDisplayDateTime(Date date) { + ZonedDateTime dateTime = ZonedDateTime.ofInstant(date.toInstant(), ZoneId.systemDefault()) + dateTime.format(DISPLAY_DATE_TIME_FORMATTER) + } + /** * Returns a formatted string representing the financial year a report or activity falls into, based on * the end date. This method won't necessarily work for start dates as it will subtract a day from the value diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index f17761fd9..a2911ce00 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -156,7 +156,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest Date: Fri, 26 Apr 2024 06:35:26 +1000 Subject: [PATCH 51/52] AtlasOfLivingAustralia/fieldcapture#3049 - delete activity when re-synced - no species created if species data is null - update tests --- .../au/org/ala/ecodata/ParatooService.groovy | 26 ++++++++++--------- .../ecodata/converter/ListConverter.groovy | 20 +++++++------- .../org/ala/ecodata/ParatooServiceSpec.groovy | 3 ++- .../ecodata/SpeciesReMatchServiceSpec.groovy | 2 +- 4 files changed, 28 insertions(+), 23 deletions(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 0491b6ac1..9de90a127 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -240,7 +240,6 @@ class ParatooService { Map asyncFetchCollection(ParatooCollection collection, Map authHeader, String userId, ParatooProject project) { Activity.withSession { session -> int counter = 0 - boolean forceActivityCreation = grailsApplication.config.getProperty('paratoo.forceActivityCreation', Boolean, false) Map surveyDataAndObservations = null Map response = null Map dataSet = project.project.custom?.dataSets?.find{it.dataSetId == collection.orgMintedUUID} @@ -281,20 +280,23 @@ class ParatooService { dataSet.siteId = createSiteFromSurveyData(surveyDataAndObservations, collection, surveyId, project.project, config, form) } - // make sure activity has not been created for this data set - if (!dataSet.activityId || forceActivityCreation) { - // delete previously created activity if forceActivityCreation is true? + // plot layout is of type geoMap. Therefore, expects a site id. + if (surveyDataAndObservations.containsKey(PARATOO_DATAMODEL_PLOT_LAYOUT) && dataSet.siteId) { + surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_LAYOUT] = dataSet.siteId + } - // plot layout is of type geoMap. Therefore, expects a site id. - if (surveyDataAndObservations.containsKey(PARATOO_DATAMODEL_PLOT_LAYOUT) && dataSet.siteId) { - surveyDataAndObservations[PARATOO_DATAMODEL_PLOT_LAYOUT] = dataSet.siteId - } - String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet.siteId, userId) - List records = recordService.getAllByActivity(activityId) - dataSet.areSpeciesRecorded = records?.size() > 0 - dataSet.activityId = activityId + // Delete previously created activity so that duplicate species records are not created. + // Updating existing activity will also create duplicates since it relies on outputSpeciesId to determine + // if a record is new and new ones are created by code. + if (dataSet.activityId) { + activityService.delete(dataSet.activityId, true) } + String activityId = createActivityFromSurveyData(form, surveyDataAndObservations, surveyId, dataSet.siteId, userId) + List records = recordService.getAllByActivity(activityId) + dataSet.areSpeciesRecorded = records?.size() > 0 + dataSet.activityId = activityId + dataSet.startDate = config.getStartDate(surveyDataAndObservations) dataSet.endDate = config.getEndDate(surveyDataAndObservations) dataSet.format = DATASET_DATABASE_TABLE diff --git a/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy b/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy index daedd0ec0..024d3a97d 100644 --- a/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy +++ b/src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy @@ -54,15 +54,17 @@ class ListConverter implements RecordFieldConverter { speciesModels?.each { Map dataModel -> RecordFieldConverter converter = RecordConverter.getFieldConverter(dataModel.dataType) List recordFieldSets = converter.convert(row, dataModel) - Map speciesRecord = RecordConverter.overrideFieldValues(baseRecord, recordFieldSets[0]) - - // We want to create a record in the DB only if species information is present - if(speciesRecord.outputSpeciesId) { - speciesRecord.outputItemId = index++ - records << speciesRecord - } else { - log.warn("Record [${speciesRecord}] does not contain full species information. " + - "This is most likely a bug.") + if (recordFieldSets) { + Map speciesRecord = RecordConverter.overrideFieldValues(baseRecord, recordFieldSets[0]) + + // We want to create a record in the DB only if species information is present + if (speciesRecord.outputSpeciesId) { + speciesRecord.outputItemId = index++ + records << speciesRecord + } else { + log.warn("Record [${speciesRecord}] does not contain full species information. " + + "This is most likely a bug.") + } } } diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index a2911ce00..f097243e0 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -179,7 +179,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest> [status: 'ok'] 1 * projectService.update([custom: [dataSets: [expectedDataSetSync]]], 'p1', false) >> [status: 'ok'] 1 * activityService.create(_) >> [activityId: '123'] + 1 * activityService.delete("123", true) >> [status: 'ok'] 1 * recordService.getAllByActivity('123') >> [] 1 * settingService.getSetting('paratoo.surveyData.mapping') >> { (["guid-2": [ diff --git a/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy index 70bcb4af8..c848689b5 100644 --- a/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/SpeciesReMatchServiceSpec.groovy @@ -10,7 +10,7 @@ class SpeciesReMatchServiceSpec extends Specification implements ServiceUnitTest void setup() { service.cacheService = new CacheService() service.webService = Mock(WebService) - grailsApplication.config.bie.url = "http://localhost:8080" + grailsApplication.config.bie.url = "http://localhost:8080/" } void "test searchBie"() { From 12a166512d3aed74fb58333ee2c8272f1dc0b5fc Mon Sep 17 00:00:00 2001 From: temi Date: Fri, 26 Apr 2024 11:30:07 +1000 Subject: [PATCH 52/52] AtlasOfLivingAustralia/fieldcapture#3049 - fixes species extraction from lut fields --- .../au/org/ala/ecodata/ParatooService.groovy | 7 ++- .../org/ala/ecodata/ParatooServiceSpec.groovy | 61 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy index 9de90a127..c0d745bf9 100644 --- a/grails-app/services/au/org/ala/ecodata/ParatooService.groovy +++ b/grails-app/services/au/org/ala/ecodata/ParatooService.groovy @@ -491,7 +491,12 @@ class ParatooService { case "species": String speciesName try { - speciesName = getProperty(output, model.name)?.first() + if(model.containsKey(PARATOO_LUT_REF)) { + speciesName = getProperty(output, model.name)?.label?.first() + } else { + speciesName = getProperty(output, model.name)?.first() + } + output[model.name] = transformSpeciesName(speciesName) } catch (Exception e) { log.info("Error getting species name for ${model.name}: ${e.message}") diff --git a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy index f097243e0..a57dc2dfe 100644 --- a/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy @@ -1201,6 +1201,67 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest