Skip to content

Commit

Permalink
AtlasOfLivingAustralia/fieldcapture#3049
Browse files Browse the repository at this point in the history 
- async fetch from monitor
- refactored code
  • Loading branch information
@temi
temi committed Mar 28, 2024
1 parent 78a4408 commit cb01b50
Show file tree
Hide file tree
Showing 10 changed files with 202 additions and 218 deletions.
4 changes: 2 additions & 2 deletions grails-app/controllers/au/org/ala/ecodata/ParatooController.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,10 +289,10 @@ class ParatooController {
boolean hasProtocol = paratooService.protocolWriteCheck(userId, dataSet.project.id, collectionId.protocolId)
if (hasProtocol) {
Map result = paratooService.submitCollection(collection, dataSet.project)
if (!result.error) {
if (!result.updateResult.error) {
respond([success: true])
} else {
error(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.error)
error(HttpStatus.SC_INTERNAL_SERVER_ERROR, result.updateResult.error)
}
} else {
error(HttpStatus.SC_FORBIDDEN, "Project / protocol combination not available")
Expand Down
212 changes: 144 additions & 68 deletions grails-app/services/au/org/ala/ecodata/ParatooService.groovy
View file

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions src/main/groovy/au/org/ala/ecodata/converter/GenericFieldConverter.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class GenericFieldConverter implements RecordFieldConverter {

record << getDwcAttributes(data, dwcMappings)

if (data.dwcAttribute) {
if (data?.dwcAttribute) {
record[data.dwcAttribute] = data.value
}

Expand All @@ -34,9 +34,9 @@ class GenericFieldConverter implements RecordFieldConverter {
private Double getLatitude(Map data) {
Double latitude = null

if (data.decimalLatitude) {
if (data?.decimalLatitude) {
latitude = toDouble(data.decimalLatitude)
} else if (data.locationLatitude) {
} else if (data?.locationLatitude) {
latitude = toDouble(data.locationLatitude)
}

Expand All @@ -46,9 +46,9 @@ class GenericFieldConverter implements RecordFieldConverter {
private Double getLongitude(Map data) {
Double longitude = null

if (data.decimalLongitude) {
if (data?.decimalLongitude) {
longitude = toDouble(data.decimalLongitude)
} else if (data.locationLongitude) {
} else if (data?.locationLongitude) {
longitude = toDouble(data.locationLongitude)
}

Expand Down
4 changes: 4 additions & 0 deletions src/main/groovy/au/org/ala/ecodata/converter/ListConverter.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ class ListConverter implements RecordFieldConverter {

// delegate the conversion of each column in each row to a specific converter for the column type
data[outputMetadata.name].each { row ->
if (row == null) {
return
}

Map baseRecord = [:]
List singleItemModels
List multiItemModels
Expand Down
4 changes: 4 additions & 0 deletions src/main/groovy/au/org/ala/ecodata/converter/SpeciesConverter.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ class SpeciesConverter implements RecordFieldConverter {
]

List<Map> convert(Map data, Map metadata = [:]) {
if (data == null) {
return []
}

Map record = [:]

record.scientificNameID = record.guid = data[metadata.name].guid
Expand Down
1 change: 0 additions & 1 deletion src/main/groovy/au/org/ala/ecodata/metadata/OutputMetadata.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,6 @@ class OutputMetadata {
context?.each { data ->
if(isNestedDataModelType(data)){
String contextPath = "${path}.${data.name}"
log.info("${contextPath}")
// recursive call for nested data model
childrenNames = getNamesForDataType(type, getNestedDataModelNodes(data), depth + 1, contextPath);
if(childrenNames?.size()){
Expand Down
124 changes: 19 additions & 105 deletions src/main/groovy/au/org/ala/ecodata/paratoo/ParatooProtocolConfig.groovy
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,11 @@
package au.org.ala.ecodata.paratoo

import au.org.ala.ecodata.ActivityForm
import au.org.ala.ecodata.DateUtil
import au.org.ala.ecodata.FormSection
import au.org.ala.ecodata.GeometryUtils
import au.org.ala.ecodata.ParatooService
import au.org.ala.ecodata.*
import au.org.ala.ecodata.metadata.OutputMetadata
import au.org.ala.ecodata.metadata.PropertyAccessor
import com.fasterxml.jackson.annotation.JsonIgnoreProperties
import groovy.util.logging.Slf4j
import org.locationtech.jts.geom.Geometry

import java.util.regex.Matcher
/**
* Configuration about how to work with a Paratoo/Monitor protocol
*/
Expand All @@ -21,8 +15,6 @@ class ParatooProtocolConfig {

String name
String apiEndpoint
String observationEndpoint
String surveyType
boolean usesPlotLayout = true
List tags
String geometryType = 'Polygon'
Expand All @@ -31,12 +23,6 @@ class ParatooProtocolConfig {
String startDatePath = 'attributes.start_date_time'
String endDatePath = 'attributes.end_date_time'
String surveyIdPath = 'attributes.survey_metadata'
String observationSurveyIdPath = ''
String observationSpeciesPath = 'attributes.species'
String observationRecordedByPath = 'attributes.observers.observer'
String observationIndividualCountPath = 'attributes.number_of_individuals'
String observationEventDatePath = 'attributes.date_time'
String observationGeometryPath = 'attributes.location'
String plotVisitPath = 'attributes.plot_visit.data.attributes'
String plotLayoutPath = "${plotVisitPath}.plot_layout.data.attributes"
String plotLayoutIdPath = 'attributes.plot_visit.data.attributes.plot_layout.data.id'
Expand All @@ -57,84 +43,36 @@ class ParatooProtocolConfig {
}

String getStartDate(Map surveyData) {
removeMilliseconds(getProperty(surveyData, startDatePath))
}

String getEndDate(Map surveyData) {
removeMilliseconds(getProperty(surveyData, endDatePath))
}

Map getSurveyId(Map surveyData) {
getProperty(surveyData, surveyIdPath)
}

String getSpecies (Map observation) {
getProperty(observation, observationSpeciesPath)
}

Map parseSpecies (String species) {
extractSpeciesName(species)
}

List findObservationsBelongingToSurvey (List surveyData, ParatooCollectionId surveyId) {
surveyData?.findAll { Map data ->
Map dataLinkedToSurvey = getSurveyIdOfObservation( data )
surveyEqualityTest(dataLinkedToSurvey, surveyId)
String date = getProperty(surveyData, startDatePath)
if (date == null) {
date = getPropertyFromSurvey(surveyData, startDatePath)
}
}

Map getSurveyIdOfObservation (Map data) {
getProperty(data, observationSurveyIdPath)
removeMilliseconds(date)
}

Map getEventDate (Map data) {
getProperty(data, observationEventDatePath)
def getPropertyFromSurvey(Map surveyData, String path) {
surveyData = getSurveyDataFromObservation(surveyData)
path = path.replaceFirst("^attributes.", '')
getProperty(surveyData, path)
}

Double getDecimalLatitude (Map data) {
def shape = extractObservationSiteDataFromPath(data)
switch (shape.type) {
case 'Point':
return shape.coordinates[1]
}
}

Double getDecimalLongitude (Map data) {
def shape = extractObservationSiteDataFromPath(data)
switch (shape.type) {
case 'Point':
return shape.coordinates[0]
String getEndDate(Map surveyData) {
String date = getProperty(surveyData, endDatePath)
if (date == null) {
date = getPropertyFromSurvey(surveyData, endDatePath)
}

removeMilliseconds(date)
}

Integer getIndividualCount (Map data) {
getProperty(data, observationIndividualCountPath)
}

/**
* todo: this is a temporary hack to get around the fact that the survey doesn't have a field for this and it has to
* be interpreted from the protocol
* @param data
* @return
*/
String getIndividualOrGroup (Map data) {
'Individuals'
}

String getRecordedBy (Map data) {
def result = getProperty(data, observationRecordedByPath)
if (result instanceof List) {
return result.join(', ')
}
else if (result instanceof String) {
return result
Map getSurveyId(Map surveyData) {
Map result = getProperty(surveyData, surveyIdPath)
if (result == null) {
result = getPropertyFromSurvey(surveyData, surveyIdPath)
}
}

private Map extractObservationSiteDataFromPath(Map surveyData) {
def geometryData = getProperty(surveyData, observationGeometryPath)
extractGeometryFromSiteData(geometryData)
result
}

private Map extractSiteDataFromPath(Map surveyData) {
Expand Down Expand Up @@ -376,28 +314,4 @@ class ParatooProtocolConfig {
]
geoJson
}

static Map extractSpeciesName (String species) {
Map result
Matcher m = (species =~ /([^\[]*)\[([^\]]*)\]\s*\(scientific: ([^\)]*)\)/)
if (m.matches()) {
def scientificName = m.group(3).trim()
def commonName = m.group(1)?.trim()
def name = "$scientificName ($commonName)"
result = [
vernacularName: commonName,
scientificName: scientificName,
name: name
]
}
else {
result = [
vernacularName: species,
scientificName: species,
name: species
]
}

result
}
}
4 changes: 2 additions & 2 deletions src/test/groovy/au/org/ala/ecodata/ParatooControllerSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ class ParatooControllerSpec extends Specification implements ControllerUnitTest<
1 * userService.currentUserDetails >> [userId:userId]
1 * paratooService.findDataSet(userId, collection.orgMintedUUID) >> searchResults
1 * paratooService.protocolWriteCheck(userId, 'p1', "guid-1") >> true
1 * paratooService.submitCollection({it.orgMintedUUID == "c1"}, searchResults.project) >> [:]
1 * paratooService.submitCollection({it.orgMintedUUID == "c1"}, searchResults.project) >> [updateResult: [:], promise: null]

and:
response.status == HttpStatus.SC_OK
Expand All @@ -230,7 +230,7 @@ class ParatooControllerSpec extends Specification implements ControllerUnitTest<
1 * userService.currentUserDetails >> [userId:userId]
1 * paratooService.findDataSet(userId, collection.orgMintedUUID) >> searchResults
1 * paratooService.protocolWriteCheck(userId, 'p1', "guid-1") >> true
1 * paratooService.submitCollection({it.orgMintedUUID == "c1"}, searchResults.project) >> [error:"Error"]
1 * paratooService.submitCollection({it.orgMintedUUID == "c1"}, searchResults.project) >> [updateResult: [error:"Error"], promise: null]

and:
response.status == HttpStatus.SC_INTERNAL_SERVER_ERROR
Expand Down
36 changes: 18 additions & 18 deletions src/test/groovy/au/org/ala/ecodata/ParatooServiceSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import groovy.json.JsonSlurper
import org.codehaus.jackson.map.ObjectMapper
import org.grails.web.converters.marshaller.json.CollectionMarshaller
import org.grails.web.converters.marshaller.json.MapMarshaller
import static grails.async.Promises.waitAll

/**
* Tests for the ParatooService.
Expand Down Expand Up @@ -47,6 +48,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
service.recordService = recordService
service.cacheService = new CacheService()
service.userService = userService
service.elasticSearchService = new ElasticSearchService()

JSON.registerObjectMarshaller(new MapMarshaller())
JSON.registerObjectMarshaller(new CollectionMarshaller())
Expand Down Expand Up @@ -177,17 +179,20 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
ParatooCollectionId paratooCollectionId = buildCollectionId()
Map dataSet = [dataSetId:'d1', grantId:'g1', surveyId:paratooCollectionId.toMap()]
dataSet.surveyId.survey_metadata.orgMintedUUID = orgMintedId
Map expectedDataSet = dataSet + [progress: Activity.STARTED, startDate: null, endDate: null, areSpeciesRecorded: false, activityId: '123', siteId: null]
Map expectedDataSetSync = dataSet + [progress: Activity.STARTED]
Map expectedDataSetAsync = dataSet + [progress: Activity.STARTED, startDate: null, endDate: null, areSpeciesRecorded: false, activityId: '123', siteId: null]
ParatooProject project = new ParatooProject(id: projectId, project: new Project(projectId: projectId, custom: [dataSets: [dataSet]]))

when:
Map result = service.submitCollection(collection, project)
waitAll(result.promise)

then:
1 * webService.getJson({it.indexOf('/coarse-woody-debris-surveys') >= 0}, null, _, false) >> [data:[], meta:[pagination:[total:0]]]
1 * webService.doPost(*_) >> [resp: [collections: ["coarse-woody-debris-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z"]]]]
2 * tokenService.getAuthToken(true) >> Mock(AccessToken)
1 * projectService.update([custom: [dataSets: [expectedDataSet]]], 'p1', false) >> [status: 'ok']
1 * tokenService.getAuthToken(true) >> Mock(AccessToken)
1 * projectService.update([custom: [dataSets: [expectedDataSetAsync]]], 'p1', false) >> [status: 'ok']
1 * projectService.update([custom: [dataSets: [expectedDataSetSync]]], 'p1', false) >> [status: 'ok']
1 * activityService.create(_) >> [activityId: '123']
1 * recordService.getAllByActivity('123') >> []
1 * settingService.getSetting('paratoo.surveyData.mapping') >> {
Expand All @@ -205,7 +210,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
1 * userService.getCurrentUserDetails() >> [userId: userId]

and:
result == [status: 'ok']
result.updateResult == [status: 'ok']
}

void "The service can create a site from a submitted plot-selection"() {
Expand Down Expand Up @@ -289,12 +294,13 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer

when:
Map result = service.submitCollection(collection, project)
waitAll(result.promise)

then:
1 * webService.getJson({ it.indexOf('/basal-area-dbh-measure-surveys') >= 0 }, null, _, false) >> [data: [surveyData], meta: [pagination: [total: 0]]]
1 * webService.doPost(*_) >> [resp: [collections: ["basal-area-dbh-measure-survey": [uuid: "1", createdAt: "2023-09-01T00:00:00.123Z"]]]]
2 * tokenService.getAuthToken(true) >> Mock(AccessToken)
1 * projectService.update(_, projectId, false) >> [status: 'ok']
1 * tokenService.getAuthToken(true) >> Mock(AccessToken)
2 * projectService.update(_, projectId, false) >> [status: 'ok']
1 * siteService.create(_) >> { site = it[0]; [siteId: 's1'] }
1 * activityService.create(_) >> [activityId: '123']
1 * recordService.getAllByActivity('123') >> []
Expand All @@ -321,7 +327,7 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
site.externalIds[0].externalId == "4"
site.externalIds[0].idType == ExternalId.IdType.MONITOR_PLOT_GUID

result == [status: 'ok']
result.updateResult == [status: 'ok']

}

Expand Down Expand Up @@ -760,19 +766,13 @@ class ParatooServiceSpec extends MongoSpec implements ServiceUnitTest<ParatooSer
output = mapper.readValue(output, Map.class)
def relationship = '{"b": {"e": {"a": {} } }, "c": {} }'
relationship = mapper.readValue(relationship, Map.class)

def apiOutputRelationship = '{ "e": {"e": {} }, "a": "a", "b.c": {"c": {}} }'
apiOutputRelationship = mapper.readValue(apiOutputRelationship, Map.class)
when:
def result = service.rearrangeSurveyData(output, output, relationship)

def result = service.rearrangeSurveyData(output, output, relationship, apiOutputRelationship)
then:
result.size() == 2
result["b"].size() == 3
result["b"]["c"] == 1
result["b"]["d"] == 2
result["b"]["e"].size() == 2
result["b"]["e"]["a"].size() == 0
result["b"]["e"]["g"] == 3
result["f"] == 4
result.size() == 3
result == ["f": 4, "b": ["c":1, "d": 2, "e": ["g": 3, "a": [:]]], "c": 1]
}

void "resolveModelReferences should swap model with definitions"() {
Expand Down
Loading

0 comments on commit cb01b50

Please sign in to comment.