Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/dev' into feature/issue994
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrisala
chrisala committed Sep 6, 2024
2 parents d2e7dbd + 2051935 commit b6aba4e
Show file tree
Hide file tree
Showing 8 changed files with 193 additions and 64 deletions.
16 changes: 11 additions & 5 deletions grails-app/controllers/au/org/ala/ecodata/MetadataController.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import static au.org.ala.ecodata.Status.DELETED

@au.ala.org.ws.security.RequireApiKey(scopesFromProperty=["app.readScope"])
class MetadataController {
static responseFormats = ['json']

def metadataService, activityService, commonService, projectService, webService

Expand Down Expand Up @@ -234,17 +235,22 @@ class MetadataController {
if (file && outputName) {

def data = metadataService.excelWorkbookToMap(file.inputStream, outputName, true)
def status = 200

def result
if (!data) {
response.status = 400
if (data.error) {
status = 500
data.status = 500
result = data
}
else if (!data.success) {
status = 400
result = [status:400, error:'No data was found that matched the output description identified by the type parameter, please check the template you used to upload the data. ']
}
else {
result = [status: 200, data:data]
result = [status: 200, data:data.success]
}
render result as JSON

respond(result, status: status)
}
else {
response.status = 400
Expand Down
6 changes: 5 additions & 1 deletion grails-app/i18n/messages.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,4 +309,8 @@ projectAcitivity.attribution={0}. ({1}) {2} dataset download. Retrieved from {3}
activityForm.invalidIndex=Form template {0} has invalid index fields {1}
activityForm.latestVersionIsInDraft = Cannot create a new draft of a form when the current version is a draft

user.userHubs.hubId.unique=Each UserHub must have a unique hubId
user.userHubs.hubId.unique=Each UserHub must have a unique hubId

bulkimport.conversionToObjectError=Error parsing data into an object in serial number {0}
bulkimport.errorGroupBySerialNumber=Error making nested object from multiple rows in serial number {0}
bulkimport.conversionToFormSectionError=Error parsing data for form section (output) - {0}
85 changes: 54 additions & 31 deletions grails-app/services/au/org/ala/ecodata/MetadataService.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -736,48 +736,71 @@ class MetadataService {
excelImportService.mapSheet(workbook, config)
}

List excelWorkbookToMap(InputStream excelWorkbookIn, String activityFormName, Boolean normalise, Integer formVersion = null) {
Map excelWorkbookToMap(InputStream excelWorkbookIn, String activityFormName, Boolean normalise, Integer formVersion = null) {
List result = []
List errors = []
Workbook workbook = WorkbookFactory.create(excelWorkbookIn)
ActivityForm form = activityFormService.findActivityForm(activityFormName, formVersion)
form?.sections?.each { FormSection section ->
String sectionName = section.name
List model = annotatedOutputDataModel(sectionName)
String sheetName = XlsExporter.sheetName(sectionName)
Sheet sheet = workbook.getSheet(sheetName)
def columnMap = excelImportService.getDataHeaders(sheet)
def config = [
sheet:sheetName,
startRow:2,
columnMap:columnMap
]
List data = excelImportService.mapSheet(workbook, config)
List normalisedData = []
if(normalise) {
data.collect { Map row ->
def normalisedRow = [:]
row.each { cell ->
excelImportService.convertDotNotationToObject(normalisedRow, cell.key, cell.value)
excelImportService.removeEmptyObjects(normalisedRow)
try {
String sectionName = section.name
List model = annotatedOutputDataModel(sectionName)
String sheetName = XlsExporter.sheetName(sectionName)
Sheet sheet = workbook.getSheet(sheetName)
def columnMap = excelImportService.getDataHeaders(sheet)
def config = [
sheet:sheetName,
startRow:2,
columnMap:columnMap
]
List data = excelImportService.mapSheet(workbook, config)
List normalisedData = []
if(normalise) {
data.collect { Map row ->
def normalisedRow = [:]
try {
row.each { cell ->
excelImportService.convertDotNotationToObject(normalisedRow, cell.key, cell.value)
excelImportService.removeEmptyObjects(normalisedRow)
}

addOutputSpeciesIdToSpeciesData(normalisedRow, model)
normalisedData << normalisedRow
}
catch (Exception ex) {
errors << [message: messageSource.getMessage('bulkimport.conversionToObjectError', [row.serial].toArray(), '', Locale.default), error: ex.message]
}
}
}

addOutputSpeciesIdToSpeciesData(normalisedRow, model)
normalisedData << normalisedRow

List rollUpData = []
Map groupedBySerial = normalisedData.groupBy {it[OutputUploadTemplateBuilder.SERIAL_NUMBER_DATA]}
groupedBySerial.each { key, List rows ->
try {
rollUpData << rollUpDataIntoSingleElement(rows, model)
}
catch (Exception ex) {
errors << [message: messageSource.getMessage('bulkimport.errorGroupBySerialNumber', [key].toArray(), '', Locale.default), error: ex.message]
}
}
}

List rollUpData = []
Map groupedBySerial = normalisedData.groupBy {it[OutputUploadTemplateBuilder.SERIAL_NUMBER_DATA]}
groupedBySerial.each { key, List rows ->
rollUpData << rollUpDataIntoSingleElement(rows, model)
result.addAll(rollUpData.collect {
[[outputName: activityFormName, data: it]]
})
}
catch (Exception ex) {
errors << [message: messageSource.getMessage('bulkimport.conversionToFormSectionError', [section.name].toArray(), '', Locale.default), error: ex.message]
}

result.addAll(rollUpData.collect {
[[outputName: activityFormName, data: it]]
})
}

result
if (errors) {
String errorMessage = errors.collect { it.message + " - " + it.error }.join("<br/>")
return [error: errorMessage]
}
else {
return [success: result]
}
}

boolean isRowValidNextMemberOfArray(Map row, List models) {
Expand Down
9 changes: 7 additions & 2 deletions grails-app/services/au/org/ala/ecodata/SpatialService.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ class SpatialService {
List checkForBoundaryIntersectionInLayers = metadataService.getGeographicConfig().checkForBoundaryIntersectionInLayers
if (!mainGeometry.isValid()) {
log.info("Main geometry invalid. Cannot check intersection is near boundary.")
return response
return [response, [:]]
}
Map filteredResponse = [:]
Map intersectionAreaByFacets = [:].withDefault { [:] }
Expand Down Expand Up @@ -241,11 +241,16 @@ class SpatialService {
double intersectArea = intersection.getArea()
double mainGeometryArea = mainGeometry.getArea()
double proportion = 0.0
double area = 0.0d
if (mainGeometryArea != 0.0d) {
proportion = intersectArea/mainGeometryArea
}

[proportion, GeometryUtils.area(intersection)]
if (intersectArea != 0.0d) {
area = GeometryUtils.area(intersection)
}

[proportion, area]
}

/**
Expand Down
42 changes: 21 additions & 21 deletions src/main/groovy/au/org/ala/ecodata/metadata/OutputUploadTemplateBuilder.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -439,31 +439,31 @@ class ValidationHandler implements OutputModelProcessor.Processor<ExcelValidatio
def text(Object node, ExcelValidationContext context) {
if (node.constraints) {
List constraints = []

if (node.constraints instanceof Map && node.constraints.type == 'computed') {
constraints = node.constraints.options?.collect{it.value}?.flatten()?.unique()
}
else {
constraints = node.constraints
}
constraints.eachWithIndex { value, i ->
Row row = context.validationSheet.getRow(i)
if (!row) {
row = context.validationSheet.createRow(i)
if (node.constraints instanceof Map == false || node.constraints instanceof Map && node.constraints.type != 'pre-populated') {
if (node.constraints instanceof Map && node.constraints.type == 'computed') {
constraints = node.constraints.options?.collect{it.value}?.flatten()?.unique()
}
Cell cell = row.createCell(context.currentColumn)
cell.setCellValue(value)
}
def colString = CellReference.convertNumToColString(context.currentColumn)
def rangeFormula = "'${context.validationSheet.getSheetName()}'!\$${colString}\$1:\$${colString}\$${constraints.size()}"

DataValidationHelper dvHelper = context.currentSheet.getDataValidationHelper();
DataValidationConstraint dvConstraint =
dvHelper.createFormulaListConstraint(rangeFormula)
else {
constraints = node.constraints
}
constraints.eachWithIndex { value, i ->
Row row = context.validationSheet.getRow(i)
if (!row) {
row = context.validationSheet.createRow(i)
}
Cell cell = row.createCell(context.currentColumn)
cell.setCellValue(value)
}
def colString = CellReference.convertNumToColString(context.currentColumn)
def rangeFormula = "'${context.validationSheet.getSheetName()}'!\$${colString}\$1:\$${colString}\$${constraints.size()}"

DataValidationHelper dvHelper = context.currentSheet.getDataValidationHelper();
DataValidationConstraint dvConstraint =
dvHelper.createFormulaListConstraint(rangeFormula)

addValidation(node, context, dvConstraint)

addValidation(node, context, dvConstraint)
}
}
}

Expand Down
4 changes: 2 additions & 2 deletions src/main/groovy/au/org/ala/ecodata/reporting/ProjectXlsExporter.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,8 @@ class ProjectXlsExporter extends ProjectExporter {
List<String> rdpSTHeaders=commonProjectHeaders +["Service", "Target measure", "Project Outcome/s", "Total to be delivered","2023/2024","2024/2025","2025/2026","2026/2027","2027/2028","2028/2029","2029/2030"]
List<String> rdpSTProperties=commonProjectProperties +["service", "targetMeasure", "relatedOutcomes", "total", "2023/2024","2024/2025","2025/2026","2026/2027","2027/2028","2028/2029","2029/2030"]

List<String> rlpSTProperties=commonProjectProperties +["service", "targetMeasure", "total", "2018/2019","2019/2020", "2020/2021", "2021/2022", "2022/2023", "targetDate" ]
List<String> rlpSTHeaders=commonProjectHeaders +["Service", "Target measure", "Total to be delivered", "2018/2019","2019/2020", "2020/2021", "2021/2022", "2022/2023", "Target Date"]
List<String> rlpSTProperties=commonProjectProperties +["service", "targetMeasure", "relatedOutcomes", "total", "2018/2019","2019/2020", "2020/2021", "2021/2022", "2022/2023", "targetDate" ]
List<String> rlpSTHeaders=commonProjectHeaders +["Service", "Target measure", "Project Outcome/s", "Total to be delivered", "2018/2019","2019/2020", "2020/2021", "2021/2022", "2022/2023", "Target Date"]

List<String> rlpKeyThreatHeaders =commonProjectHeaders + ['Key threats and/or threatening processes', 'Interventions to address threats']
List<String> rlpKeyThreatProperties =commonProjectProperties + ['keyThreat', 'keyTreatIntervention']
Expand Down
95 changes: 93 additions & 2 deletions src/test/groovy/au/org/ala/ecodata/MetadataServiceSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,8 @@ class MetadataServiceSpec extends MongoSpec implements ServiceUnitTest<MetadataS
def file = new File("src/test/resources/bulk_import_example.xlsx").newInputStream()

when:
def content = service.excelWorkbookToMap(file, 'form1', true, null)
def resp = service.excelWorkbookToMap(file, 'form1', true, null)
def content = resp.success

then:
content.size() == 2
Expand Down Expand Up @@ -329,7 +330,97 @@ class MetadataServiceSpec extends MongoSpec implements ServiceUnitTest<MetadataS
content[1][0].data.get("g")[1].i.outputSpeciesId != null
}

void "getGeographicConfig should get geographic configuration either from hub or system default"(){
def "excelWorkbookToMap: getting content from bulk_import_example_error.xlsx should result in an error"() {
setup:
service.activityFormService = new ActivityFormService()
service.cacheService = new CacheService()
service.excelImportService = new ExcelImportService()
ActivityForm form1 = new ActivityForm(
name: 'form1',
formVersion: 1,
status: Status.ACTIVE,
publicationStatus: PublicationStatus.PUBLISHED,
type: 'Activity',
sections: [
new FormSection (
name: 'form1',
templateName: 'form1',
template: [
name: 'form1',
dataModel: [
[
name: 'a',
dataType: 'string',
label: 'A',
required: true
],
[
name: 'b',
dataType: 'list',
required: true,
columns: [
[
name: 'c',
dataType: 'stringList',
required: true
]
]
],
[
name: 'd',
dataType: 'list',
required: true,
columns: [
[
name: 'e',
dataType: 'string',
required: true
],
[
name: 'f',
dataType: 'species',
required: true
]
]
],
[
name: 'g',
dataType: 'list',
required: true,
columns: [
[
name: 'h',
dataType: 'stringList',
required: true
],
[
name: 'i',
dataType: 'species',
required: true
]
]
]
]
]
)
]
)
form1.save(flush: true, failOnError: true)
def file = new File("src/test/resources/bulk_import_example_error.xlsx").newInputStream()

when:
def content = service.excelWorkbookToMap(file, 'form1', true, null)

then:
messageSource.getMessage("bulkimport.conversionToObjectError", _, "", Locale.default) >> { code, array, msg, locale-> "Error parsing data into an object in serial number ${array[0]}" }
messageSource.getMessage("bulkimport.errorGroupBySerialNumber", _, "", Locale.default) >> { code, array, msg, locale-> "Error making nested object from multiple rows in serial number ${array[0]}" }
messageSource.getMessage("bulkimport.conversionToFormSectionError", _, "", Locale.default) >> { code, array, msg, locale-> "Error parsing data for form section (output) ${array[0]}" }
content.error != null
content.error.contains("300")
content.error.contains("55")
}

void "getGeographicConfig should get geographic configuration either from hub or system default"() {
when:
def result = service.getGeographicConfig()

Expand Down
Binary file added src/test/resources/bulk_import_example_error.xlsx
View file
Binary file not shown.

0 comments on commit b6aba4e

Please sign in to comment.