diff --git a/.gitignore b/.gitignore index d80feff3..1078f06f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ out .gradle - +build .slcache /target /.idea diff --git a/build.gradle b/build.gradle index 902b25de..736a4ee3 100644 --- a/build.gradle +++ b/build.gradle @@ -18,7 +18,7 @@ plugins { } -version "1.0.6" +version "1.0.7-SNAPSHOT" group "au.org.ala" @@ -64,7 +64,7 @@ dependencies { compile group: 'org.grails.plugins', name: 'ala-bootstrap3', version: '3.2.1', changing: true compile group: 'org.grails.plugins', name: 'ala-admin-plugin', version: '2.1', changing: true compile group: 'org.grails.plugins', name: 'ala-auth', version: '3.2.2', changing: true - compile group: 'au.org.ala.plugins.grails', name:'images-client-plugin', version: '1.0.1-SNAPSHOT', changing: true + compile group: 'au.org.ala.plugins.grails', name:'images-client-plugin', version: '1.1', changing: true // Added dependencies compile 'org.grails.plugins:external-config:1.2.2' @@ -81,7 +81,7 @@ dependencies { compile ('au.org.ala:image-utils:1.8.6'){ exclude group: 'org.slf4j', module: 'slf4j-log4j12' } - compile 'org.grails.plugins:ala-ws-security-plugin:2.0' + compile 'org.grails.plugins:ala-ws-security-plugin:2.0.1' // Swagger compile 'org.grails.plugins:swagger:1.0.1' diff --git a/db-upgrade-0_9_to_1_0.sql b/db-upgrade-0_9_to_1_0.sql index ee1029c6..3b5ce663 100644 --- a/db-upgrade-0_9_to_1_0.sql +++ b/db-upgrade-0_9_to_1_0.sql @@ -1,20 +1,23 @@ alter table image add date_deleted timestamp; alter table image add recognised_license_id bigint; +alter table image add occurrence_id character varying(255); +alter table image add calibrated_by_user character varying(255); + CREATE TABLE public.license ( - id bigint NOT NULL, - version bigint NOT NULL, - url character varying(255) NOT NULL, - name character varying(255) NOT NULL, - image_url character varying(255), - acronym character varying(255) NOT NULL + id bigint NOT NULL, + version bigint NOT NULL, + url character varying(255) NOT NULL, + name character varying(255) NOT NULL, + image_url character varying(255), + acronym character varying(255) NOT NULL ); ALTER TABLE ONLY public.license ADD CONSTRAINT license_pkey PRIMARY KEY (id); CREATE TABLE public.license_mapping ( - id bigint NOT NULL, - version bigint NOT NULL, - value character varying(255) NOT NULL, - license_id bigint NOT NULL + id bigint NOT NULL, + version bigint NOT NULL, + value character varying(255) NOT NULL, + license_id bigint NOT NULL ); ALTER TABLE ONLY public.license_mapping ADD CONSTRAINT license_mapping_pkey PRIMARY KEY (id); @@ -22,4 +25,14 @@ ALTER TABLE ONLY public.license_mapping ADD CONSTRAINT fktetx2lihs6cq4swvdhaqeco7s FOREIGN KEY (license_id) REFERENCES public.license(id); -update image set mime_type = 'audio/mpeg' where original_filename like '%.mp3'; \ No newline at end of file +update image set mime_type = 'audio/mpeg' where original_filename like '%.mp3'; + + +/* Add the occurrence_id to top level table */ +CREATE TEMP TABLE temp_image_occurrence as select image_id, value as occurrence_id from image_meta_data_item imdi where imdi.name='occurrenceId'; +UPDATE image +SET occurrence_id = subquery.occurrence_id +FROM (SELECT io.image_id, io.occurrence_id from temp_image_occurrence io) AS subquery +WHERE image.id = subquery.image_id; + +DROP TABLE temp_image_occurrence; diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index a3c68745..d0d7503d 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -91,10 +91,11 @@ dataSource: environments: development: - hibernate: - dialect: org.hibernate.dialect.PostgreSQLDialect +# hibernate: +# dialect: org.hibernate.dialect.PostgreSQLDialect dataSource: dbCreate: update +# driverClassName: org.postgresql.Driver url: jdbc:postgresql://localhost/images?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8 username: "images" password: "images" @@ -136,6 +137,7 @@ environments: dialect: org.hibernate.dialect.PostgreSQLDialect dataSource: dbCreate: update + driverClassName: org.postgresql.Driver url: jdbc:postgresql://localhost/images?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8 properties: jmxEnabled: true @@ -159,7 +161,7 @@ security: cas: appServerName: 'http://dev.ala.org.au:8080' uriFilterPattern: '/admin/.*,/admin,/admin/,/image/deleteImage/.*' - authenticateOnlyIfLoggedInPattern: '' + authenticateOnlyIfLoggedInPattern: '/ws/createSubimage' bypass: false disableCAS: false diff --git a/grails-app/controllers/UrlMappings.groovy b/grails-app/controllers/UrlMappings.groovy index b0d2238d..a367d55b 100644 --- a/grails-app/controllers/UrlMappings.groovy +++ b/grails-app/controllers/UrlMappings.groovy @@ -32,6 +32,7 @@ class UrlMappings { "/ws"(controller: 'webService', action: 'swagger') // legacy URLS + "/image/imageTooltipFragment"(controller: "image", action: "imageTooltipFragment") "/image/proxyImageThumbnail"(controller: "image", action: "proxyImageThumbnail") "/image/proxyImageThumbnailLarge"(controller: "image", action: "proxyImageThumbnailLarge") "/image/proxyImageTile"(controller: "image", action: "proxyImageTile") diff --git a/grails-app/controllers/au/org/ala/images/AdminController.groovy b/grails-app/controllers/au/org/ala/images/AdminController.groovy index 48a2bf09..8cc70369 100644 --- a/grails-app/controllers/au/org/ala/images/AdminController.groovy +++ b/grails-app/controllers/au/org/ala/images/AdminController.groovy @@ -91,7 +91,7 @@ class AdminController { if (storeResult.image) { imageService.schedulePostIngestTasks(storeResult.image.id, storeResult.image.imageIdentifier, storeResult.image.originalFilename, userId) } else { - imageService.scheduleNonImagePostIngestTasks(storeResult.image.id, storeResult.image.imageIdentifier, storeResult.image.originalFilename, userId) + imageService.scheduleNonImagePostIngestTasks(storeResult.image.id) } flash.message = "Image uploaded with identifier: ${storeResult.image?.imageIdentifier}" redirect(action:'upload') @@ -112,23 +112,27 @@ class AdminController { def headers = [] def batch = [] - file.inputStream.eachCsvLine { tokens -> - if (lineCount == 0) { - headers = tokens - } else { - def m = [:] - for (int i = 0; i < headers.size(); ++i) { - m[headers[i]] = tokens[i] + try { + file.inputStream.eachCsvLine { tokens -> + if (lineCount == 0) { + headers = tokens + } else { + def m = [:] + for (int i = 0; i < headers.size(); ++i) { + m[headers[i]] = tokens[i] + } + batch << m } - batch << m + lineCount++ } - lineCount++ + scheduleImagesUpload(batch, authService.getUserId()) + renderResults([success: true, message:'Image upload started']) + } catch (Exception e){ + log.error(e.getMessage(), e) + renderResults([success: false, message: "Problem reading CSV file. Please check contents."]) } - - scheduleImagesUpload(batch, '-2') - renderResults([success: true, message:'Image upload started']) } else { - renderResults([success: false, message: "Expected multipart request containing 'csvfile' file parameter"]) + renderResults([success: false, message: "Problem reading CSV file from upload."]) } } @@ -145,6 +149,12 @@ class AdminController { } } + def scheduleDeletedImagesPurge(){ + imageService.scheduleBackgroundTask(new DeletedImagesPurgeBackgroundTask(imageService)) + flash.message = "Deleted images purge started. Refresh dashboard for progress." + redirect(action:'dashboard') + } + def licences(){} def updateStoredLicences(){ diff --git a/grails-app/controllers/au/org/ala/images/ImageController.groovy b/grails-app/controllers/au/org/ala/images/ImageController.groovy index 187f6d8f..b4c9fdea 100644 --- a/grails-app/controllers/au/org/ala/images/ImageController.groovy +++ b/grails-app/controllers/au/org/ala/images/ImageController.groovy @@ -365,35 +365,45 @@ class ImageController { flash.errorMessage = "Could not find image with id ${params.int("id") ?: params.imageId }!" redirect(action:'list', controller: 'search') } else { - def subimages = Subimage.findAllByParentImage(image)*.subimage - def sizeOnDisk = imageStoreService.getConsumedSpaceOnDisk(image.imageIdentifier) - - //accessible from cookie - def userEmail = AuthenticationUtils.getEmailAddress(request) - def userDetails = authService.getUserForEmailAddress(userEmail, true) - def userId = userDetails ? userDetails.id : "" - - def isAdmin = false - if (userDetails){ - if (userDetails.getRoles().contains("ROLE_ADMIN")) - isAdmin = true - } + getImageModel(image) + } + } + } - def thumbUrls = imageService.getAllThumbnailUrls(image.imageIdentifier) + private def getImageModel(Image image){ + def subimages = Subimage.findAllByParentImage(image)*.subimage + def sizeOnDisk = imageStoreService.getConsumedSpaceOnDisk(image.imageIdentifier) - boolean isImage = imageService.isImageType(image) + //accessible from cookie + def userEmail = AuthenticationUtils.getEmailAddress(request) + def userDetails = authService.getUserForEmailAddress(userEmail, true) + def userId = userDetails ? userDetails.id : "" - //add additional metadata - def resourceLevel = collectoryService.getResourceLevelMetadata(image.dataResourceUid) + def isAdmin = false + if (userDetails){ + if (userDetails.getRoles().contains("ROLE_ADMIN")) + isAdmin = true + } - if (grailsApplication.config.analytics.trackDetailedView.toBoolean()) { - sendAnalytics(image, 'imagedetailedview') - } + if (!userId){ + userId = authService.getUserId() + } - [imageInstance: image, subimages: subimages, sizeOnDisk: sizeOnDisk, - squareThumbs: thumbUrls, isImage: isImage, resourceLevel: resourceLevel, isAdmin:isAdmin, userId:userId] - } + def thumbUrls = imageService.getAllThumbnailUrls(image.imageIdentifier) + + boolean isImage = imageService.isImageType(image) + + //add additional metadata + def resourceLevel = collectoryService.getResourceLevelMetadata(image.dataResourceUid) + + if (grailsApplication.config.analytics.trackDetailedView.toBoolean()) { + sendAnalytics(image, 'imagedetailedview') } + + [imageInstance: image, subimages: subimages, + parentImage: image.parent, + sizeOnDisk: sizeOnDisk, + squareThumbs: thumbUrls, isImage: isImage, resourceLevel: resourceLevel, isAdmin:isAdmin, userId:userId] } def view() { @@ -450,6 +460,12 @@ class ImageController { [imageInstance: imageInstance, metaData: metaData?.sort { it.name }, source: source] } + def coreImageMetadataTableFragment() { + def imageInstance = imageService.getImageFromParams(params) + + render(view: '_coreImageMetadataFragment', model: getImageModel(imageInstance)) + } + def imageTooltipFragment() { def imageInstance = imageService.getImageFromParams(params) [imageInstance: imageInstance] diff --git a/grails-app/controllers/au/org/ala/images/WebServiceController.groovy b/grails-app/controllers/au/org/ala/images/WebServiceController.groovy index 36b36d9a..5154afda 100644 --- a/grails-app/controllers/au/org/ala/images/WebServiceController.groovy +++ b/grails-app/controllers/au/org/ala/images/WebServiceController.groovy @@ -5,7 +5,6 @@ import au.org.ala.cas.util.AuthenticationUtils import au.org.ala.ws.security.ApiKeyInterceptor import grails.converters.JSON import grails.converters.XML -import groovyx.net.http.HTTPBuilder import io.swagger.annotations.Api import io.swagger.annotations.ApiImplicitParam import io.swagger.annotations.ApiImplicitParams @@ -618,7 +617,8 @@ class WebServiceController { ]) def getImageInfo() { def results = [success:false] - def image = Image.findByImageIdentifier(params.imageID as String) + def imageId = params.id ? params.id : params.imageID + def image = Image.findByImageIdentifier(imageId as String) if (image) { results.success = true addImageInfoToMap(image, results, params.boolean("includeTags"), params.boolean("includeMetadata")) @@ -695,6 +695,9 @@ class WebServiceController { def getRepositoryStatistics() { def results = [:] results.imageCount = Image.count() + results.deletedImageCount = Image.countByDateDeletedIsNotNull() + results.licenceCount = License.count() + results.licenceMappingCount = LicenseMapping.count() results.sizeOnDisk = renderResults(results) } @@ -739,9 +742,43 @@ class WebServiceController { renderResults(results) } + def createSubimage() { + def image = Image.findByImageIdentifier(params.id as String) + if (!image) { + renderResults([success:false, message:"Image not found: ${params.id}"]) + return + } + + if (!params.x || !params.y || !params.height || !params.width) { + renderResults([success:false, message:"Rectangle not correctly specified. Use x, y, height and width params"]) + return + } + + def x = params.int('x') + def y = params.int('y') + def height = params.int('height') + def width = params.int('width') + def title = params.title + def description = params.description + + if (height == 0 || width == 0) { + renderResults([success:false, message:"Rectangle not correctly specified. Height and width cannot be zero"]) + return + } + + def userId = getUserIdForRequest(request) + if(!userId){ + renderResults([success:false, message:"User needs to be logged in to create sub image"]) + return + } + + def subimage = imageService.createSubimage(image, x, y, width, height, userId, [title:title, description:description] ) + renderResults([success: subimage != null, subImageId: subimage?.imageIdentifier]) + } + @ApiOperation( value = "Create Subimage", - nickname = "createSubimage", + nickname = "subimage", produces = "application/json", consumes = "application/json", httpMethod = "PUT", @@ -755,7 +792,7 @@ class WebServiceController { @ApiResponse(code = 404, message = "Image Not Found")] ) @RequireApiKey - def createSubimage() { + def subimage() { def image = Image.findByImageIdentifier(params.id as String) if (!image) { renderResults([success:false, message:"Image not found: ${params.id}"]) @@ -789,6 +826,7 @@ class WebServiceController { renderResults([success: subimage != null, subImageId: subimage?.imageIdentifier]) } + def getSubimageRectangles() { def image = Image.findByImageIdentifier(params.id as String) @@ -1180,19 +1218,20 @@ class WebServiceController { } @ApiOperation( - value = "Find images by image metadata", + value = "Find images by image metadata - deprecated. Use /search instead", nickname = "findImagesByMetadata", produces = "application/json", consumes = "application/json", httpMethod = "POST", response = Map.class, - tags = ["Search"] + tags = ["Deprecated"] ) @ApiResponses([ @ApiResponse(code = 200, message = "OK"), @ApiResponse(code = 405, message = "Method Not Allowed. Only GET is allowed"), @ApiResponse(code = 404, message = "Image Not Found")] ) + @Deprecated def findImagesByMetadata() { def query = request.JSON @@ -1212,22 +1251,13 @@ class WebServiceController { return } - def images = elasticSearchService.searchByMetadata(key, values, params) - def results = [:] - def keyValues = imageService.getMetadataItemValuesForImages(images.list, key) - images?.list?.each { image -> - def map = imageService.getImageInfoMap(image) - collectoryService.addMetadataForResource(image) - def keyValue = keyValues[image.id] - def list = results[keyValue] - if (!list) { - list = [] - results[keyValue] = list - } - list << map + def results = elasticSearchService.searchByMetadata(key, values, params) + def totalCount = 0 + results.values().each { + totalCount = totalCount + it.size() } - renderResults([success: true, images: results, count:images?.totalCount ?: 0]) + renderResults([success: true, images: results, count: totalCount]) return } @@ -1515,7 +1545,6 @@ class WebServiceController { } def calibrateImageScale() { - def userId = getUserIdForRequest(request) def image = Image.findByImageIdentifier(params.imageId) def units = params.units ?: "mm" @@ -1524,9 +1553,9 @@ class WebServiceController { if (image && units && pixelLength && actualLength) { def pixelsPerMM = imageService.calibrateImageScale(image, pixelLength, actualLength, units, userId) renderResults([success: true, pixelsPerMM:pixelsPerMM, message:"Image is scaled at ${pixelsPerMM} pixels per mm"]) - return + } else { + renderResults([success: false, message: 'Missing one or more required parameters: imageId, pixelLength, actualLength, units']) } - renderResults([success:false, message:'Missing one or more required parameters: imageId, pixelLength, actualLength, units']) } def resetImageCalibration() { diff --git a/grails-app/domain/au/org/ala/images/Image.groovy b/grails-app/domain/au/org/ala/images/Image.groovy index 17373df3..bc333b7f 100644 --- a/grails-app/domain/au/org/ala/images/Image.groovy +++ b/grails-app/domain/au/org/ala/images/Image.groovy @@ -43,7 +43,10 @@ class Image { String rightsHolder @SearchableProperty(description="A legal document giving official permission to do something with the resource.") String license + @SearchableProperty(description="Associated occurrence ID.") + String occurrenceId + String calibratedByUser License recognisedLicense @SearchableProperty(valueType = CriteriaValueType.NumberRangeInteger, units = "pixels", description = "The height of the thumbnail in pixels") @@ -89,9 +92,11 @@ class Image { thumbWidth nullable: true squareThumbSize nullable: true mmPerPixel nullable: true + calibratedByUser nullable:true harvestable nullable: true dateDeleted nullable: true + occurrenceId nullable: true } static mapping = { diff --git a/grails-app/services/au/org/ala/images/ElasticSearchService.groovy b/grails-app/services/au/org/ala/images/ElasticSearchService.groovy index ef07696d..3e720eb7 100644 --- a/grails-app/services/au/org/ala/images/ElasticSearchService.groovy +++ b/grails-app/services/au/org/ala/images/ElasticSearchService.groovy @@ -2,11 +2,16 @@ package au.org.ala.images import com.opencsv.CSVWriter import grails.converters.JSON +import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsRequest +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse import org.elasticsearch.action.delete.DeleteRequest import org.elasticsearch.action.search.SearchScrollRequest +import org.elasticsearch.client.indices.GetFieldMappingsRequest +import org.elasticsearch.cluster.metadata.MappingMetaData import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.common.xcontent.XContentType import groovy.json.JsonOutput @@ -149,7 +154,7 @@ class ElasticSearchService { data.thumbWidth, data.harvestable, data.recognisedLicence, - data.occurrenceID + data.occurrenceId ) ct.stop(true) } @@ -178,7 +183,7 @@ class ElasticSearchService { thumbWidth, harvestable, recognisedLicence, - occurrenceID + occurrenceId ){ def data = [ imageIdentifier: imageIdentifier, @@ -204,7 +209,7 @@ class ElasticSearchService { thumbWidth:thumbWidth, harvestable:harvestable, recognisedLicence: recognisedLicence, - occurrenceID: occurrenceID + occurrenceID: occurrenceId ] addAdditionalIndexFields(data) @@ -265,15 +270,17 @@ class ElasticSearchService { } else if (imageSize < 1000000){ data.imageSize = "less than 1m" } else { - data.imageSize = (imageSize / 1000000).intValue() +"m" + data.imageSize = (imageSize / 1000000).intValue() + "m" } data } def deleteImage(Image image) { if (image) { - DeleteResponse response = client.delete(new DeleteRequest(grailsApplication.config.elasticsearch.indexName, image.id.toString()), RequestOptions.DEFAULT) - log.info(response.status()) + DeleteResponse response = client.delete(new DeleteRequest(grailsApplication.config.elasticsearch.indexName, image.imageIdentifier), RequestOptions.DEFAULT) + if (response.status() && response.status().status){ + log.info(response.status().status.toString()) + } } } @@ -609,7 +616,19 @@ class ElasticSearchService { }, "creator": { "type": "keyword" - } + }, + "width": { + "type": "integer" + }, + "height": { + "type": "integer" + }, + "thumbHeight": { + "type": "integer" + }, + "thumbWidth": { + "type": "integer" + } } }""", XContentType.JSON) @@ -658,48 +677,22 @@ class ElasticSearchService { boolQueryBuilder } - QueryResults searchByMetadata(String key, List values, GrailsParameterMap params) { - def queryString = values.collect { key.toLowerCase() + ":\"" + it + "\""}.join(" OR ") - QueryStringQueryBuilder builder = QueryBuilders.queryStringQuery(queryString) - builder.defaultField("content") - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() - searchSourceBuilder.query(builder) - return executeSearch(searchSourceBuilder, params) - } - - private QueryResults executeSearch(GrailsParameterMap params) { + Map searchByMetadata(String key, List values, GrailsParameterMap params) { - try { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() - - QueryBuilder queryBuilder = QueryBuilders.queryStringQuery(params.q) - - def filterQueries = params.findAll { it.key == 'fq'} - - def filters = [] + def properties = getMetadataKeys() + def caseInsensitive = [:] + properties.each { caseInsensitive.put(it.toLowerCase(), it)} + def indexField = caseInsensitive.get(key.toLowerCase()) - if (filterQueries) { - filterQueries.each { + def queryString = values.collect { "\"${it}\"" }.join(" OR ") + QueryStringQueryBuilder queryBuilder = QueryBuilders.queryStringQuery(queryString) - if(it.value instanceof String[]){ - it.value.each { filter -> - def kv = filter.split(":") - filters << QueryBuilders.termQuery(kv[0], kv[1]) - } - } else { - def kv = it.value.split(":") - filters << QueryBuilders.termQuery(kv[0], kv[1]) - } - } - } + //find indexed field...... + queryBuilder.defaultField(indexField) + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + searchSourceBuilder.query(queryBuilder) - if (filters) { - BoolQueryBuilder builder = QueryBuilders.boolQuery() - filters.each { - builder.must(it) - } - queryBuilder = builder.must(queryBuilder) //QueryBuilders.termQuery(qsQuery). builder) - } + try { if (params?.offset) { searchSourceBuilder.from(params.int("offset")) @@ -708,7 +701,7 @@ class ElasticSearchService { if (params?.max) { searchSourceBuilder.size(params.int("max")) } else { - searchSourceBuilder.size(Integer.MAX_VALUE) // probably way too many! + searchSourceBuilder.size(grailsApplication.config.elasticsearch.maxOffset) // probably way too many! } if (params?.sort) { @@ -716,11 +709,6 @@ class ElasticSearchService { searchSourceBuilder.sort(params.sort as String, order) } - // aggregations = facets - grailsApplication.config.facets.each { facet -> - searchSourceBuilder.aggregation(AggregationBuilders.terms(facet as String).field(facet as String)) - } - def ct = new CodeTimer("Index search") SearchRequest searchRequest = new SearchRequest() searchRequest.indices(grailsApplication.config.elasticsearch.indexName as String) @@ -731,41 +719,44 @@ class ElasticSearchService { ct = new CodeTimer("Object retrieval (${searchResponse.hits.hits.length} of ${searchResponse.hits.totalHits} hits)") def imageList = [] - def aggregations = [:] if (searchResponse.hits) { searchResponse.hits.each { hit -> - imageList << hit.getSourceAsMap() - } - searchResponse.aggregations.each { - def facet = [:] - it.buckets.each { bucket -> - facet[bucket.getKeyAsString()] = bucket.getDocCount() - } - aggregations.put(it.name, facet) + def image = Image.findByImageIdentifier(hit.id) + image.metadata = null + image.tags = null + imageList << image } } ct.stop(true) - return new QueryResults(list: imageList, aggregations:aggregations, totalCount: searchResponse?.hits?.totalHits.value ?: 0) + def resultsKeyedByValue = [:] + + imageList.each { + + def caseInsensitiveMap = [:] + it.properties.each { k, v -> caseInsensitiveMap[k.toLowerCase()] = v } + def keyValue = caseInsensitiveMap.get(indexField.toLowerCase()) + def list = resultsKeyedByValue.get(keyValue, []) + list << it + resultsKeyedByValue.put(keyValue, list) + } + + return resultsKeyedByValue } catch (SearchPhaseExecutionException e) { log.warn(".SearchPhaseExecutionException thrown - this is expected behaviour for a new empty system.") - return new QueryResults(list: [], totalCount: 0) + return [:] } catch (Exception e) { e.printStackTrace() log.warn("Exception thrown - this is expected behaviour for a new empty system.") - return new QueryResults(list: [], totalCount: 0) + return [:] } } def getMetadataKeys() { - ClusterState cs = client.admin().cluster().prepareState().execute().actionGet().getState(); - IndexMetaData imd = cs.getMetaData().index(grailsApplication.config.elasticsearch.indexName as String) - Map mdd = imd.mapping("image").sourceAsMap() - Map metadata = mdd?.properties?.metadata?.properties - def names = [] - if (metadata) { - names = metadata.collect { it.key } - } - names + GetMappingsRequest request = new GetMappingsRequest(); + request.indices("images") + GetMappingsResponse getMappingResponse = client.indices().getMapping(request, RequestOptions.DEFAULT) + Map properties = getMappingResponse.mappings().values().first().value.first().value.sourceAsMap().properties + properties.keySet() } } diff --git a/grails-app/services/au/org/ala/images/ImageService.groovy b/grails-app/services/au/org/ala/images/ImageService.groovy index 7eb73f06..c4143665 100644 --- a/grails-app/services/au/org/ala/images/ImageService.groovy +++ b/grails-app/services/au/org/ala/images/ImageService.groovy @@ -38,6 +38,8 @@ class ImageService { private static int BACKGROUND_TASKS_BATCH_SIZE = 100 + Map imagePropertyMap = null + ImageStoreResult storeImage(MultipartFile imageFile, String uploader, Map metadata = [:]) { if (imageFile) { @@ -96,7 +98,7 @@ class ImageService { result.success = true auditService.log(storeResult.image, "Image (batch) downloaded from ${imageUrl}", uploader ?: "") } catch (Exception ex) { - ex.printStackTrace() + log.error("Problem storing image - " + ex.getMessage(), ex) result.message = ex.message } results[imageUrl] = result @@ -171,9 +173,10 @@ class ImageService { //update metadata metadata.each { kvp -> - if(image.hasProperty(kvp.key) && kvp.value){ - if(!(kvp.key in ["dateTaken", "dateUploaded", "id"])){ - image[kvp.key] = kvp.value + def propertyName = hasImageCaseFriendlyProperty(image, kvp.key) + if (propertyName && kvp.value){ + if(!(propertyName in ["dateTaken", "dateUploaded", "id"])){ + image[propertyName] = kvp.value } } } @@ -183,13 +186,25 @@ class ImageService { new ImageStoreResult(image, preExisting) } + + def hasImageCaseFriendlyProperty(Image image, String propertyName){ + if (!imagePropertyMap) { + def properties = image.getProperties().keySet() + imagePropertyMap = [:] + properties.each { imagePropertyMap.put(it.toLowerCase(), it) } + } + imagePropertyMap.get(propertyName.toLowerCase()) + } + + + def schedulePostIngestTasks(Long imageId, String identifier, String fileName, String uploaderId){ scheduleArtifactGeneration(imageId, uploaderId) scheduleImageIndex(imageId) scheduleImageMetadataPersist(imageId,identifier, fileName, MetaDataSourceType.Embedded, uploaderId) } - def scheduleNonImagePostIngestTasks(Long imageId, String identifier, String fileName, String uploaderId){ + def scheduleNonImagePostIngestTasks(Long imageId){ scheduleImageIndex(imageId) } @@ -420,44 +435,7 @@ class ImageService { if (image) { - // need to delete it from user selections - def selected = SelectedImage.findAllByImage(image) - selected.each { selectedImage -> - selectedImage.delete() - } - - // Need to delete tags - def tags = ImageTag.findAllByImage(image) - tags.each { tag -> - tag.delete() - } - - // Delete keywords - def keywords = ImageKeyword.findAllByImage(image) - keywords.each { keyword -> - keyword.delete() - } - - // If this image is a subimage, also need to delete any subimage rectangle records - def subimagesRef = Subimage.findAllBySubimage(image) - subimagesRef.each { subimage -> - subimage.delete() - } - - // This image may also be a parent image - def subimages = Subimage.findAllByParentImage(image) - subimages.each { subimage -> - // need to detach this image from the child images, but we do not actually delete the sub images. They - // will live on as root images in their own right - subimage.subimage.parent = null - subimage.delete() - } - - // thumbnail records... - def thumbs = ImageThumbnail.findAllByImage(image) - thumbs.each { thumb -> - thumb.delete() - } + deleteRelatedArtefacts(image) // Delete from the index... elasticSearchService.deleteImage(image) @@ -474,6 +452,59 @@ class ImageService { return false } + private def deleteRelatedArtefacts(Image image){ + + // need to delete it from user selections + def selected = SelectedImage.findAllByImage(image) + selected.each { selectedImage -> + selectedImage.delete() + } + + // Need to delete tags + def tags = ImageTag.findAllByImage(image) + tags.each { tag -> + tag.delete() + } + + // Delete keywords + def keywords = ImageKeyword.findAllByImage(image) + keywords.each { keyword -> + keyword.delete() + } + + // If this image is a subimage, also need to delete any subimage rectangle records + def subimagesRef = Subimage.findAllBySubimage(image) + subimagesRef.each { subimage -> + subimage.delete() + } + + // This image may also be a parent image + def subimages = Subimage.findAllByParentImage(image) + subimages.each { subimage -> + // need to detach this image from the child images, but we do not actually delete the sub images. They + // will live on as root images in their own right + subimage.subimage.parent = null + subimage.delete() + } + + // thumbnail records... + def thumbs = ImageThumbnail.findAllByImage(image) + thumbs.each { thumb -> + thumb.delete() + } + } + + def deleteImagePurge(Image image) { + if (image && image.dateDeleted) { + deleteRelatedArtefacts(image) + imageStoreService.deleteImage(image.imageIdentifier) + //hard delete + image.delete(flush:true) + return true + } + return false + } + List listStagedImages() { def files = [] def inboxLocation = grailsApplication.config.imageservice.imagestore.inbox as String @@ -724,8 +755,8 @@ class ImageService { def subimage = storeImageBytes(results.bytes,filename, results.bytes.length, results.contentType, userId, metadata).image def subimageRect = new Subimage(parentImage: parentImage, subimage: subimage, x: x, y: y, height: height, width: width) - subimageRect.save() subimage.parent = parentImage + subimageRect.save(flush:true) auditService.log(parentImage, "Subimage created ${subimage.imageIdentifier}", userId) auditService.log(subimage, "Subimage created from parent image ${parentImage.imageIdentifier}", userId) @@ -811,7 +842,8 @@ class ImageService { def resetImageLinearScale(Image image) { image.mmPerPixel = null; - image.save() + image.calibratedByUser = null + image.save(flush:true) scheduleImageIndex(image.id) } @@ -835,7 +867,8 @@ class ImageService { def mmPerPixel = (actualLength * scale) / pixelLength image.mmPerPixel = mmPerPixel - image.save() + image.calibratedByUser = userId + image.save(flush:true) scheduleImageIndex(image.id) return mmPerPixel diff --git a/grails-app/services/au/org/ala/images/SearchService.groovy b/grails-app/services/au/org/ala/images/SearchService.groovy index 620e24ec..d0e14aca 100644 --- a/grails-app/services/au/org/ala/images/SearchService.groovy +++ b/grails-app/services/au/org/ala/images/SearchService.groovy @@ -25,7 +25,7 @@ class SearchService { return elasticSearchService.simpleImageDownload(getSearchCriteriaList(), params, output) } - QueryResults findImagesByMetadata(String metaDataKey, List values, GrailsParameterMap params) { + Map findImagesByMetadata(String metaDataKey, List values, GrailsParameterMap params) { return elasticSearchService.searchByMetadata(metaDataKey, values, params) } diff --git a/grails-app/taglib/au/org/ala/images/ImagesTagLib.groovy b/grails-app/taglib/au/org/ala/images/ImagesTagLib.groovy index ae6ff038..228bc0ba 100644 --- a/grails-app/taglib/au/org/ala/images/ImagesTagLib.groovy +++ b/grails-app/taglib/au/org/ala/images/ImagesTagLib.groovy @@ -1,6 +1,7 @@ package au.org.ala.images import groovy.xml.MarkupBuilder +import org.apache.commons.lang3.StringUtils class ImagesTagLib { @@ -112,12 +113,19 @@ class ImagesTagLib { if(attrs.image.dataResourceUid){ def metadata = collectoryService.getResourceLevelMetadata(attrs.image.dataResourceUid) out << """
""" - out << "${metadata.name} ${attrs.image.title? ' - ' + attrs.image.title: ''} ${attrs.image.creator ? ' - ' + attrs.image.creator : ''}" + out << "${metadata.name?:''}" + if (metadata.name && (attrs.image.title || attrs.image.creator)){ + out << ' - ' + } + def text = "${attrs.image.title? attrs.image.title: ''} ${attrs.image.creator ? attrs.image.creator : ''}" + text = StringUtils.abbreviate(text, 100) + out << "${text}" out << '
' } else { if(attrs.image.dataResourceUid || attrs.image.title || attrs.image.creator){ out << """
""" - out << "${attrs.image.dataResourceUid ? attrs.image.dataResourceUid: ''} ${attrs.image.title ? attrs.image.title :''} ${attrs.image.creator ? attrs.image.creator : ''}" + def output = "${attrs.image.dataResourceUid ? attrs.image.dataResourceUid: ''} ${attrs.image.title ? attrs.image.title :''} ${attrs.image.creator ? attrs.image.creator : ''}" + out << StringUtils.abbreviate(output, 100) out << '
' } } @@ -267,7 +275,7 @@ class ImagesTagLib { } def imageMetadata = { attrs, body -> - if(attrs.image[attrs.field]){ + if (attrs.image[attrs.field]){ out << attrs.image[attrs.field] } else if(attrs.resource && attrs.resource.imageMetadata && attrs.resource.imageMetadata[attrs.field]){ out << attrs.resource.imageMetadata[attrs.field] + " (resource level metadata) " diff --git a/grails-app/views/admin/dashboard.gsp b/grails-app/views/admin/dashboard.gsp index cea07b6d..e3f6f2fe 100644 --- a/grails-app/views/admin/dashboard.gsp +++ b/grails-app/views/admin/dashboard.gsp @@ -10,19 +10,39 @@ Dashboard + +
${flash.message}
+
+ +
${flash.errorMessage}
+
+
WARNING: CAS authentication disabled - this means admin functions are exposed!
-

Statistics

+

Database statistics

- + + + + + + + + + + + + +
Image countImage count
Deleted image count
Licences count
Licence mapping count
-

Background processing

+

Note: these counts are taken from the database, not the search index.

+

Background processing

+ + + +
@@ -57,6 +77,9 @@ function updateRepoStatistics() { $.ajax("${createLink(controller:'webService', action:'getRepositoryStatistics')}").done(function(data) { $("#statImageCount").html(data.imageCount); + $("#statDeletedImageCount").html(data.deletedImageCount); + $("#statLicenceCount").html(data.licenceCount); + $("#statLicenceMappingCount").html(data.licenceMappingCount); }); } diff --git a/grails-app/views/admin/tools.gsp b/grails-app/views/admin/tools.gsp index 8b544ebd..c5dc6c7c 100644 --- a/grails-app/views/admin/tools.gsp +++ b/grails-app/views/admin/tools.gsp @@ -114,6 +114,14 @@
+ + + This will run a background task that will remove deleted images from the filesystem and the database. +
diff --git a/grails-app/views/image/_coreImageMetadataFragment.gsp b/grails-app/views/image/_coreImageMetadataFragment.gsp new file mode 100644 index 00000000..9ff1fce6 --- /dev/null +++ b/grails-app/views/image/_coreImageMetadataFragment.gsp @@ -0,0 +1,172 @@ + + +
This image is deleted.
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Data resource + + ${resourceLevel.name} + +
Image Identifier${imageInstance.imageIdentifier}
Title${imageInstance.title}
Creator
Description${imageInstance.description}
Zoom levels${imageInstance.zoomLevels}
Linear scale + + ${imageInstance.mmPerPixel} mm per pixel + + + + <not calibrated> + +
Date uploaded
Uploaded by
Date taken/created
Rights
Rights holder
Licence + + + + + + + + +
Date deleted${imageInstance.dateDeleted}
+
Parent image
+ + ${ parentImage.imageIdentifier} + + +
+
Sub images
+
    + +
  • + + ${ subimage.imageIdentifier} + + +
  • +
    +
+
+ + + + + + + + + + + + + + + + + + Admin view + +
+ \ No newline at end of file diff --git a/grails-app/views/image/details.gsp b/grails-app/views/image/details.gsp index 190aa552..f3cde10d 100644 --- a/grails-app/views/image/details.gsp +++ b/grails-app/views/image/details.gsp @@ -14,6 +14,7 @@ .tab-pane { padding-top: 20px !important; } .tabbable { font-size: 9pt; margin-top:10px; } div#main { padding-top: 0px; } + .subimages_thumbs { max-height:100px; } @@ -61,132 +62,9 @@
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Data resource - - ${resourceLevel.name} - -
Image Identifier${imageInstance.imageIdentifier}
Title${imageInstance.title}
Creator
Description${imageInstance.description}
Parent image - ${imageInstance.parent.originalFilename ?: imageInstance.parent.imageIdentifier} - -
Zoom levels${imageInstance.zoomLevels}
Linear scale - - ${imageInstance.mmPerPixel} mm per pixel - - - - <not calibrated> - -
Date uploaded
Uploaded by
Date taken/created
Rights
Rights holder
Licence - ${imageInstance.license} - - - - - - - - -
Sub-images -
    - -
  • - ${subimage.originalFilename ?: subimage.imageIdentifier} - -
  • -
    -
-
- - - - - - - - - - - - - Admin view - -
+
+ +
@@ -273,6 +151,12 @@ }); } + function refreshCoreMetadata() { + $.ajax("${grailsApplication.config.grails.serverURL}${createLink(controller:'image', action:'coreImageMetadataTableFragment', id: imageInstance.id)}").done(function(content) { + $('#imageTabs').find('.coreMetadataContainer').html(content); + }); + } + function refreshAuditTrail() { @@ -311,20 +195,6 @@ $('#viewerContainerId .document-icon').css('background-position', 'center'); - $("#btnResetLinearScale").click(function(e) { - e.preventDefault(); - imgvwr.areYouSure({ - title:"Reset calibration for this image?", - message:"Are you sure you wish to reset the linear scale for this image?", - affirmativeAction: function() { - var url = "${grailsApplication.config.grails.serverURL}${createLink(controller:'webService', action:'resetImageCalibration')}?imageId=${imageInstance.imageIdentifier}"; - $.ajax(url).done(function(result) { - window.location.reload(true); - }); - } - }); - }); - $('a[data-toggle="tab"]').on('click', function (e) { var dest = $($(this).attr("href")); @@ -356,7 +226,8 @@ $("#btnDeleteImage").click(function(e) { e.preventDefault(); var options = { - content: "Warning! This operation cannot be undone. Are you sure you wish to permanently delete this image?", + message: "Warning! This operation cannot be undone. Are you sure you wish to permanently delete this image?", + title: "Delete this image", affirmativeAction: function() { $.ajax("${grailsApplication.config.grails.serverURL}${createLink(controller:'image', action:'deleteImage', id: imageInstance.imageIdentifier)}").done(function() { window.location = "${grailsApplication.config.grails.serverURL}${createLink(controller:'search', action:'list')}"; @@ -373,11 +244,11 @@ content: { text: function(event, api) { $.ajax("${grailsApplication.config.grails.serverURL}${createLink(controller:'image', action:"imageTooltipFragment")}/" + imageId).then(function(content) { - api.set("content.text", content); - }, - function(xhr, status, error) { - api.set("content.text", status + ": " + error); - }); + api.set("content.text", content); + }, + function(xhr, status, error) { + api.set("content.text", status + ": " + error); + }); } } }); @@ -392,6 +263,14 @@ $("#tagsSection").html(html); }); } + + function calibrationCallback(data){ + refreshCoreMetadata(); + } + + function createSubImageCallback(){ + refreshCoreMetadata(); + } diff --git a/grails-app/views/webService/swagger.gsp b/grails-app/views/webService/swagger.gsp index 51a78263..4054885b 100644 --- a/grails-app/views/webService/swagger.gsp +++ b/grails-app/views/webService/swagger.gsp @@ -31,6 +31,8 @@ + +
@@ -38,7 +40,7 @@ window.onload = function() { // Begin Swagger UI call region const ui = SwaggerUIBundle({ - url: "/ws/api", + url: "${g.createLink(controller: 'ws', params: [json:true])}", dom_id: '#swagger-ui', deepLinking: true, presets: [ @@ -54,5 +56,8 @@ window.ui = ui } + diff --git a/src/integration-test/groovy/au/org/ala/images/SearchSpec.groovy b/src/integration-test/groovy/au/org/ala/images/SearchSpec.groovy new file mode 100644 index 00000000..f554b191 --- /dev/null +++ b/src/integration-test/groovy/au/org/ala/images/SearchSpec.groovy @@ -0,0 +1,84 @@ +package au.org.ala.images + +import grails.plugins.rest.client.RestBuilder +import grails.plugins.rest.client.RestResponse +import grails.testing.mixin.integration.Integration +import grails.transaction.Rollback +import groovy.json.JsonSlurper +import image.service.Application +import org.springframework.util.LinkedMultiValueMap +import org.springframework.util.MultiValueMap +import spock.lang.Ignore +import spock.lang.Shared +import spock.lang.Specification + +@Integration(applicationClass = Application.class) +@Rollback +class SearchSpec extends Specification { + + @Shared RestBuilder rest = new RestBuilder() + + def setup() {} + + def cleanup() {} + + void 'test upload'() { + + when: + def occurrenceID = "f4c13adc-2926-44c8-b2cd-fb2d62378a1a" + + //first upload an image + def testUrl = "https://upload.wikimedia.org/wikipedia/commons/thumb/f/f1/Red_kangaroo_-_melbourne_zoo.jpg/800px-Red_kangaroo_-_melbourne_zoo.jpg" + + RestResponse uploadResponse = rest.post("http://localhost:${serverPort}/ws/uploadImagesFromUrls", { + json { + [images: [[ + sourceUrl : testUrl, + occurrenceID: occurrenceID + ]]] + } + }) + def jsonUploadResponse = new JsonSlurper().parseText(uploadResponse.body) + + then: + uploadResponse.status == 200 + jsonUploadResponse.success == true + } + + + void 'test search for previous upload'(){ + + when: + + boolean hasBacklog = true + int counter = 0 + int MAX_CHECKS = 10 + + + while (hasBacklog && counter < MAX_CHECKS){ + RestResponse statsResp = rest.get("http://localhost:${serverPort}/ws/backgroundQueueStats") + def json = new JsonSlurper().parseText(statsResp.body) + if(json.queueLength > 0){ + Thread.sleep(5000) + } else { + hasBacklog = false + } + counter +=1 + } + + //search by occurrence ID + RestResponse resp = rest.post("http://localhost:${serverPort}/ws/findImagesByMetadata",{ + json { + [ + "key": "occurrenceid", + "values": ["f4c13adc-2926-44c8-b2cd-fb2d62378a1a"] + ] + } + }) + def jsonResponse = new JsonSlurper().parseText(resp.body) + + then: + resp.status == 200 + jsonResponse.count > 0 + } +} diff --git a/src/main/groovy/au/org/ala/images/DeletedImagesPurgeBackgroundTask.groovy b/src/main/groovy/au/org/ala/images/DeletedImagesPurgeBackgroundTask.groovy new file mode 100644 index 00000000..a5e0efbb --- /dev/null +++ b/src/main/groovy/au/org/ala/images/DeletedImagesPurgeBackgroundTask.groovy @@ -0,0 +1,18 @@ +package au.org.ala.images + +class DeletedImagesPurgeBackgroundTask extends BackgroundTask { + + ImageService imageService + + DeletedImagesPurgeBackgroundTask(ImageService imageService) { + this.imageService = imageService + } + + @Override + void execute() { + def images = Image.findAllByDateDeletedIsNotNull() + images.each { + imageService.deleteImagePurge(it) + } + } +}