Skip to content

Commit

Permalink
Merge branch 'release/1.1.5'
Browse files Browse the repository at this point in the history
  • Loading branch information
sbearcsiro committed Jun 15, 2021
2 parents 9a0102c + 71a816a commit 326cbb4
Show file tree
Hide file tree
Showing 12 changed files with 322 additions and 59 deletions.
8 changes: 3 additions & 5 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ plugins {
}


version "1.1.4"
version "1.1.5"

group "au.org.ala"

Expand Down Expand Up @@ -133,13 +133,11 @@ dependencies {
testRuntime "org.seleniumhq.selenium:selenium-htmlunit-driver:2.47.1"
testRuntime "net.sourceforge.htmlunit:htmlunit:2.18"
compile 'com.opencsv:opencsv:4.6'
compile 'org.apache.ant:ant:1.7.1'
compile 'org.apache.ant:ant-launcher:1.7.1'
compile group: 'org.codehaus.groovy', name: 'groovy-ant', version: '2.0.0'
compile group: 'com.googlecode.owasp-java-html-sanitizer', name: 'owasp-java-html-sanitizer', version: '20200713.1'
runtime group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.12.0'
runtime group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.12.0'
testCompile 'org.grails.plugins:embedded-postgres:1.1.2'
testCompile 'cloud.localstack:localstack-utils:0.2.5'
testCompile 'cloud.localstack:localstack-utils:0.2.11'
testCompile "com.amazonaws:aws-java-sdk:$amazonAwsSdkVersion" // full AWS SDK included in test scope for localstack config
testCompile 'com.palantir.docker.compose:docker-compose-rule-junit4:1.5.0'
}
Expand Down
17 changes: 6 additions & 11 deletions grails-app/controllers/au/org/ala/images/BatchController.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ class BatchController {
nickname = "dataresource/{dataResourceUid}",
produces = "application/json",
httpMethod = "GET",
response = Map.class,
response = List.class,
tags = ["BatchUpdate"]
)
@ApiResponses([
Expand All @@ -133,17 +133,12 @@ class BatchController {
def statusForDataResource(){

//write zip file to filesystem
def uploads = batchService.getBatchFileUploadsFor(params.dataResourceUid)
if (uploads){
//return an async response
def responses = []
uploads.each { upload ->
responses << createResponse(upload)
}
render (responses as JSON)
} else {
response.sendError(404)
def uploads = batchService.getBatchFileUploadsFor(params.dataResourceUid) ?: []
//return an async response
def responses = uploads.collect { upload ->
createResponse(upload)
}
render (responses as JSON)
}

Map createResponse(BatchFileUpload upload){
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ class ImageController {
render(text: "Image not found in storage", status: SC_NOT_FOUND, contentType: 'text/plain')
} catch (ClientAbortException e) {
// User hung up, just ignore this exception since we can't recover into a nice error response.
throw e
} catch (Exception e) {
log.error("Exception serving image", e)
cache(false)
Expand Down
133 changes: 133 additions & 0 deletions grails-app/services/au/org/ala/images/SanitiserService.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
package au.org.ala.images

import groovy.util.logging.Slf4j
import org.apache.commons.lang.StringUtils
import org.owasp.html.HtmlChangeListener
import org.owasp.html.HtmlPolicyBuilder
import org.owasp.html.HtmlStreamEventProcessor
import org.owasp.html.HtmlStreamEventReceiver
import org.owasp.html.HtmlStreamEventReceiverWrapper
import org.owasp.html.PolicyFactory
import org.owasp.html.Sanitizers

import javax.annotation.Nullable

@Slf4j
class SanitiserService {

/** Allow simple formatting, links and text within p and divs by default */
def policy = (Sanitizers.FORMATTING & Sanitizers.LINKS) & new HtmlPolicyBuilder().allowTextIn("p", "div").toFactory()


/**
* Sanitise an input string with the default policy and no warnings
* @param input
* @return
*/
String sanitise(String input) {
internalSanitise(policy, input)
}

String truncateAndSanitise(String input, int length) {
internalSanitise(policy & truncater(length), input)
}



/**
* Sanitise an image property, providiing the imager and property name as context
* so that the offending data can be logged.
*
* @param input The input string
* @param image The image the input string belongs to
* @param propertyName The image property name the input string belongs to
* @return
*/
String sanitise(String input, String imageId, String propertyName) {
internalSanitise(policy, imageId, propertyName)
}

String truncateAndSanitise(String input, String imageId, String propertyName, int length) {
def truncatingPolicy = policy & truncater(length)
internalSanitise(truncatingPolicy, input, imageId, propertyName)
}

private static PolicyFactory truncater(int length) {
new HtmlPolicyBuilder().withPostprocessor(new HtmlStreamEventProcessor() {
@Override
HtmlStreamEventReceiver wrap(HtmlStreamEventReceiver sink) {
return new TextTruncater(sink, length)
}
}).toFactory()
}

private static String internalSanitise(PolicyFactory policyFactory, String input, String imageId = '', String metadataName = '') {
policyFactory.sanitize(input, new HtmlChangeListener<Object>() {
void discardedTag(@Nullable Object context, String elementName) {
SanitiserService.log.warn("Dropping element $elementName in $imageId.$metadataName")
}
void discardedAttributes(@Nullable Object context, String tagName, String... attributeNames) {
SanitiserService.log.warn("Dropping attributes $attributeNames from $tagName in $imageId.$metadataName")
}
}, null)
}

/**
* Allows up to length characters of text from the underlying HTML to be sent to the output, truncating the text if
* it's longer than length and omitting any subsequent tags.
*/
static class TextTruncater extends HtmlStreamEventReceiverWrapper {

private final int length
private int spent = 0
private int closeNextTag = 0
private int level = 0

TextTruncater(HtmlStreamEventReceiver underlying, int length) {
super(underlying)
this.length = length
}

@Override
void text(String text) {
int newLength = text.length()
if (spent > length) {
// already spent our text budget, do nothing
} else if (spent + newLength > length) {
if (length - spent < 4) {
this.underlying.text('...')
} else {
this.underlying.text(StringUtils.abbreviate(text, length - spent))
}
closeNextTag = level
spent += newLength
} else {
this.underlying.text(text)
spent += newLength
}
}

@Override
void openTag(String elementName, List<String> attrs) {
level++
if (spent > length) {
// already spent our length budget, omit new tags
} else {
super.openTag(elementName, attrs)
}

}

@Override
void closeTag(String elementName) {
if (spent <= length) {
super.closeTag(elementName)
} else if (closeNextTag == level) {
// need to close this tag as it was the last open tag before the length budget was exhausted
closeNextTag--
super.closeTag(elementName)
}
level--
}
}
}
82 changes: 62 additions & 20 deletions grails-app/taglib/au/org/ala/images/ImagesTagLib.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@ class ImagesTagLib {

static namespace = 'img'

static returnObjectForTags = ['sanitiseString']

def imageService
def groovyPageLocator
def authService
def searchCriteriaService
def collectoryService
def sanitiserService

/**
* @attr title
Expand Down Expand Up @@ -108,6 +111,8 @@ class ImagesTagLib {

def imageSearchResult = { attrs, body ->

def mb = new MarkupBuilder(out)

if (attrs.image) {
def creator = ''
if (attrs.image.creator && attrs.image.creator != ElasticSearchService.NOT_SUPPLIED){
Expand All @@ -116,22 +121,25 @@ class ImagesTagLib {

if(attrs.image.dataResourceUid){
def metadata = collectoryService.getResourceLevelMetadata(attrs.image.dataResourceUid)
out << """<div class="thumb-caption caption-detail ${attrs.css?:''}">"""
out << "<span class='resource-name'>${metadata.name?:''}</span>"
if (metadata.name && (attrs.image.title || creator)){
out << ' - '
}
mb.div(class: ['thumb-caption', 'caption-detail', attrs.css ?: ''].findAll().join(' ')) {
mb.span(class: 'resource-name') {
mkp.yield(metadata.name ?: '')
}
if (metadata.name && (attrs.image.title || creator)) {
mkp.yield(' - ')
}

def text = "${attrs.image.title? attrs.image.title: ''} ${creator}"
text = StringUtils.abbreviate(text, 100)
out << "<span>${text}</span>"
out << '</div>'
def text = "${attrs.image.title? attrs.image.title: ''} ${creator}"
mb.span {
mkp.yieldUnescaped(sanitiserService.truncateAndSanitise(text, attrs.image.imageIdentifier, 'title+creator', 100))
}
}
} else {
if (attrs.image.dataResourceUid || attrs.image.title || creator){
out << """<div class="thumb-caption caption-detail ${attrs.css?:''}">"""
def output = "${attrs.image.dataResourceUid ? attrs.image.dataResourceUid: ''} ${attrs.image.title ? attrs.image.title :''} ${creator}"
out << StringUtils.abbreviate(output, 100)
out << '</div>'
if (attrs.image.dataResourceUid || attrs.image.title || creator) {
mb.div(class: ['thumb-caption', 'caption-detail', attrs.css ?: ''].findAll().join(' ')) {
def output = "${attrs.image.dataResourceUid ? attrs.image.dataResourceUid: ''} ${attrs.image.title ? attrs.image.title :''} ${creator}"
mkp.yieldUnescaped(sanitiserService.truncateAndSanitise(output, attrs.image.imageIdentifier, 'drUid+title+creator', 100))
}
}
}
}
Expand All @@ -145,6 +153,10 @@ class ImagesTagLib {
} else {
valueToRender = message(code: attrs.dataResourceUid, default: attrs.dataResourceUid)
}
def mb = new MarkupBuilder(out)
mb.span(class: 'resource-name') {
mkp.yield(valueToRender)
}
out << "<span class='resource-name'>${valueToRender}</span>"
}

Expand Down Expand Up @@ -253,7 +265,7 @@ class ImagesTagLib {
}

def currentUserId = authService.getUserId()
out << (displayName ?: userId ?: '&lt;Unknown&gt;')
out << (sanitiserService.sanitise(displayName ?: userId ?: '&lt;Unknown&gt;'))

if(currentUserId && currentUserId == userId){
out << " (thats you!)"
Expand All @@ -267,11 +279,41 @@ class ImagesTagLib {
}
}

def sanitiseString = { attrs, body ->
return sanitiseInternal(attrs)
}

def sanitise = { attrs, body ->
out << sanitiseInternal(attrs)
}

private String sanitiseInternal(attrs) {
def value = attrs.value
def image = attrs.image
def key = attrs.key
def length = attrs.length
def result
if (image && key) {
if (length) {
result = sanitiserService.truncateAndSanitise(value, length, image, key)
} else {
result = sanitiserService.sanitise(value, image, key)
}
} else {
if (length) {
result = sanitiserService.truncateAndSanitise(value, length)
} else {
result = sanitiserService.sanitise(value)
}
}
return result
}

def imageMetadata = { attrs, body ->
if (attrs.image[attrs.field]){
out << attrs.image[attrs.field]
} else if(attrs.resource && attrs.resource.imageMetadata && attrs.resource.imageMetadata[attrs.field]){
out << attrs.resource.imageMetadata[attrs.field] + "<small> (resource level metadata) </small>"
if (attrs.image[attrs.field]) {
out << sanitiserService.sanitise(attrs.image[attrs.field])
} else if (attrs.resource && attrs.resource.imageMetadata && attrs.resource.imageMetadata[attrs.field]) {
out << sanitiserService.sanitise(attrs.resource.imageMetadata[attrs.field]) + "<small> (resource level metadata) </small>"
}
}

Expand Down Expand Up @@ -317,7 +359,7 @@ class ImagesTagLib {
def renderMetaDataValue = { attrs, body ->
ImageMetaDataItem md = attrs.metaDataItem as ImageMetaDataItem
if (md) {
out << new MetaDataValueFormatRules(grailsApplication).formatValue(md)
out << sanitiserService.sanitise(new MetaDataValueFormatRules(grailsApplication).formatValue(md))
}
}
}
16 changes: 8 additions & 8 deletions grails-app/views/image/_coreImageMetadataFragment.gsp
Original file line number Diff line number Diff line change
Expand Up @@ -24,35 +24,35 @@
</tr>
<tr>
<td class="property-name"><g:message code="core.image.metadata.title" /></td>
<td class="property-value">${imageInstance.title}</td>
<td class="property-value"><img:sanitise value="${imageInstance.title}" image="${imageInstance.imageIdentifier}" key="title"/></td>
</tr>
<tr>
<td class="property-name"><g:message code="core.image.metadata.creator" /></td>
<td class="property-value"><img:imageMetadata image="${imageInstance}" resource="${resourceLevel}" field="creator"/></td>
<td class="property-value"><img:imageMetadata image="${imageInstance.imageIdentifier}" resource="${resourceLevel}" field="creator"/></td>
</tr>
<tr>
<td class="property-name"><g:message code="core.image.metadata.created" /></td>
<td class="property-value">${imageInstance.created}</td>
<td class="property-value"><img:sanitise value="${imageInstance.created}" image="${imageInstance.imageIdentifier}" key="created"/></td>
</tr>
<tr>
<td class="property-name"><g:message code="core.image.metadata.description" /></td>
<td class="property-value">${imageInstance.description}</td>
<td class="property-value"><img:sanitise value="${imageInstance.description}" image="${imageInstance.imageIdentifier}" key="description"/></td>
</tr>
<tr>
<td class="property-name"><g:message code="core.image.metadata.publisher" /></td>
<td class="property-value">${imageInstance.publisher}</td>
<td class="property-value"><img:sanitise value="${imageInstance.publisher}" image="${imageInstance.imageIdentifier}" key="publisher"/></td>
</tr>
<tr>
<td class="property-name"><g:message code="core.image.metadata.references" /></td>
<td class="property-value">${imageInstance.references}</td>
<td class="property-value"><img:sanitise value="${imageInstance.references}" image="${imageInstance.imageIdentifier}" key="references"/></td>
</tr>
<tr>
<td class="property-name"><g:message code="core.image.metadata.audience" /></td>
<td class="property-value">${imageInstance.audience}</td>
<td class="property-value"><img:sanitise value="${imageInstance.audience}" image="${imageInstance.imageIdentifier}" key="audience"/></td>
</tr>
<tr>
<td class="property-name"><g:message code="core.image.metadata.source" /></td>
<td class="property-value">${imageInstance.source}</td>
<td class="property-value"><img:sanitise value="${imageInstance.source}" image="${imageInstance.imageIdentifier}" key="source"/></td>
</tr>

<g:if test="${isAdminView}">
Expand Down
Loading

0 comments on commit 326cbb4

Please sign in to comment.