From 4d1ae2d0bef29ea407aba906f79f703dd920cf13 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 26 Oct 2017 10:38:38 +0200 Subject: [PATCH 001/104] Avoid out of memory exception, when uploading huge files --- data-showcase/build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data-showcase/build.gradle b/data-showcase/build.gradle index a190c84..626d9df 100644 --- a/data-showcase/build.gradle +++ b/data-showcase/build.gradle @@ -147,3 +147,6 @@ publishing { publishToMavenLocal.dependsOn 'executableWar' publish.dependsOn 'executableWar' +bootRun { + jvmArgs = ['-Xmx4096m'] +} From e91c004d69b91c3412142834ca19fdd68be6ca01 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 26 Oct 2017 10:39:34 +0200 Subject: [PATCH 002/104] Remove .save(flush: true) from domain classes --- .../nl/thehyve/datashowcase/Concept.groovy | 3 +-- .../nl/thehyve/datashowcase/Project.groovy | 3 +-- .../datashowcase/DataImportService.groovy | 27 ++++++++++++++++--- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Concept.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Concept.groovy index b78f4a5..972d620 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Concept.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Concept.groovy @@ -51,8 +51,7 @@ class Concept { @BindUsing({ obj, source -> def keywords = source['keywords'].collect { if (it) { - Keyword existingKeyword = Keyword.findByKeyword(it) - existingKeyword ?: new Keyword(keyword: it).save(flush: true) + Keyword.findByKeyword(it) } } keywords diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Project.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Project.groovy index 53ca3b5..c856859 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Project.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Project.groovy @@ -27,8 +27,7 @@ class Project { * The line of research the project belongs to. */ @BindUsing({ obj, source -> - LineOfResearch existingResearchLine = LineOfResearch.findByName(source['lineOfResearch']) - existingResearchLine ?: new LineOfResearch(name: source['lineOfResearch']).save(flush: true) + LineOfResearch.findByName(source['lineOfResearch']) }) LineOfResearch lineOfResearch diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy index 80cf2af..05c17c7 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy @@ -6,6 +6,7 @@ package nl.thehyve.datashowcase +import com.sun.org.apache.xpath.internal.compiler.Keywords import grails.gorm.transactions.Transactional import grails.validation.ValidationException import nl.thehyve.datashowcase.exception.InvalidDataException @@ -26,9 +27,19 @@ class DataImportService { def upload(JSONObject json) { try { // clear database + log.info('Clearing database...') dataService.clearDatabase() - // save concepts and related keywords + // save keywords + def keywords = json.concepts?.keywords?.flatten().unique().collect { + if(it?.trim()) new Keyword(keyword: it) + } + keywords.removeAll([null]) + validate(keywords) + log.info('Saving keywords...') + keywords*.save(flush: true, failOnError: true) + + // save concepts def concepts = json.concepts?.collect { new Concept(it) } validate(concepts) log.info('Saving concepts...') @@ -43,10 +54,18 @@ class DataImportService { log.info('Saving tree nodes...') tree_nodes*.save(flush: true, failOnError: true) - // save projects and related research_lines + //save research_lines + def linesOfResearch = json.projects?.lineOfResearch.unique().collect { + if (it) new LineOfResearch(name: it) + } + validate(linesOfResearch) + log.info("Saving lines of research...") + linesOfResearch*.save(flush: true, failOnError: true) + + // save projects def projects = json.projects?.collect { new Project(it) } validate(projects) - log.info('Saving projects and research lines...') + log.info("Saving projects...") projects*.save(flush: true, failOnError: true) // save items, related summaries and values @@ -62,7 +81,7 @@ class DataImportService { item } validate(items) - log.info('Saving items, summaries, values...') + log.info("Saving $items.size() items, summaries, values...") items*.save(flush: true, failOnError: true) } catch (ValidationException e) { From 85d5455b256b9e9cdc897dd00554a6b5f9de0647 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 26 Oct 2017 15:23:47 +0200 Subject: [PATCH 003/104] First attempt to speed up the data upload --- .../nl/thehyve/datashowcase/Item.groovy | 4 +++ .../nl/thehyve/datashowcase/Summary.groovy | 1 + .../datashowcase/DataImportService.groovy | 29 ++++++++++++++----- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy index a21a0d3..6bf58e1 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy @@ -7,6 +7,9 @@ package nl.thehyve.datashowcase import grails.databinding.BindUsing +import javax.persistence.OneToOne +import javax.persistence.CascadeType + /** * An item represents a variable in a study or survey. @@ -59,6 +62,7 @@ class Item { } summary }) + @OneToOne(cascade=CascadeType.PERSIST) Summary summary String getLabel() { diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy index 216da22..ddc1df8 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy @@ -59,6 +59,7 @@ class Summary { static mapping = { version false + values cascade: 'all-delete-orphan' } static constraints = { diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy index 05c17c7..da7f034 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy @@ -6,7 +6,6 @@ package nl.thehyve.datashowcase -import com.sun.org.apache.xpath.internal.compiler.Keywords import grails.gorm.transactions.Transactional import grails.validation.ValidationException import nl.thehyve.datashowcase.exception.InvalidDataException @@ -14,6 +13,9 @@ import org.grails.datastore.gorm.GormEntity import org.grails.web.json.JSONArray import org.grails.web.json.JSONObject import org.springframework.beans.factory.annotation.Autowired +import org.hibernate.StatelessSession +import org.hibernate.Transaction +import org.hibernate.SessionFactory @Transactional class DataImportService { @@ -24,12 +26,16 @@ class DataImportService { @Autowired DataService dataService + SessionFactory sessionFactory + def upload(JSONObject json) { try { // clear database log.info('Clearing database...') dataService.clearDatabase() + StatelessSession session = sessionFactory.openStatelessSession() + Transaction tx = session.beginTransaction() // save keywords def keywords = json.concepts?.keywords?.flatten().unique().collect { if(it?.trim()) new Keyword(keyword: it) @@ -37,13 +43,15 @@ class DataImportService { keywords.removeAll([null]) validate(keywords) log.info('Saving keywords...') - keywords*.save(flush: true, failOnError: true) + keywords.each{ session.insert(it)} + //keywords*.save(flush: true, failOnError: true) // save concepts def concepts = json.concepts?.collect { new Concept(it) } validate(concepts) log.info('Saving concepts...') - concepts*.save(flush: true, failOnError: true) + concepts.each{ session.insert(it) } + //concepts*.save(flush: true, failOnError: true) // save tree_nodes def tree_nodes = json.tree_nodes?.collect { JSONObject it -> @@ -52,7 +60,8 @@ class DataImportService { } validate(tree_nodes) log.info('Saving tree nodes...') - tree_nodes*.save(flush: true, failOnError: true) + tree_nodes.each{ session.insert(it) } + //tree_nodes*.save(flush: true, failOnError: true) //save research_lines def linesOfResearch = json.projects?.lineOfResearch.unique().collect { @@ -60,13 +69,15 @@ class DataImportService { } validate(linesOfResearch) log.info("Saving lines of research...") - linesOfResearch*.save(flush: true, failOnError: true) + linesOfResearch.each{ session.insert(it) } + //linesOfResearch*.save(flush: true, failOnError: true) // save projects def projects = json.projects?.collect { new Project(it) } validate(projects) log.info("Saving projects...") - projects*.save(flush: true, failOnError: true) + projects.each{ session.insert(it) } + //projects*.save(flush: true, failOnError: true) // save items, related summaries and values if (!dataShowcaseEnvironment.internalInstance && !allItemsArePublic((JSONArray) json.items)) { @@ -82,7 +93,11 @@ class DataImportService { } validate(items) log.info("Saving $items.size() items, summaries, values...") - items*.save(flush: true, failOnError: true) + items.each{ session.insert(it) } + //items*.save(flush: true, failOnError: true) + + tx.commit() + session.close() } catch (ValidationException e) { log.error e.message From 9c9b2edd942fcbf58099a6c0e652e0396e3c9a5b Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 27 Oct 2017 15:29:45 +0200 Subject: [PATCH 004/104] Turn off Hibernate query logging in production. --- data-showcase/grails-app/conf/application.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data-showcase/grails-app/conf/application.yml b/data-showcase/grails-app/conf/application.yml index f9a6a71..4676864 100644 --- a/data-showcase/grails-app/conf/application.yml +++ b/data-showcase/grails-app/conf/application.yml @@ -164,6 +164,8 @@ environments: username: sa password: '' production: + hibernate: + show_sql: false dataSource: dbCreate: update url: jdbc:postgresql://localhost:5432/data_showcase From 5f077468f2162657f14962ee80245d4bf39e3371 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 27 Oct 2017 15:33:26 +0200 Subject: [PATCH 005/104] Add joins and batch size to domain classes, refactor data model. --- .../nl/thehyve/datashowcase/Concept.groovy | 21 +++--- .../nl/thehyve/datashowcase/Item.groovy | 35 +++------- .../nl/thehyve/datashowcase/Project.groovy | 7 +- .../nl/thehyve/datashowcase/Summary.groovy | 9 ++- .../nl/thehyve/datashowcase/TreeNode.groovy | 9 ++- .../thehyve/datashowcase/DataService.groovy | 21 +----- .../thehyve/datashowcase/ItemService.groovy | 70 ++++++++++--------- .../thehyve/datashowcase/TreeService.groovy | 60 ++++++++++++++-- .../ConceptRepresentation.groovy | 5 ++ .../representation/ItemRepresentation.groovy | 5 -- 10 files changed, 130 insertions(+), 112 deletions(-) diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Concept.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Concept.groovy index 972d620..3797ede 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Concept.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Concept.groovy @@ -6,7 +6,6 @@ package nl.thehyve.datashowcase -import grails.databinding.BindUsing import nl.thehyve.datashowcase.enumeration.VariableType /** @@ -45,18 +44,12 @@ class Concept { */ VariableType variableType - /** - * Associated key words. - */ - @BindUsing({ obj, source -> - def keywords = source['keywords'].collect { - if (it) { - Keyword.findByKeyword(it) - } - } - keywords - }) - List keywords + static hasMany = [ + /** + * Associated key words. + */ + keywords: Keyword + ] @Override String toString() { @@ -65,6 +58,8 @@ class Concept { static mapping = { version false + + keywords batchSize: 1000 } static constraints = { diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy index a21a0d3..4cb13af 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy @@ -6,8 +6,6 @@ package nl.thehyve.datashowcase -import grails.databinding.BindUsing - /** * An item represents a variable in a study or survey. * Besides links to the concept and project it belongs to, @@ -35,10 +33,6 @@ class Item { /** * The concept that the item represents in a project. */ - @BindUsing({ obj, source -> - String conceptCode = source['concept'] - Concept.findByConceptCode(conceptCode) - }) Concept concept /** @@ -46,20 +40,12 @@ class Item { */ static belongsTo = [project: Project] - /** - * Summary data for the variable: aggregate values and value frequencies. - */ - @BindUsing({ obj, source -> - def s = source['summary'] - def summary = s instanceof Summary ? s : new Summary(s) - if (s.values) { - s.values.each { - summary.addToValues(it) - } - } - summary - }) - Summary summary + static hasOne = [ + /** + * Summary data for the variable: aggregate values and value frequencies. + */ + summary: Summary + ] String getLabel() { concept.label @@ -77,21 +63,22 @@ class Item { concept.labelNlLong } - List getKeywords() { - concept.keywords - } - String getType() { concept.variableType } static mapping = { version false + + concept fetch: 'join' + project fetch: 'join' + summary fetch: 'join' } static constraints = { name unique: true itemPath maxSize: 700 + summary nullable: true } } diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Project.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Project.groovy index c856859..f5fd8ae 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Project.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Project.groovy @@ -6,8 +6,6 @@ package nl.thehyve.datashowcase -import grails.databinding.BindUsing - /** * Entity class to store projects (surveys). */ @@ -26,9 +24,6 @@ class Project { /** * The line of research the project belongs to. */ - @BindUsing({ obj, source -> - LineOfResearch.findByName(source['lineOfResearch']) - }) LineOfResearch lineOfResearch @Override @@ -43,6 +38,8 @@ class Project { static mapping = { version false + + lineOfResearch fetch: 'join' } static constraints = { diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy index 216da22..5099f82 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy @@ -48,7 +48,12 @@ class Summary { */ Double stdDevValue - static belongsTo = [item: Item] + static belongsTo = [ + /** + * The item this summary belongs to. + */ + item: Item + ] static hasMany = [ /** @@ -59,6 +64,8 @@ class Summary { static mapping = { version false + + values batchSize: 1000 } static constraints = { diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/TreeNode.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/TreeNode.groovy index 9e650a4..49cafa6 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/TreeNode.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/TreeNode.groovy @@ -6,7 +6,6 @@ package nl.thehyve.datashowcase -import grails.databinding.BindUsing import nl.thehyve.datashowcase.enumeration.NodeType import nl.thehyve.datashowcase.exception.InvalidDataException @@ -38,10 +37,6 @@ class TreeNode { * The concept the node is associated with, if it is a concept node. * A concept node always is a leaf node. */ - @BindUsing({ obj, source -> - String conceptCode = source['concept'] - Concept.findByConceptCode(conceptCode) - }) Concept concept /** @@ -84,6 +79,10 @@ class TreeNode { static mapping = { version false + + concept fetch: 'join' + parent fetch: 'join' + children batchSize: 1000 } static constraints = { diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataService.groovy index 9fcc495..1852550 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataService.groovy @@ -7,7 +7,6 @@ package nl.thehyve.datashowcase import grails.gorm.transactions.Transactional -import nl.thehyve.datashowcase.representation.ItemRepresentation import org.springframework.beans.factory.annotation.Autowired @Transactional @@ -18,8 +17,8 @@ class DataService { def clearDatabase() { Value.executeUpdate('delete from Value') - Item.executeUpdate('delete from Item') Summary.executeUpdate('delete from Summary') + Item.executeUpdate('delete from Item') TreeNode.executeUpdate('delete from TreeNode') Concept.executeUpdate('delete from Concept') Keyword.executeUpdate('delete from Keyword') @@ -27,22 +26,4 @@ class DataService { LineOfResearch.executeUpdate('delete from LineOfResearch') } - /** - * Upload concepts, tree nodes and items. - * - * @param concepts The concepts to save. - * @param nodes The nodes to save. - * @param items The items to save. - */ - def uploadData(List concepts, List nodes, List items) { - if (!dataShowcaseEnvironment.internalInstance) { - // TODO - } - // FIXME - // - check if no public data is being uploaded. - // - truncate tables - // - upload concepts - // - upload items and associated data - // - upload tree nodes - } } diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index 84b329a..c6f8625 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -7,12 +7,12 @@ package nl.thehyve.datashowcase import grails.gorm.transactions.Transactional +import grails.plugin.cache.CacheEvict import grails.plugin.cache.Cacheable -import nl.thehyve.datashowcase.enumeration.NodeType import nl.thehyve.datashowcase.exception.ResourceNotFoundException import nl.thehyve.datashowcase.mapping.ItemMapper import nl.thehyve.datashowcase.representation.ItemRepresentation -import nl.thehyve.datashowcase.representation.TreeNodeRepresentation +import org.grails.core.util.StopWatch import org.springframework.beans.factory.annotation.Autowired @Transactional @@ -27,48 +27,58 @@ class ItemService { @Transactional(readOnly = true) List getItems() { if (dataShowcaseEnvironment.internalInstance) { - Item.findAll().collect({ + def stopWatch = new StopWatch('Fetch items') + stopWatch.start('Retrieve from database') + def items = Item.findAll() + stopWatch.stop() + stopWatch.start('Map to representations') + def result = items.collect { itemMapper.map(it) - }) + } + stopWatch.stop() + log.info "Items fetched.\n${stopWatch.prettyPrint()}" + result } else { - Item.findAllByPublicItem(true).collect({ + Item.findAllByPublicItem(true).collect { itemMapper.map(it) - }) + } } } @Transactional(readOnly = true) - @Cacheable('itemcounts') - Long countItemsForNode(String path) { + @Cacheable('itemCountPerNode') + Map getItemCountPerNode() { if (dataShowcaseEnvironment.internalInstance) { - Item.executeQuery( - """ select count(distinct i) from Item i, TreeNode n + def itemCountMap = Item.executeQuery( + """ select n.path as path, count(distinct i) as itemCount + from Item i, TreeNode n join i.concept c where n.concept = c - and n.path = :path - """, - [path: path] - )[0] + group by n.path + """ + ) as List + itemCountMap.collectEntries { + [(it[0]): it[1] as Long] + } } else { - Item.executeQuery( - """ select count(distinct i) from Item i, TreeNode n + def itemCountMap = Item.executeQuery( + """ select n.path as path, count(distinct i) as itemCount + from Item i, TreeNode n join i.concept c where n.concept = c - and n.path = :path - and i.publicItem = true - """, - [path: path] - )[0] + and i.publicItem = true + group by n.path + """ + ) as List + itemCountMap.collectEntries { + [(it[0]): it[1] as Long] + } } } - @Transactional(readOnly = true) - Long countItemsForNode(TreeNodeRepresentation treeNode) { - if (treeNode.nodeType == NodeType.Domain) { - 0 - } else { - countItemsForNode(treeNode.path) - } + @CacheEvict(value = 'itemCountPerNode', allEntries = true) + void clearItemCountsCache() { + log.info "Clear items counts cache." } @Transactional(readOnly = true) @@ -85,8 +95,4 @@ class ItemService { itemMapper.map(item) } - def saveItems(List items) { - - } - } diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy index 284e2cc..1d6f098 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy @@ -8,7 +8,9 @@ package nl.thehyve.datashowcase import grails.gorm.transactions.Transactional import groovy.transform.CompileStatic +import nl.thehyve.datashowcase.representation.ConceptRepresentation import nl.thehyve.datashowcase.representation.TreeNodeRepresentation +import org.grails.core.util.StopWatch import org.modelmapper.ModelMapper import org.springframework.beans.factory.annotation.Autowired @@ -28,26 +30,70 @@ class TreeService { * @param node the node (tree) to enrich. */ @CompileStatic - private void enrichWithItemCounts(TreeNodeRepresentation node) { - long itemCount = itemService.countItemsForNode(node) + private static void enrichWithItemCounts(TreeNodeRepresentation node, final Map itemCountMap) { + if (node == null) { + return + } + long itemCount = itemCountMap[node.path] ?: 0 node.itemCount = itemCount node.accumulativeItemCount = itemCount + (long)(node.children?.sum { TreeNodeRepresentation child -> - enrichWithItemCounts(child) + enrichWithItemCounts(child, itemCountMap) child.accumulativeItemCount } ?: 0) } + @CompileStatic + static ConceptRepresentation map(Concept concept) { + if (concept == null) { + return null + } + new ConceptRepresentation( + conceptCode: concept.conceptCode, + label: concept.label, + labelLong: concept.labelLong, + labelNl: concept.labelNl, + labelNlLong: concept.labelNlLong, + variableType: concept.variableType + ) + } + + @CompileStatic + static TreeNodeRepresentation map(TreeNode node) { + def result = new TreeNodeRepresentation( + nodeType: node.nodeType, + label: node.label, + concept: map(node.concept), + path: node.path, + ) + result.children = node.children.collect { + map(it) + } + result + } + /** * Fetches all top nodes of the tree. * @return the list of top nodes with child nodes embedded. */ List getNodes() { - TreeNode.findAllByParentIsNull().collect { domain -> - def node = modelMapper.map(domain, TreeNodeRepresentation.class) - enrichWithItemCounts(node) - node + def stopWatch = new StopWatch('Fetch tree nodes') + stopWatch.start('Retrieve from database') + def nodes = TreeNode.findAllByParentIsNull() + stopWatch.stop() + stopWatch.start('Map to representations') + def result = nodes.collect { domain -> + map(domain) + } + stopWatch.stop() + stopWatch.start('Enrich with counts') + def itemCountMap = itemService.itemCountPerNode + result.each { node -> + enrichWithItemCounts(node, itemCountMap) } + stopWatch.stop() + log.info "Fetched tree nodes.\n${stopWatch.prettyPrint()}" + result } } diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ConceptRepresentation.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ConceptRepresentation.groovy index db92ce1..7bc07c1 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ConceptRepresentation.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ConceptRepresentation.groovy @@ -12,6 +12,11 @@ import nl.thehyve.datashowcase.enumeration.VariableType @CompileStatic class ConceptRepresentation { + /** + * The unique code of the concept in Transmart. + */ + String conceptCode + /** * The short name of the variable. */ diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy index 61219df..bf7dbf1 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy @@ -42,11 +42,6 @@ abstract class ItemRepresentation { */ String labelNlLong - /** - * Associated key words. - */ - List keywords - /** * The full path of the item that can be used in tranSMART */ From 0bfc85ef0952989177341ed2f682051f9ab0e910 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 27 Oct 2017 15:34:31 +0200 Subject: [PATCH 006/104] Change implementation of item serialisation. --- .../thehyve/datashowcase/ItemController.groovy | 5 ++++- .../datashowcase/mapping/ItemMapper.groovy | 16 +++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index 8b14894..fe67b73 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -6,6 +6,7 @@ package nl.thehyve.datashowcase +import com.fasterxml.jackson.databind.ObjectMapper import org.springframework.beans.factory.annotation.Autowired class ItemController { @@ -21,7 +22,9 @@ class ItemController { * @return the list of items as JSON. */ def index() { - respond items: itemService.items + response.status = 200 + response.contentType = 'application/json' + new ObjectMapper().writeValue(response.outputStream, [items: itemService.items]) } /** diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/mapping/ItemMapper.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/mapping/ItemMapper.groovy index c37f0bc..ab4ffaa 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/mapping/ItemMapper.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/mapping/ItemMapper.groovy @@ -26,12 +26,14 @@ class ItemMapper { @Autowired Environment dataShowcaseEnvironment - ItemRepresentation map(Item item) { - if (dataShowcaseEnvironment.internalInstance) { - modelMapper.map(item, InternalItemRepresentation.class) - } else { - modelMapper.map(item, PublicItemRepresentation.class) - } - } + @Lazy + Closure map = (dataShowcaseEnvironment.internalInstance) ? + { Item item -> + modelMapper.map(item, InternalItemRepresentation.class) + } as Closure + : + { Item item -> + modelMapper.map(item, PublicItemRepresentation.class) + } as Closure } From 0a7713eed1aab4536e161baca8d254ef0ea24e4d Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 27 Oct 2017 15:34:59 +0200 Subject: [PATCH 007/104] Add concepts endpoint. --- .../datashowcase/ConceptController.groovy | 22 ++++++++++++++++ .../thehyve/datashowcase/UrlMappings.groovy | 1 + .../datashowcase/ConceptService.groovy | 26 +++++++++++++++++++ 3 files changed, 49 insertions(+) create mode 100644 data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ConceptController.groovy create mode 100644 data-showcase/grails-app/services/nl/thehyve/datashowcase/ConceptService.groovy diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ConceptController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ConceptController.groovy new file mode 100644 index 0000000..db57f31 --- /dev/null +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ConceptController.groovy @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +package nl.thehyve.datashowcase + +import org.springframework.beans.factory.annotation.Autowired + +class ConceptController { + + static responseFormats = ['json'] + + @Autowired + ConceptService conceptService + + def index() { + respond concepts: conceptService.concepts + } + +} diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy index 3ac28f2..f543b71 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy @@ -18,6 +18,7 @@ class UrlMappings { "/api/items"(resources: 'item', includes: ['index', 'show']) "/api/keywords"(controller: 'keyword', includes: ['index']) "/api/projects"(controller: 'project', includes: ['index']) + "/api/concepts"(controller: 'concept', includes: ['index']) "/api/lines_of_research"(controller: 'researchLine', includes: ['index']) "/api/tree_nodes"(controller: 'tree', includes: ['index']) "/api/file/logo/${type}"(controller: 'file', includes: ['getLogo']) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ConceptService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ConceptService.groovy new file mode 100644 index 0000000..33497ab --- /dev/null +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ConceptService.groovy @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +package nl.thehyve.datashowcase + +import grails.gorm.transactions.Transactional +import nl.thehyve.datashowcase.representation.ConceptRepresentation +import org.modelmapper.ModelMapper +import org.springframework.beans.factory.annotation.Autowired + +@Transactional(readOnly = true) +class ConceptService { + + @Autowired + ModelMapper modelMapper + + List getConcepts() { + Concept.findAll().collect({ Concept concept -> + modelMapper.map(concept, ConceptRepresentation) + }) + } + +} From 74f2e31559cfa60bb40f336d72bddaec1ca13d42 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 27 Oct 2017 15:35:37 +0200 Subject: [PATCH 008/104] Fix front end to use concepts call for keywords. --- .../checkbox-filter.component.ts | 9 +- .../src/app/constants/endpoints.constants.ts | 1 + .../src/app/models/CheckboxOption.ts | 10 ++ .../user-interface/src/app/models/concept.ts | 6 + .../user-interface/src/app/models/item.ts | 1 - .../src/app/services/data.service.ts | 134 ++++++++++++++---- .../src/app/services/resource.service.ts | 17 ++- .../app/tree-nodes/tree-nodes.component.ts | 4 +- 8 files changed, 143 insertions(+), 39 deletions(-) create mode 100644 data-showcase/src/main/user-interface/src/app/models/CheckboxOption.ts diff --git a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.ts b/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.ts index 7a33399..3ab4e4c 100644 --- a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.ts +++ b/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.ts @@ -7,6 +7,7 @@ import {Component, OnInit} from '@angular/core'; import {DataService} from "../services/data.service"; import {Item} from "../models/item"; +import { CheckboxOption } from '../models/CheckboxOption'; @Component({ selector: 'app-checkbox-filter', @@ -16,9 +17,9 @@ import {Item} from "../models/item"; export class CheckboxFilterComponent implements OnInit { items: Item[]; - keywords: string[] = []; - projects: string[] = []; - researchLines: string[] = []; + keywords: CheckboxOption[] = []; + projects: CheckboxOption[] = []; + researchLines: CheckboxOption[] = []; selectedKeywords: string[] = []; selectedProjects: string[] = []; selectedResearchLines: string[] = []; @@ -27,7 +28,7 @@ export class CheckboxFilterComponent implements OnInit { this.items = this.dataService.getItems(); this.keywords = this.dataService.getKeywords(); this.projects = this.dataService.getProjects(); - this.researchLines = this.dataService.getReasearchLines(); + this.researchLines = this.dataService.getResearchLines(); } ngOnInit() { diff --git a/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts b/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts index c90b3ed..726060b 100644 --- a/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts +++ b/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts @@ -7,5 +7,6 @@ export const PATH_TREE_NODES = "/api/tree_nodes"; export const PATH_ITEMS = "/api/items"; export const PATH_PROJECTS = "/api/projects"; +export const PATH_CONCEPTS = "/api/concepts"; export const PATH_LOGOS = "/api/file/logo"; export const PATH_ENVIRONMENT = "/api/environment"; diff --git a/data-showcase/src/main/user-interface/src/app/models/CheckboxOption.ts b/data-showcase/src/main/user-interface/src/app/models/CheckboxOption.ts new file mode 100644 index 0000000..430f4a7 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/models/CheckboxOption.ts @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +export class CheckboxOption { + label: string; + value: string; +} diff --git a/data-showcase/src/main/user-interface/src/app/models/concept.ts b/data-showcase/src/main/user-interface/src/app/models/concept.ts index 1677352..e19db4b 100644 --- a/data-showcase/src/main/user-interface/src/app/models/concept.ts +++ b/data-showcase/src/main/user-interface/src/app/models/concept.ts @@ -7,5 +7,11 @@ type VariableType = "Numerical"| "Categorical" | "Text" | "None"; export class Concept { + conceptCode: string; + label: string; + labelLong: string; + labelNl: string; + labelNlLong: string; variableType: VariableType; + keywords: string[]; } diff --git a/data-showcase/src/main/user-interface/src/app/models/item.ts b/data-showcase/src/main/user-interface/src/app/models/item.ts index aa027ee..b5f656e 100644 --- a/data-showcase/src/main/user-interface/src/app/models/item.ts +++ b/data-showcase/src/main/user-interface/src/app/models/item.ts @@ -12,7 +12,6 @@ export class Item { itemPath: string; type: string; project: string; - keywords: string[]; researchLine: string; concept: string; summary: ItemSummary; diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index cac3670..8365118 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -13,6 +13,8 @@ import {Project} from "../models/project"; import {Subject} from "rxjs/Subject"; import {BehaviorSubject} from "rxjs/BehaviorSubject"; import {Environment} from "../models/environment"; +import { Concept } from '../models/concept'; +import { CheckboxOption } from '../models/CheckboxOption'; type LoadingState = 'loading' | 'complete'; @@ -39,6 +41,7 @@ export class DataService { private globalFilterSource = new Subject(); public globalFilter$ = this.globalFilterSource.asObservable(); + private selectedTreeNode: TreeNode = null; // selected checkboxes for keywords filter private selectedKeywords: string[] = []; // selected checkboxes for projects filter @@ -47,11 +50,14 @@ export class DataService { private selectedResearchLines: string[] = []; // list of keywords available for current item list - private keywords: string[] = []; - // list of keywords available for current item list - private projects: string[] = []; - // list of keywords available for current item list - private researchLines: string[] = []; + private keywords: CheckboxOption[] = []; + // list of project names available for current item list + private projects: CheckboxOption[] = []; + // list of research lines available for current item list + private researchLines: CheckboxOption[] = []; + + // list of all concepts + private concepts: Concept[] = []; // list of all projects private availableProjects: Project[] = []; @@ -74,6 +80,7 @@ export class DataService { constructor(private resourceService: ResourceService) { this.updateAvailableProjects(); + this.updateConcepts(); this.updateNodes(); this.updateItems(); this.setFilteredItems(); @@ -150,6 +157,19 @@ export class DataService { ); } + updateConcepts() { + this.resourceService.getConcepts() + .subscribe((concepts: Concept[]) => { + this.concepts = concepts; + let keywords: string[] = [].concat.apply([], this.concepts.map((concept: Concept) => concept.keywords)); + this.keywords.length = 0; + keywords.forEach(keyword => this.keywords.push({label: keyword, value: keyword} as CheckboxOption)); + console.info(`Loaded ${this.keywords.length} key words.`); + }, + err => console.error(err) + ); + } + updateNodes() { this.loadingTreeNodes = 'loading'; // Retrieve all tree nodes @@ -173,7 +193,6 @@ export class DataService { this.resourceService.getItems() .subscribe( (items: Item[]) => { - console.log('item loading'); for (let item of items) { if (this.availableProjects) { item['researchLine'] = this.availableProjects.find(p => p.name == item['project']).lineOfResearch; @@ -184,18 +203,44 @@ export class DataService { } this.setFilteredItems(); this.getUniqueFilterValues(); + console.info(`Loaded ${items.length} items ...`); this.loadingItems = false; }, err => console.error(err) ); } - updateItemTable(treeNode: TreeNode) { + static treeConceptCodes(treeNode: TreeNode) : Set { + if (treeNode == null) { + return new Set(); + } + let conceptCodes = new Set(); + if (treeNode.concept != null) { + conceptCodes.add(treeNode.concept.conceptCode); + } + if (treeNode.children != null) { + treeNode.children.forEach((node: TreeNode) => + DataService.treeConceptCodes(node).forEach((conceptCode: string) => + conceptCodes.add(conceptCode) + ) + ) + } + return conceptCodes; + } + + selectTreeNode(treeNode: TreeNode) { + this.selectedTreeNode = treeNode; + this.updateItemTable(); + } + + updateItemTable() { this.items.length = 0; - let nodeItems = treeNode ? this.itemsPerNode.filter(item => item.itemPath.startsWith(treeNode.path)) - : this.itemsPerNode; - for (let node of nodeItems) { - this.items.push(node); + if (this.selectedTreeNode == null) { + this.itemsPerNode.forEach(item => this.items.push(item)) + } else { + let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); + let nodeItems = this.itemsPerNode.filter(item => selectedConceptCodes.has(item.concept)); + nodeItems.forEach(item => this.items.push(item)) } this.setFilteredItems(); this.getUniqueFilterValues(); @@ -206,7 +251,7 @@ export class DataService { this.setFilteredItems(); this.projects.length = 0; for (let item of this.filteredItems) { - DataService.collectUnique(item['project'], this.projects); + DataService.collectUnique(item.project, this.projects); } } @@ -229,20 +274,54 @@ export class DataService { return this.projects; } - getReasearchLines() { + getResearchLines() { return this.researchLines; } + findConceptCodesByKeywords(keywords: string[]): Set { + return new Set(this.concepts.filter(concept => + concept.keywords != null && concept.keywords.some((keyword: string) => + keywords.includes(keyword)) + ).map(concept => concept.conceptCode)); + } + + static intersection(a: Set, b: Set): Set { + return new Set( + Array.from(a).filter(item => b.has(item))); + } + + getItemFilter(): (Item) => boolean { + let conceptCodesFromTree = DataService.treeConceptCodes(this.selectedTreeNode); + let conceptCodesFromKeywords = this.findConceptCodesByKeywords(this.selectedKeywords); + let selectedConceptCodes: Set; + if (conceptCodesFromKeywords.size > 0 && conceptCodesFromTree.size > 0) { + selectedConceptCodes = DataService.intersection(conceptCodesFromTree, conceptCodesFromTree); + } else if (conceptCodesFromKeywords.size > 0) { + selectedConceptCodes = conceptCodesFromKeywords; + } else { + selectedConceptCodes = conceptCodesFromTree; + } + + return (item: Item) => { + return ((selectedConceptCodes.size == 0 || selectedConceptCodes.has(item.concept)) + && (this.selectedProjects.length == 0 || this.selectedProjects.includes(item.project)) + && (this.selectedResearchLines.length == 0 || this.selectedResearchLines.includes(item.researchLine)) + ); + }; + } + setFilteredItems() { + let t1 = new Date(); + console.debug(`Filtering items ...`); this.filteredItems.length = 0; - for (let item of this.items) { - if ((this.selectedKeywords.length == 0 || item['keywords'].some(k => this.selectedKeywords.includes(k))) - && (this.selectedProjects.length == 0 || this.selectedProjects.includes(item['project'])) - && (this.selectedResearchLines.length == 0 || this.selectedResearchLines.includes(item['researchLine'])) - ) { - this.filteredItems.push(item); + let filter = this.getItemFilter(); + this.items.forEach(item => { + if (filter(item)) { + this.filteredItems.push(item) } - } + }); + let t2 = new Date(); + console.info(`Selected ${this.filteredItems.length} / ${this.items.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); } setGlobalFilter(globalFilter: string) { @@ -250,8 +329,7 @@ export class DataService { } addToShoppingCart(newItemSelection: Item[]) { - let items: Item[] = this.shoppingCartItems.getValue(); - let newItems: Item[] = items; + let newItems: Item[] = this.shoppingCartItems.getValue(); for (let item of newItemSelection) { if (!newItems.includes(item)) { newItems.push(item); @@ -265,25 +343,21 @@ export class DataService { } private getUniqueFilterValues() { - this.keywords.length = 0; this.projects.length = 0; this.researchLines.length = 0; for (let item of this.items) { - for (let keyword of item['keywords']) { - DataService.collectUnique(keyword, this.keywords); - } - DataService.collectUnique(item['project'], this.projects); - DataService.collectUnique(item['researchLine'], this.researchLines); + DataService.collectUnique(item.project, this.projects); + DataService.collectUnique(item.researchLine, this.researchLines); } } - private static collectUnique(element, list) { + private static collectUnique(element, list: CheckboxOption[]) { let values = list.map(function (a) { return a.value; }); if (element && !values.includes(element)) { - list.push({label: element, value: element}); + list.push({label: element, value: element} as CheckboxOption); } } diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index 48a7a50..fe22272 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -13,12 +13,14 @@ import {Http, Response, Headers, ResponseContentType} from '@angular/http'; import {Endpoint} from "../models/endpoint"; import {AppConfig} from "../config/app.config"; import { - PATH_ENVIRONMENT, PATH_ITEMS, PATH_LOGOS, PATH_PROJECTS, - PATH_TREE_NODES + PATH_CONCEPTS, + PATH_ENVIRONMENT, PATH_ITEMS, PATH_LOGOS, PATH_PROJECTS, + PATH_TREE_NODES } from "../constants/endpoints.constants"; import {Item} from "../models/item"; import {Project} from "../models/project"; import {Environment} from "../models/environment"; +import { Concept } from '../models/concept'; @Injectable() @@ -79,6 +81,17 @@ export class ResourceService { .catch(this.handleError.bind(this)); } + getConcepts(): Observable { + let headers = new Headers(); + let url = this.endpoint.apiUrl + PATH_CONCEPTS; + + return this.http.get(url, { + headers: headers + }) + .map((response: Response) => response.json().concepts as Concept[]) + .catch(this.handleError.bind(this)); + } + getLogo(type: string): Observable { let headers = new Headers(); let url = this.endpoint.apiUrl + PATH_LOGOS +"/" + type; diff --git a/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.ts b/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.ts index 3c38f92..4f3903e 100644 --- a/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.ts +++ b/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.ts @@ -77,7 +77,7 @@ export class TreeNodesComponent implements OnInit, AfterViewInit { selectNode(event) { if (event.node) { - this.dataService.updateItemTable(event.node) + this.dataService.selectTreeNode(event.node) } } @@ -87,7 +87,7 @@ export class TreeNodesComponent implements OnInit, AfterViewInit { // https://github.com/primefaces/primeng/issues/2882 this.autocompleteCharge.inputEL.nativeElement.value = ''; this.searchTerm = ''; - this.dataService.updateItemTable(null); + this.dataService.selectTreeNode(null); this.onFiltering(event); } From e01239f7ab7857900025a37d9f1aef20900844c8 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 27 Oct 2017 15:36:22 +0200 Subject: [PATCH 009/104] Update data import script. --- .../datashowcase/DataImportService.groovy | 180 ++++++++++++++---- 1 file changed, 148 insertions(+), 32 deletions(-) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy index 05c17c7..89c9a0f 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy @@ -6,13 +6,18 @@ package nl.thehyve.datashowcase -import com.sun.org.apache.xpath.internal.compiler.Keywords import grails.gorm.transactions.Transactional import grails.validation.ValidationException import nl.thehyve.datashowcase.exception.InvalidDataException +import org.grails.core.util.StopWatch import org.grails.datastore.gorm.GormEntity import org.grails.web.json.JSONArray import org.grails.web.json.JSONObject +import org.hibernate.HibernateException +import org.hibernate.Session +import org.hibernate.SessionFactory +import org.hibernate.StatelessSession +import org.hibernate.Transaction import org.springframework.beans.factory.annotation.Autowired @Transactional @@ -24,49 +29,100 @@ class DataImportService { @Autowired DataService dataService + @Autowired + ItemService itemService + + SessionFactory sessionFactory + def upload(JSONObject json) { + def stopWatch = new StopWatch('Upload data') + // clear database + log.info('Clearing database...') + stopWatch.start('Clear database') + dataService.clearDatabase() + stopWatch.stop() + + Session session = null + Transaction tx = null try { - // clear database - log.info('Clearing database...') - dataService.clearDatabase() + session = sessionFactory.openSession() + tx = session.beginTransaction() // save keywords def keywords = json.concepts?.keywords?.flatten().unique().collect { - if(it?.trim()) new Keyword(keyword: it) - } + if (it?.trim()) new Keyword(keyword: it) + } as List keywords.removeAll([null]) validate(keywords) log.info('Saving keywords...') - keywords*.save(flush: true, failOnError: true) + stopWatch.start('Save keywords') + keywords*.save() + stopWatch.stop() + def keywordMap = keywords.collectEntries { [(it.keyword): it] } as Map // save concepts - def concepts = json.concepts?.collect { new Concept(it) } + def concepts = json.concepts?.collect { + def conceptKeywords = it.remove('keywords') + def concept = new Concept(it as JSONObject) + conceptKeywords.each { + def keyword = keywordMap[it?.trim() as String] + if (keyword) { + concept.addToKeywords(keyword) + } + } + concept + } as List validate(concepts) log.info('Saving concepts...') - concepts*.save(flush: true, failOnError: true) + stopWatch.start('Save concepts') + concepts*.save() + stopWatch.stop() + def conceptMap = concepts.collectEntries { [(it.conceptCode): it] } as Map + + session.flush() + session.clear() // save tree_nodes - def tree_nodes = json.tree_nodes?.collect { JSONObject it -> - copyConceptCode(it) - new TreeNode(it) - } + def tree_nodes = flatten(buildTree(json.tree_nodes as JSONArray, conceptMap)) validate(tree_nodes) log.info('Saving tree nodes...') - tree_nodes*.save(flush: true, failOnError: true) + stopWatch.start('Save tree nodes') + tree_nodes*.save() + stopWatch.stop() + + session.flush() + session.clear() //save research_lines def linesOfResearch = json.projects?.lineOfResearch.unique().collect { if (it) new LineOfResearch(name: it) - } + } as List validate(linesOfResearch) log.info("Saving lines of research...") - linesOfResearch*.save(flush: true, failOnError: true) + stopWatch.start('Save lines of research') + linesOfResearch*.save() + stopWatch.stop() + def lineOfResearchMap = linesOfResearch.collectEntries { [(it.name): it]} as Map // save projects - def projects = json.projects?.collect { new Project(it) } + def projects = json.projects?.collect { JSONObject it -> + def lineOfResearch = it.remove('lineOfResearch') as String + def project = new Project(it) + project.lineOfResearch = lineOfResearchMap[lineOfResearch] + if (!project.lineOfResearch) { + throw new InvalidDataException("No valid line of research set for project ${project.name}.") + } + project + } as List validate(projects) log.info("Saving projects...") - projects*.save(flush: true, failOnError: true) + stopWatch.start('Save projects') + projects*.save() + stopWatch.stop() + def projectMap = projects.collectEntries { [(it.name): it] } as Map + + session.flush() + session.clear() // save items, related summaries and values if (!dataShowcaseEnvironment.internalInstance && !allItemsArePublic((JSONArray) json.items)) { @@ -74,22 +130,61 @@ class DataImportService { throw new InvalidDataException("Data validation exception. " + "Non public item cannot be loaded to a public environment") } + def summaryMap = [:] as Map def items = json.items?.collect { JSONObject it -> - it.concept = it.conceptCode + def summaryData = it.remove('summary') as JSONObject def item = new Item(it) - item.project = Project.findByName(it.projectName as String) + summaryMap[item.name] = new Summary(summaryData) + item.concept = conceptMap[it.conceptCode as String] + if (!item.concept) { + throw new InvalidDataException("No valid concept code set for item ${item.name}.") + } + item.project = projectMap[it.projectName as String] + if (!item.project) { + throw new InvalidDataException("No valid project name set for item ${item.name}.") + } item - } + } as List validate(items) - log.info("Saving $items.size() items, summaries, values...") - items*.save(flush: true, failOnError: true) + log.info"Saving ${items.size()} items ..." + stopWatch.start('Save items') + def count = 0 + def countWidth = items.size().toString().size() + items.collate(500).each{ sublist -> + sublist*.save() + count += sublist.size() + log.info " [${count.toString().padLeft(countWidth)} / ${items.size()}] items saved" + sublist.each { + it.setSummary(summaryMap[it.name]) + it.summary?.item = it + } + log.info " [${count.toString().padLeft(countWidth)} / ${items.size()}] summaries saved" + sublist*.summary*.save() + + session.flush() + session.clear() + } + stopWatch.stop() + log.info "All items saved." + + stopWatch.start('Commit transaction') + tx.commit() + stopWatch.stop() + + log.info "Upload completed.\n${stopWatch.prettyPrint()}" + + itemService.clearItemCountsCache() } catch (ValidationException e) { - log.error e.message - throw new InvalidDataException(e.message) + log.error "Invalid data uploaded", e + tx?.rollback() + throw new InvalidDataException("An error occured when uploading the data: ${e.message}") } catch (Exception e) { - log.error e.message - throw new InvalidDataException("An error occured when uploading the data: $e.message") + log.error "Error while saving data", e + tx?.rollback() + throw new InvalidDataException("An error occured when uploading the data: ${e.message}") + } finally{ + session?.close() } } @@ -106,12 +201,33 @@ class DataImportService { return items.every { it.publicItem == true } } - private static void copyConceptCode(JSONObject obj) { - obj.concept = obj.conceptCode - if (obj.children) { - (obj.children as JSONArray).each { - copyConceptCode(it as JSONObject) + private static List flatten(Collection nodes) { + if (nodes == null) { + return [] + } + nodes.collectMany { node -> + [node] + flatten(node.children) + } + } + + private static List buildTree(JSONArray nodes, final Map conceptMap) { + if (nodes == null) { + return null + } + nodes.collect { JSONObject nodeData -> + def childrenData = nodeData.remove('children') as JSONArray + def node = new TreeNode(nodeData) + if (nodeData.conceptCode) { + node.concept = conceptMap[nodeData.conceptCode as String] + if (!node.concept) { + throw new InvalidDataException("Invalid concept code set for tree node ${node.path}.") + } + } + node.children = buildTree(childrenData, conceptMap) + node.children?.each { + it.parent = node } + node } } } From 0b8bbd8737295e825f903bd1dbe77e9220fc28da Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 27 Oct 2017 16:47:34 +0200 Subject: [PATCH 010/104] Fix data import tests. --- .../datashowcase/DataImportService.groovy | 4 ++-- .../thehyve/datashowcase/TreeService.groovy | 6 +++-- .../datashowcase/DataImportServiceSpec.groovy | 22 +++++++++---------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy index 1232439..4942676 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy @@ -167,6 +167,8 @@ class DataImportService { stopWatch.start('Commit transaction') tx.commit() + session.flush() + session.clear() stopWatch.stop() log.info "Upload completed.\n${stopWatch.prettyPrint()}" @@ -181,8 +183,6 @@ class DataImportService { log.error "Error while saving data", e tx?.rollback() throw new InvalidDataException("An error occured when uploading the data: ${e.message}") - } finally{ - session?.close() } } diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy index 1d6f098..f55d459 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy @@ -66,8 +66,10 @@ class TreeService { concept: map(node.concept), path: node.path, ) - result.children = node.children.collect { - map(it) + if (node.children) { + result.children = node.children.collect { + map(it) + } } result } diff --git a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/DataImportServiceSpec.groovy b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/DataImportServiceSpec.groovy index f582c0c..788c310 100644 --- a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/DataImportServiceSpec.groovy +++ b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/DataImportServiceSpec.groovy @@ -8,19 +8,19 @@ package nl.thehyve.datashowcase import grails.converters.JSON import grails.testing.mixin.integration.Integration +import grails.transaction.Rollback import groovy.util.logging.Slf4j import nl.thehyve.datashowcase.enumeration.NodeType import nl.thehyve.datashowcase.enumeration.VariableType import nl.thehyve.datashowcase.exception.InvalidDataException import org.grails.web.json.JSONObject import org.springframework.beans.factory.annotation.Autowired -import org.springframework.transaction.annotation.Transactional import spock.lang.Requires import spock.lang.Specification @Slf4j @Integration -@Transactional +@Rollback class DataImportServiceSpec extends Specification { @Autowired @@ -76,7 +76,7 @@ class DataImportServiceSpec extends Specification { '/Personal information/Extended information/Height'] items.every { it.publicItem == true } items*.summary == summaries - items*.summary*.values?.flatten() == values + items*.summary*.values?.flatten() as Set == values as Set items*.concept.every { concepts.indexOf(it) != -1 } items*.project.every { projects.indexOf(it) != -1 } @@ -104,7 +104,7 @@ class DataImportServiceSpec extends Specification { summaries*.patientCount == [100, 200] summaries*.observationCount == [102, 402] summaries*.dataStability == ['Committed', 'Committed'] - summaries*.values.flatten() == values + summaries*.values.flatten() as Set == values as Set treeNodes.size() == 9 treeNodes*.label.sort() == ['Personal information', 'Basic information', 'Age', 'Weight', 'Extended information', @@ -125,10 +125,10 @@ class DataImportServiceSpec extends Specification { '/Other information', '/Other information/Some information'].sort() - values.size() == 8 - values*.value.every { v -> v in ['<= 65', '> 65', '<= 175', '> 175'] } - values*.label.every { l -> l in ['Young', 'Old', 'Short', 'Tall'] } - values*.frequency.every { f -> (int)f in [35, 65, 36, 64] } + values.size() == 4 + values*.value as Set == ['<= 65', '> 65', '<= 175', '> 175'] as Set + values*.label as Set == ['Young', 'Old', 'Short', 'Tall'] as Set + values*.frequency as Set == [35, 65, 36, 64] as Set } @Requires({ -> Environment.grailsEnvironmentIn(Constants.PUBLIC_ENVIRONMENTS) }) @@ -157,7 +157,7 @@ class DataImportServiceSpec extends Specification { '/Personal information/Extended information/Height'] items.every { it.publicItem == true } items*.summary == summaries - items*.summary*.values?.flatten() == values + items*.summary*.values?.flatten() as Set == values as Set items*.concept.every { concepts.indexOf(it) != -1 } items*.project.every { projects.indexOf(it) != -1 } @@ -185,7 +185,7 @@ class DataImportServiceSpec extends Specification { summaries*.patientCount == [100, 200] summaries*.observationCount == [102, 402] summaries*.dataStability == ['Committed', 'Committed'] - summaries*.values.flatten() == values + summaries*.values.flatten() as Set == values as Set treeNodes.size() == 9 treeNodes*.label.sort() == ['Personal information', 'Basic information', 'Age', 'Weight', 'Extended information', @@ -206,7 +206,7 @@ class DataImportServiceSpec extends Specification { '/Other information', '/Other information/Some information'].sort() - values.size() == 8 + values.size() == 4 values*.value.every { v -> v in ['<= 65', '> 65', '<= 175', '> 175'] } values*.label.every { l -> l in ['Young', 'Old', 'Short', 'Tall'] } values*.frequency.every { f -> (int)f in [35, 65, 36, 64] } From 57e78b4556d83821873f2f653546fb0ce24dc6cc Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 30 Oct 2017 09:30:38 +0100 Subject: [PATCH 011/104] Performance improvements for fetching items. --- .../grails-app/conf/spring/resources.groovy | 2 - .../datashowcase/ItemController.groovy | 1 + .../nl/thehyve/datashowcase/Item.groovy | 6 +- .../datashowcase/ConceptService.groovy | 8 ++ .../datashowcase/DataImportService.groovy | 5 +- .../thehyve/datashowcase/DataService.groovy | 16 +++ .../thehyve/datashowcase/ItemService.groovy | 110 +++++++++++++----- .../thehyve/datashowcase/TreeService.groovy | 8 ++ .../datashowcase/mapping/ItemMapper.groovy | 39 ------- .../InternalItemRepresentation.groovy | 3 + .../representation/ItemRepresentation.groovy | 7 +- .../PublicItemRepresentation.groovy | 3 + .../user-interface/src/app/models/item.ts | 1 + .../src/app/services/data.service.ts | 8 +- .../src/app/services/resource.service.ts | 11 ++ 15 files changed, 142 insertions(+), 86 deletions(-) delete mode 100644 data-showcase/src/main/groovy/nl/thehyve/datashowcase/mapping/ItemMapper.groovy diff --git a/data-showcase/grails-app/conf/spring/resources.groovy b/data-showcase/grails-app/conf/spring/resources.groovy index fdc9f4c..ce610e2 100644 --- a/data-showcase/grails-app/conf/spring/resources.groovy +++ b/data-showcase/grails-app/conf/spring/resources.groovy @@ -6,7 +6,6 @@ import nl.thehyve.datashowcase.Environment import nl.thehyve.datashowcase.StartupMessage -import nl.thehyve.datashowcase.mapping.ItemMapper import org.modelmapper.ModelMapper import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder @@ -14,7 +13,6 @@ import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder beans = { dataShowcaseEnvironment(Environment) {} modelMapper(ModelMapper) {} - itemMapper(ItemMapper) {} startupMessage(StartupMessage) {} bcryptEncoder(BCryptPasswordEncoder) {} } diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index fe67b73..2ebdd22 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -24,6 +24,7 @@ class ItemController { def index() { response.status = 200 response.contentType = 'application/json' + response.characterEncoding = 'utf-8' new ObjectMapper().writeValue(response.outputStream, [items: itemService.items]) } diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy index 4cb13af..2c68322 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Item.groovy @@ -6,6 +6,8 @@ package nl.thehyve.datashowcase +import nl.thehyve.datashowcase.enumeration.VariableType + /** * An item represents a variable in a study or survey. * Besides links to the concept and project it belongs to, @@ -63,16 +65,16 @@ class Item { concept.labelNlLong } - String getType() { + VariableType getType() { concept.variableType } static mapping = { version false + summary lazy: true concept fetch: 'join' project fetch: 'join' - summary fetch: 'join' } static constraints = { diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ConceptService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ConceptService.groovy index 33497ab..ee6f8e7 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ConceptService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ConceptService.groovy @@ -7,6 +7,8 @@ package nl.thehyve.datashowcase import grails.gorm.transactions.Transactional +import grails.plugin.cache.CacheEvict +import grails.plugin.cache.Cacheable import nl.thehyve.datashowcase.representation.ConceptRepresentation import org.modelmapper.ModelMapper import org.springframework.beans.factory.annotation.Autowired @@ -17,10 +19,16 @@ class ConceptService { @Autowired ModelMapper modelMapper + @Cacheable('concepts') List getConcepts() { Concept.findAll().collect({ Concept concept -> modelMapper.map(concept, ConceptRepresentation) }) } + @CacheEvict(value = 'concepts', allEntries = true) + void clearConceptsCache() { + log.info "Clear concepts cache." + } + } diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy index 4942676..11c3a46 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy @@ -27,9 +27,6 @@ class DataImportService { @Autowired DataService dataService - @Autowired - ItemService itemService - SessionFactory sessionFactory def upload(JSONObject json) { @@ -173,7 +170,7 @@ class DataImportService { log.info "Upload completed.\n${stopWatch.prettyPrint()}" - itemService.clearItemCountsCache() + dataService.clearCaches() } catch (ValidationException e) { log.error "Invalid data uploaded", e diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataService.groovy index 1852550..28af44d 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataService.groovy @@ -15,6 +15,22 @@ class DataService { @Autowired Environment dataShowcaseEnvironment + @Autowired + ItemService itemService + + @Autowired + ConceptService conceptService + + @Autowired + TreeService treeService + + def clearCaches() { + itemService.clearItemsCache() + itemService.clearItemCountsCache() + conceptService.clearConceptsCache() + treeService.clearTreeNodesCache() + } + def clearDatabase() { Value.executeUpdate('delete from Value') Summary.executeUpdate('delete from Summary') diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index c6f8625..0351329 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -9,10 +9,17 @@ package nl.thehyve.datashowcase import grails.gorm.transactions.Transactional import grails.plugin.cache.CacheEvict import grails.plugin.cache.Cacheable +import groovy.transform.CompileStatic +import nl.thehyve.datashowcase.enumeration.VariableType import nl.thehyve.datashowcase.exception.ResourceNotFoundException -import nl.thehyve.datashowcase.mapping.ItemMapper +import nl.thehyve.datashowcase.representation.InternalItemRepresentation import nl.thehyve.datashowcase.representation.ItemRepresentation +import nl.thehyve.datashowcase.representation.PublicItemRepresentation import org.grails.core.util.StopWatch +import org.hibernate.Session +import org.hibernate.SessionFactory +import org.hibernate.transform.Transformers +import org.modelmapper.ModelMapper import org.springframework.beans.factory.annotation.Autowired @Transactional @@ -22,46 +29,84 @@ class ItemService { Environment dataShowcaseEnvironment @Autowired - ItemMapper itemMapper + ModelMapper modelMapper + SessionFactory sessionFactory + + @CompileStatic + static ItemRepresentation map(Map itemData) { + new ItemRepresentation( + id: itemData.id as Long, + name: itemData.name as String, + labelLong: itemData.labelLong as String, + project: itemData.projectName as String, + concept: itemData.conceptCode as String, + itemPath: itemData.itemPath as String, + type: itemData.type as VariableType, + publicItem: itemData.publicItem as boolean + ) + } + + @Cacheable('items') @Transactional(readOnly = true) List getItems() { - if (dataShowcaseEnvironment.internalInstance) { - def stopWatch = new StopWatch('Fetch items') - stopWatch.start('Retrieve from database') - def items = Item.findAll() - stopWatch.stop() - stopWatch.start('Map to representations') - def result = items.collect { - itemMapper.map(it) - } - stopWatch.stop() - log.info "Items fetched.\n${stopWatch.prettyPrint()}" - result - } else { - Item.findAllByPublicItem(true).collect { - itemMapper.map(it) - } + def stopWatch = new StopWatch('Fetch items') + stopWatch.start('Retrieve from database') + def session = sessionFactory.openStatelessSession() + def items = session.createQuery( + """ + select + i.id as id, + i.name as name, + i.publicItem as publicItem, + i.itemPath as itemPath, + c.conceptCode as conceptCode, + c.labelLong as labelLong, + c.variableType as type, + p.name as projectName + from Item as i + join i.concept c + join i.project p + ${dataShowcaseEnvironment.internalInstance ? + '' : 'where i.publicItem = true' + } + """ + ).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) + .list() as List + stopWatch.stop() + stopWatch.start('Map to representations') + def result = items.collect { Map itemData -> + map(itemData) } + stopWatch.stop() + log.info "Items fetched.\n${stopWatch.prettyPrint()}" + result + } + + @CacheEvict(value = 'items', allEntries = true) + void clearItemsCache() { + log.info "Clear items cache." } @Transactional(readOnly = true) @Cacheable('itemCountPerNode') Map getItemCountPerNode() { + Session session = sessionFactory.currentSession if (dataShowcaseEnvironment.internalInstance) { - def itemCountMap = Item.executeQuery( + def itemCountMap = session.createQuery( """ select n.path as path, count(distinct i) as itemCount from Item i, TreeNode n join i.concept c where n.concept = c group by n.path """ - ) as List + ).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) + .list() as List itemCountMap.collectEntries { - [(it[0]): it[1] as Long] + [(it.path): it.itemCount as Long] } } else { - def itemCountMap = Item.executeQuery( + def itemCountMap = session.createQuery( """ select n.path as path, count(distinct i) as itemCount from Item i, TreeNode n join i.concept c @@ -69,9 +114,10 @@ class ItemService { and i.publicItem = true group by n.path """ - ) as List + ).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) + .list() as List itemCountMap.collectEntries { - [(it[0]): it[1] as Long] + [(it.path): it.itemCount as Long] } } } @@ -83,16 +129,18 @@ class ItemService { @Transactional(readOnly = true) ItemRepresentation getItem(long id) { - def item if (dataShowcaseEnvironment.internalInstance) { - item = Item.findById(id) + def item = Item.findById(id) + if (item) { + return modelMapper.map(item, InternalItemRepresentation.class) + } } else { - item = Item.findByPublicItemAndId(true, id) - } - if (item == null) { - throw new ResourceNotFoundException('Item not found') + def item = Item.findByPublicItemAndId(true, id) + if (item) { + return modelMapper.map(item, PublicItemRepresentation.class) + } } - itemMapper.map(item) + throw new ResourceNotFoundException('Item not found') } } diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy index f55d459..6cdf8a3 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy @@ -7,6 +7,8 @@ package nl.thehyve.datashowcase import grails.gorm.transactions.Transactional +import grails.plugin.cache.CacheEvict +import grails.plugin.cache.Cacheable import groovy.transform.CompileStatic import nl.thehyve.datashowcase.representation.ConceptRepresentation import nl.thehyve.datashowcase.representation.TreeNodeRepresentation @@ -78,6 +80,7 @@ class TreeService { * Fetches all top nodes of the tree. * @return the list of top nodes with child nodes embedded. */ + @Cacheable('tree_nodes') List getNodes() { def stopWatch = new StopWatch('Fetch tree nodes') stopWatch.start('Retrieve from database') @@ -98,4 +101,9 @@ class TreeService { result } + @CacheEvict(value = 'tree_nodes', allEntries = true) + void clearTreeNodesCache() { + log.info "Clear tree nodes cache." + } + } diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/mapping/ItemMapper.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/mapping/ItemMapper.groovy deleted file mode 100644 index ab4ffaa..0000000 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/mapping/ItemMapper.groovy +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2017 The Hyve B.V. - * This file is distributed under the GNU Affero General Public License - * (see accompanying file LICENSE). - */ - -package nl.thehyve.datashowcase.mapping - -import groovy.transform.CompileStatic -import nl.thehyve.datashowcase.Environment -import nl.thehyve.datashowcase.Item -import nl.thehyve.datashowcase.representation.InternalItemRepresentation -import nl.thehyve.datashowcase.representation.ItemRepresentation -import nl.thehyve.datashowcase.representation.PublicItemRepresentation -import org.modelmapper.ModelMapper -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component - -@Component -@CompileStatic -class ItemMapper { - - @Autowired - ModelMapper modelMapper - - @Autowired - Environment dataShowcaseEnvironment - - @Lazy - Closure map = (dataShowcaseEnvironment.internalInstance) ? - { Item item -> - modelMapper.map(item, InternalItemRepresentation.class) - } as Closure - : - { Item item -> - modelMapper.map(item, PublicItemRepresentation.class) - } as Closure - -} diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/InternalItemRepresentation.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/InternalItemRepresentation.groovy index 917a405..7ed2d7d 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/InternalItemRepresentation.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/InternalItemRepresentation.groovy @@ -11,6 +11,9 @@ import groovy.transform.CompileStatic @CompileStatic class InternalItemRepresentation extends ItemRepresentation { + /** + * Summary data for the variable: aggregate values and value frequencies. + */ InternalSummaryRepresentation summary } diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy index bf7dbf1..ad2c6ed 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy @@ -10,7 +10,7 @@ import groovy.transform.CompileStatic import nl.thehyve.datashowcase.enumeration.VariableType @CompileStatic -abstract class ItemRepresentation { +class ItemRepresentation { /** * An id of the variable @@ -58,11 +58,6 @@ abstract class ItemRepresentation { */ boolean publicItem - /** - * Summary data for the variable: aggregate values and value frequencies. - */ - abstract SummaryRepresentation getSummary() - /** * The concept code of the associated concept. */ diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/PublicItemRepresentation.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/PublicItemRepresentation.groovy index 25bdf64..ff5a375 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/PublicItemRepresentation.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/PublicItemRepresentation.groovy @@ -11,6 +11,9 @@ import groovy.transform.CompileStatic @CompileStatic class PublicItemRepresentation extends ItemRepresentation { + /** + * Summary data for the variable: aggregate values and value frequencies. + */ PublicSummaryRepresentation summary } diff --git a/data-showcase/src/main/user-interface/src/app/models/item.ts b/data-showcase/src/main/user-interface/src/app/models/item.ts index b5f656e..df0772e 100644 --- a/data-showcase/src/main/user-interface/src/app/models/item.ts +++ b/data-showcase/src/main/user-interface/src/app/models/item.ts @@ -8,6 +8,7 @@ import {ItemSummary} from "./item-summary"; import {Concept} from "./concept"; export class Item { + id: number; name: string; itemPath: string; type: string; diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 8365118..5c6c7dc 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -118,7 +118,9 @@ export class DataService { let newNode: TreeNodeLib = node; // filter out empty domains - newNode.children = node.children.filter(value => !(value.accumulativeItemCount == 0 && value.nodeType == "Domain" )); + if (node.children) { + newNode.children = node.children.filter(value => !(value.accumulativeItemCount == 0 && value.nodeType == "Domain" )); + } let count = node.accumulativeItemCount ? node.accumulativeItemCount : 0; let countStr = ' (' + count + ')'; newNode.label = node.label + countStr; @@ -362,7 +364,9 @@ export class DataService { } displayPopup(item: Item) { - this.itemSummaryVisibleSource.next(item); + this.resourceService.getItem(item.id).subscribe(extendedItem => + this.itemSummaryVisibleSource.next(extendedItem) + ) } setEnvironment() { diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index fe22272..997bea6 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -59,6 +59,17 @@ export class ResourceService { .catch(this.handleError.bind(this)); } + getItem(id: number): Observable { + let headers = new Headers(); + let url = `${this.endpoint.apiUrl}${PATH_ITEMS}/${id}`; + + return this.http.get(url, { + headers: headers + }) + .map((response: Response) => response.json() as Item) + .catch(this.handleError.bind(this)); + } + getItems(): Observable { let headers = new Headers(); let url = this.endpoint.apiUrl + PATH_ITEMS; From 6fc50e48f3a7550393e50b46d53718a3a02d1001 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 30 Oct 2017 10:16:35 +0100 Subject: [PATCH 012/104] Fix tests. --- .../datashowcase/TestController.groovy | 2 +- .../thehyve/datashowcase/TestService.groovy | 26 +++++++++++++------ .../datashowcase/DataImportServiceSpec.groovy | 6 +++++ .../InternalItemServiceSpec.groovy | 16 ++++++++---- .../datashowcase/PublicItemServiceSpec.groovy | 2 +- 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TestController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TestController.groovy index 2937e29..45e76b3 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TestController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TestController.groovy @@ -23,7 +23,7 @@ class TestController { } def createInternalData() { - testService.createInternalTestData() + testService.createRandomInternalTestData() } def createPublicData() { diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/TestService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/TestService.groovy index 261c8db..698a36d 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/TestService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/TestService.groovy @@ -143,20 +143,27 @@ class TestService { Summary[] weightSummaries = createSummary(9, 55, 230) Item[] weightItems = createItems(9, "weight", projectC, nodes[3], weightSummaries, - true) + false) - createRandomData() concepts*.save(flush: true) nodes*.save(flush: true) - nodes.each { node -> - log.info "Created node ${node.label}, parent: ${node.parent?.path}, path: ${node.path}" - } ageItems*.save(flush: true) heightItems*.save(flush: true) weightItems*.save(flush: true) } + /** + * Creates and stores test objects for the internal environment (including + * non-public items and extensive summary data). + */ + def createRandomInternalTestData() { + checkGrailsEnvironment(Constants.INTERNAL_ENVIRONMENTS) + + createInternalTestData() + createRandomData() + } + /** * Creates and stores test objects for the public environment (with * only public items and limited summary data). @@ -219,10 +226,10 @@ class TestService { testLineOfResearch = new LineOfResearch(name: 'Test research line') testLineOfResearch.save(flush: true) testProject = new Project(name: 'Project', description: 'Test project', lineOfResearch: testLineOfResearch) - [testProject, testKeyword]*.save() + [testProject, testKeyword]*.save(flush: true) int[] domainPerLevel = [10, 2, 3, 2] - TreeNode parent = null; + TreeNode parent = null createRandomDomain(parent, domainPerLevel) } @@ -242,16 +249,19 @@ class TestService { def createRandomConceptNodeWithItems(TreeNode domain) { + def keyword = Keyword.findByKeyword('Test keyword') def concept = new Concept( conceptCode: "ConceptFor$domain.label", label: "ConceptFor$domain.label", labelLong: "ConceptFor$domain.label long description", labelNl: "NederlandsConceptVoor$domain.label", labelNlLong: "NederlandsConceptVoor$domain.label lange beschrijving", variableType: VariableType.Categorical, - keywords: ["keyword: 'Test keyword'"]) + keywords: [keyword]) concepts.add(concept) def conceptNode = new TreeNode(domain, concept) nodes.add(conceptNode) + [concept, conceptNode]*.save(flush: true) + Summary[] summaries = createSummary(8, 12, 99) Item[] items = createItems(3, concept.label + "Item", testProject, conceptNode, summaries, true) diff --git a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/DataImportServiceSpec.groovy b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/DataImportServiceSpec.groovy index 788c310..e5cb72a 100644 --- a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/DataImportServiceSpec.groovy +++ b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/DataImportServiceSpec.groovy @@ -14,6 +14,7 @@ import nl.thehyve.datashowcase.enumeration.NodeType import nl.thehyve.datashowcase.enumeration.VariableType import nl.thehyve.datashowcase.exception.InvalidDataException import org.grails.web.json.JSONObject +import org.hibernate.SessionFactory import org.springframework.beans.factory.annotation.Autowired import spock.lang.Requires import spock.lang.Specification @@ -29,18 +30,22 @@ class DataImportServiceSpec extends Specification { @Autowired DataService dataService + SessionFactory sessionFactory + URL jsonUrl = getClass().getClassLoader().getResource('test.json') File file = new File(jsonUrl.path) def setupData() { log.info "Clear database ..." dataService.clearDatabase() + sessionFactory.currentSession.flush() } def loadValidData() { log.info "Upload test data set ..." JSONObject json = (JSONObject)JSON.parse(file.text) dataImportService.upload(json) + sessionFactory.currentSession.flush() } def loadInvalidData() { @@ -49,6 +54,7 @@ class DataImportServiceSpec extends Specification { JSONObject json = (JSONObject)JSON.parse(file.text) log.info "Upload test data set ..." dataImportService.upload(json) + sessionFactory.currentSession.flush() } @Requires({ -> Environment.grailsEnvironmentIn(Constants.PUBLIC_ENVIRONMENTS) }) diff --git a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/InternalItemServiceSpec.groovy b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/InternalItemServiceSpec.groovy index 96ababa..e7a9c9f 100644 --- a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/InternalItemServiceSpec.groovy +++ b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/InternalItemServiceSpec.groovy @@ -45,13 +45,19 @@ class InternalItemServiceSpec extends Specification { setupData() when: log.info "Running test ..." - List items = itemService.items.collect { it as InternalItemRepresentation } + List items = itemService.items.collect { + itemService.getItem(it.id) as InternalItemRepresentation + } def maxValues = items*.summary.maxValue as List then: "2 items being returned, age and height, both public and internal, including aggregate values" - items.size() == 2 - items*.label == ['Age', 'Height'] - items*.publicItem == [true, false] - items*.itemPath == ['/Personal information/Basic information/Age', '/Personal information/Extended information/Height'] + items.size() == 8 + 20 + 9 + items*.labelLong as Set == ['Age at time of survey', 'Height at time of survey', 'Weight (kg)'] as Set + items*.publicItem as Set == [true, false] as Set + items*.itemPath as Set == [ + '/Personal information/Basic information/Age', + '/Personal information/Extended information/Height', + '/Personal information/Basic information/Weight' + ] as Set that(maxValues, hasItem( closeTo(99, 0.1) )) diff --git a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy index 5e41d22..8575244 100644 --- a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy +++ b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy @@ -47,7 +47,7 @@ class PublicItemServiceSpec extends Specification { then: "2 items being returned" items.size() == 2 items*.name == ['ageA', 'heightB'] - items*.label == ['Age', 'Height'] + items*.labelLong == ['Age at time of survey', 'Height at time of survey'] items*.project == ['Project A', 'Project B'] items*.itemPath == ['/Personal information/Basic information/Age', '/Personal information/Extended information/Height'] that(items*.concept, hasItem('age')) From 7116a96e1182316a4de5225a9c305a576037dc08 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 30 Oct 2017 13:11:38 +0100 Subject: [PATCH 013/104] Improve production front end performance. --- data-showcase/build.gradle | 13 +++++++++---- data-showcase/src/main/user-interface/package.json | 5 +++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/data-showcase/build.gradle b/data-showcase/build.gradle index 626d9df..4c90c19 100644 --- a/data-showcase/build.gradle +++ b/data-showcase/build.gradle @@ -84,13 +84,18 @@ node { nodeModulesDir = file('src/main/user-interface') } -task buildUserInterfaceOnce(type: NpmTask, dependsOn: 'npmInstall') { +task buildUserInterfaceProd(type: NpmTask, dependsOn: 'npmInstall') { group = 'application' description = 'Builds the user interface assets' - args = ['run', 'buildOnce'] + args = ['run', 'buildProd'] } -war.dependsOn(buildUserInterfaceOnce) -bootRun.dependsOn(buildUserInterfaceOnce) +war.dependsOn(buildUserInterfaceProd) +task buildUserInterfaceDev(type: NpmTask, dependsOn: 'npmInstall') { + group = 'application' + description = 'Builds the user interface assets' + args = ['run', 'buildDev'] +} +bootRun.dependsOn(buildUserInterfaceDev) task buildUserInterface(type: NpmTask, dependsOn: 'npmInstall') { group = 'application' diff --git a/data-showcase/src/main/user-interface/package.json b/data-showcase/src/main/user-interface/package.json index 5571c2d..10e654e 100644 --- a/data-showcase/src/main/user-interface/package.json +++ b/data-showcase/src/main/user-interface/package.json @@ -5,7 +5,8 @@ "scripts": { "ng": "ng", "start": "ng serve", - "buildOnce": "ng build --watch=false", + "buildProd": "ng build --prod --watch=false", + "buildDev": "ng build --dev --watch=false", "build": "ng build --watch=true", "test": "ng test --single-run", "lint": "ng lint", @@ -33,7 +34,7 @@ "zone.js": "^0.8.4" }, "devDependencies": { - "@angular/cli": "1.2.1", + "@angular/cli": "1.3.2", "@angular/compiler-cli": "^4.0.0", "@angular/language-service": "^4.0.0", "@types/jasmine": "~2.5.53", From 9ef69ebeb15b0675a8b701ab29df2eff19bd7f48 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 30 Oct 2017 12:16:21 +0100 Subject: [PATCH 014/104] Replace Session with StatelessSession --- .../datashowcase/DataImportService.groovy | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy index 11c3a46..c275558 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy @@ -13,10 +13,10 @@ import org.grails.core.util.StopWatch import org.grails.datastore.gorm.GormEntity import org.grails.web.json.JSONArray import org.grails.web.json.JSONObject -import org.hibernate.Session import org.springframework.beans.factory.annotation.Autowired import org.hibernate.Transaction import org.hibernate.SessionFactory +import org.hibernate.StatelessSession @Transactional class DataImportService { @@ -36,12 +36,11 @@ class DataImportService { stopWatch.start('Clear database') dataService.clearDatabase() stopWatch.stop() - - Session session = null + StatelessSession statelessSession = null Transaction tx = null try { - session = sessionFactory.openSession() - tx = session.beginTransaction() + statelessSession = sessionFactory.openStatelessSession() + tx = statelessSession.beginTransaction() // save keywords def keywords = json.concepts?.keywords?.flatten().unique().collect { @@ -73,9 +72,7 @@ class DataImportService { concepts*.save() stopWatch.stop() def conceptMap = concepts.collectEntries { [(it.conceptCode): it] } as Map - - session.flush() - session.clear() + statelessSession.managedFlush() // save tree_nodes def tree_nodes = flatten(buildTree(json.tree_nodes as JSONArray, conceptMap)) @@ -84,9 +81,7 @@ class DataImportService { stopWatch.start('Save tree nodes') tree_nodes*.save() stopWatch.stop() - - session.flush() - session.clear() + statelessSession.managedFlush() //save research_lines def linesOfResearch = json.projects?.lineOfResearch.unique().collect { @@ -115,9 +110,7 @@ class DataImportService { projects*.save() stopWatch.stop() def projectMap = projects.collectEntries { [(it.name): it] } as Map - - session.flush() - session.clear() + statelessSession.managedFlush() // save items, related summaries and values if (!dataShowcaseEnvironment.internalInstance && !allItemsArePublic((JSONArray) json.items)) { @@ -155,17 +148,14 @@ class DataImportService { } log.info " [${count.toString().padLeft(countWidth)} / ${items.size()}] summaries saved" sublist*.summary*.save() - - session.flush() - session.clear() + statelessSession.managedFlush() } stopWatch.stop() log.info "All items saved." stopWatch.start('Commit transaction') tx.commit() - session.flush() - session.clear() + statelessSession.close() stopWatch.stop() log.info "Upload completed.\n${stopWatch.prettyPrint()}" From 555774bc00647b72b76ea44cb7cd5c54e4be84e7 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 30 Oct 2017 15:13:33 +0100 Subject: [PATCH 015/104] Clear checkbox filters and items selection on node change --- .../src/app/item-table/item-table.component.ts | 5 +++++ .../src/app/services/data.service.ts | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts index 55ffc32..b4c06a2 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts @@ -38,6 +38,11 @@ export class ItemTableComponent implements OnInit { constructor(public dataService: DataService) { this.items = this.dataService.filteredItems; + this.dataService.itemsSelection$.subscribe( + selection => { + this.itemsSelection = selection; + } + ); this.dataService.globalFilter$.subscribe( filter => { this.filterValue = filter; diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 5c6c7dc..4f0b22a 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -34,6 +34,9 @@ export class DataService { public filteredItems: Item[] = []; // items available for currently selected node private itemsPerNode: Item[] = []; + // items selected in the itemTable + private itemsSelectionSource = new Subject(); + public itemsSelection$ = this.itemsSelectionSource.asObservable(); // items added to the shopping cart public shoppingCartItems = new BehaviorSubject([]); @@ -237,6 +240,8 @@ export class DataService { updateItemTable() { this.items.length = 0; + this.clearItemsSelection(); + this.clearFilterValues(); if (this.selectedTreeNode == null) { this.itemsPerNode.forEach(item => this.items.push(item)) } else { @@ -264,6 +269,16 @@ export class DataService { this.setFilteredItems(); } + clearFilterValues() { + this.selectedKeywords.length = 0; + this.selectedResearchLines.length = 0; + this.selectedProjects.length = 0; + } + + clearItemsSelection() { + this.itemsSelectionSource.next(null); + } + getItems() { return this.items; } @@ -297,7 +312,7 @@ export class DataService { let conceptCodesFromKeywords = this.findConceptCodesByKeywords(this.selectedKeywords); let selectedConceptCodes: Set; if (conceptCodesFromKeywords.size > 0 && conceptCodesFromTree.size > 0) { - selectedConceptCodes = DataService.intersection(conceptCodesFromTree, conceptCodesFromTree); + selectedConceptCodes = DataService.intersection(conceptCodesFromTree, conceptCodesFromKeywords); } else if (conceptCodesFromKeywords.size > 0) { selectedConceptCodes = conceptCodesFromKeywords; } else { From 5d7ea70e5bb4dfa3e3641374c66ff4bce5b50abb Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 30 Oct 2017 15:34:49 +0100 Subject: [PATCH 016/104] Improve tree fetching performance. --- .../datashowcase/TreeController.groovy | 6 ++- .../thehyve/datashowcase/TreeService.groovy | 18 +------ .../TreeNodeRepresentation.groovy | 7 ++- .../user-interface/src/app/models/concept.ts | 2 +- .../src/app/models/tree-node.ts | 8 +-- .../src/app/services/data.service.ts | 52 +++++++++++-------- 6 files changed, 47 insertions(+), 46 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TreeController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TreeController.groovy index 14b0133..5da664d 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TreeController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TreeController.groovy @@ -6,6 +6,7 @@ package nl.thehyve.datashowcase +import com.fasterxml.jackson.databind.ObjectMapper import org.springframework.beans.factory.annotation.Autowired class TreeController { @@ -15,6 +16,9 @@ class TreeController { TreeService treeService def index() { - respond tree_nodes: treeService.nodes + response.status = 200 + response.contentType = 'application/json' + response.characterEncoding = 'utf-8' + new ObjectMapper().writeValue(response.outputStream, [tree_nodes: treeService.nodes]) } } diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy index 6cdf8a3..95b345c 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/TreeService.groovy @@ -45,28 +45,14 @@ class TreeService { } ?: 0) } - @CompileStatic - static ConceptRepresentation map(Concept concept) { - if (concept == null) { - return null - } - new ConceptRepresentation( - conceptCode: concept.conceptCode, - label: concept.label, - labelLong: concept.labelLong, - labelNl: concept.labelNl, - labelNlLong: concept.labelNlLong, - variableType: concept.variableType - ) - } - @CompileStatic static TreeNodeRepresentation map(TreeNode node) { def result = new TreeNodeRepresentation( nodeType: node.nodeType, label: node.label, - concept: map(node.concept), path: node.path, + concept: node.concept?.conceptCode, + variableType: node.concept?.variableType ) if (node.children) { result.children = node.children.collect { diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/TreeNodeRepresentation.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/TreeNodeRepresentation.groovy index a1b895a..ca94deb 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/TreeNodeRepresentation.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/TreeNodeRepresentation.groovy @@ -6,8 +6,10 @@ package nl.thehyve.datashowcase.representation +import com.fasterxml.jackson.annotation.JsonIgnore import groovy.transform.CompileStatic import nl.thehyve.datashowcase.enumeration.NodeType +import nl.thehyve.datashowcase.enumeration.VariableType @CompileStatic class TreeNodeRepresentation { @@ -22,14 +24,17 @@ class TreeNodeRepresentation { */ String label + VariableType variableType + /** * The concept code of the concept the node refers to, if it is of type Concept. */ - ConceptRepresentation concept + String concept /** * The complete path of the node, including the name. */ + @JsonIgnore String path /** diff --git a/data-showcase/src/main/user-interface/src/app/models/concept.ts b/data-showcase/src/main/user-interface/src/app/models/concept.ts index e19db4b..e0df30a 100644 --- a/data-showcase/src/main/user-interface/src/app/models/concept.ts +++ b/data-showcase/src/main/user-interface/src/app/models/concept.ts @@ -4,7 +4,7 @@ * (see accompanying file LICENSE). */ -type VariableType = "Numerical"| "Categorical" | "Text" | "None"; +export type VariableType = "Numerical"| "Categorical" | "Text" | "None"; export class Concept { conceptCode: string; diff --git a/data-showcase/src/main/user-interface/src/app/models/tree-node.ts b/data-showcase/src/main/user-interface/src/app/models/tree-node.ts index a47d994..1aff759 100644 --- a/data-showcase/src/main/user-interface/src/app/models/tree-node.ts +++ b/data-showcase/src/main/user-interface/src/app/models/tree-node.ts @@ -4,16 +4,16 @@ * (see accompanying file LICENSE). */ -import {Concept} from "./concept"; +import { VariableType } from './concept'; -type NodeType = 'Domain' | 'Concept'; +export type NodeType = 'Domain' | 'Concept'; export class TreeNode { label: string; accumulativeItemCount: number; itemCount: number; - path: string; - concept: Concept; + concept: string; + variableType: VariableType; nodeType: NodeType; children: TreeNode[]; } diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 5c6c7dc..d14ffa4 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -92,7 +92,7 @@ export class DataService { .subscribe( (blobContent) => { let urlCreator = window.URL; - if (type == "NTR") { + if (type == 'NTR') { this.ntrLogoUrlSummary.next(urlCreator.createObjectURL(blobContent)); } else { this.vuLogoUrlSummary.next(urlCreator.createObjectURL(blobContent)); @@ -103,9 +103,13 @@ export class DataService { } private processTreeNodes(nodes: TreeNode[]): TreeNodeLib[] { + if (nodes == null) { + return []; + } let treeNodes: TreeNodeLib[] = []; for (let node of nodes) { - if (!(node.accumulativeItemCount == 0 && node.nodeType == "Domain" )) { + // filter out empty domains + if (!(node.accumulativeItemCount == 0 && node.nodeType == 'Domain')) { let newNode = this.processTreeNode(node); treeNodes.push(newNode); } @@ -114,34 +118,36 @@ export class DataService { } private processTreeNode(node: TreeNode): TreeNodeLib { + if (node == null) { + return null; + } // Add PrimeNG visual properties for tree nodes let newNode: TreeNodeLib = node; - // filter out empty domains - if (node.children) { - newNode.children = node.children.filter(value => !(value.accumulativeItemCount == 0 && value.nodeType == "Domain" )); - } let count = node.accumulativeItemCount ? node.accumulativeItemCount : 0; - let countStr = ' (' + count + ')'; - newNode.label = node.label + countStr; - - // If this newNode has children, drill down - if (node.children && node.children.length > 0) { - // Recurse - newNode.expandedIcon = 'fa-folder-open'; - newNode.collapsedIcon = 'fa-folder'; - newNode.icon = ''; - this.processTreeNodes(node.children); + newNode.label = `${node.label} (${count})`; + + // If this node has children, drill down + if (node.children) { + let children = this.processTreeNodes(node.children); + if (children.length > 0) { + newNode.children = children; + newNode.expandedIcon = 'fa-folder-open'; + newNode.collapsedIcon = 'fa-folder'; + newNode.icon = ''; + } else { + newNode.icon = 'fa-folder-o'; + } } else { - switch (node.concept.variableType) { - case "Text": - newNode['icon'] = 'icon-abc'; + switch (node.variableType) { + case 'Text': + newNode.icon = 'icon-abc'; break; - case "Numerical": - newNode['icon'] = 'icon-123'; + case 'Numerical': + newNode.icon = 'icon-123'; break; default: { - newNode['icon'] = 'fa-file-text'; + newNode.icon = 'fa-file-text'; break; } } @@ -218,7 +224,7 @@ export class DataService { } let conceptCodes = new Set(); if (treeNode.concept != null) { - conceptCodes.add(treeNode.concept.conceptCode); + conceptCodes.add(treeNode.concept); } if (treeNode.children != null) { treeNode.children.forEach((node: TreeNode) => From 8cd18d6823b4692132ebc799832ce13ea114e932 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 30 Oct 2017 15:35:54 +0100 Subject: [PATCH 017/104] Temporary fix for the global search. --- .../src/app/item-table/item-table.component.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts index 55ffc32..7034efd 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts @@ -17,10 +17,7 @@ export class ItemFilter implements PipeTransform { return filter && value ? value.filter(item => (item.name.toLocaleLowerCase().indexOf(filter) !== -1) || - (item.label.toLocaleLowerCase().indexOf(filter) !== -1) || - (item.labelLong.toLocaleLowerCase().indexOf(filter) !== -1) || - (item.labelNl && item.labelNl.toLocaleLowerCase().indexOf(filter) !== -1) || - (item.labelNlLong && item.labelNlLong.toLocaleLowerCase().indexOf(filter) !== -1) + (item.labelLong.toLocaleLowerCase().indexOf(filter) !== -1) ) : value; } } From 90e48f23b54096d787530a4ceb2ae7ff90fd9543 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Tue, 31 Oct 2017 15:27:38 +0100 Subject: [PATCH 018/104] Update angular dependencies --- data-showcase/src/main/user-interface/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data-showcase/src/main/user-interface/package.json b/data-showcase/src/main/user-interface/package.json index 10e654e..fddbb4e 100644 --- a/data-showcase/src/main/user-interface/package.json +++ b/data-showcase/src/main/user-interface/package.json @@ -28,9 +28,9 @@ "core-js": "^2.4.1", "file-saver": "^1.3.3", "font-awesome": "^4.7.0", - "primeng": "^4.1.3", + "primeng": "4.2.2", "roboto-fontface": "^0.8.0", - "rxjs": "^5.4.3", + "rxjs": "^5.5.2", "zone.js": "^0.8.4" }, "devDependencies": { @@ -41,7 +41,7 @@ "@types/jasminewd2": "~2.0.2", "@types/node": "~6.0.60", "codelyzer": "~3.0.1", - "cucumber": "^3.0.5", + "cucumber": "^3.1.0", "es6-promise": "^4.1.1", "jasmine-core": "~2.6.2", "jasmine-spec-reporter": "~4.1.0", @@ -52,7 +52,7 @@ "karma-jasmine": "~1.1.0", "karma-jasmine-html-reporter": "^0.2.2", "protractor": "~5.1.2", - "protractor-cucumber-framework": "^4.0.9", + "protractor-cucumber-framework": "^4.1.1", "request": "^2.81.0", "request-promise-native": "^1.0.4", "ts-node": "~3.0.4", From 61db5327391540d29e90c23982c5312ec2ea942e Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Tue, 31 Oct 2017 15:52:46 +0100 Subject: [PATCH 019/104] Change filters - move main text filter to a filter panel --- .../user-interface/src/app/app.component.css | 10 +--- .../user-interface/src/app/app.component.html | 3 +- .../src/app/app.component.spec.ts | 7 ++- .../main/user-interface/src/app/app.module.ts | 13 +++-- .../checkbox-filter.component.html | 32 ----------- .../checkbox-filter/checkbox-filter.module.ts | 30 ----------- .../checkbox-filter.component.css | 7 --- .../checkbox-filter.component.html | 30 +++++++++++ .../checkbox-filter.component.spec.ts | 8 +-- .../checkbox-filter.component.ts | 21 ++++++-- .../src/app/filters/filters.component.css | 26 +++++++++ .../src/app/filters/filters.component.html | 16 ++++++ .../src/app/filters/filters.component.spec.ts | 53 +++++++++++++++++++ .../src/app/filters/filters.component.ts | 20 +++++++ .../src/app/filters/filters.module.ts | 22 ++++++++ .../text-filter/text-filter.component.css | 7 +-- .../text-filter/text-filter.component.html | 15 ++++++ .../text-filter/text-filter.component.spec.ts | 8 +-- .../text-filter/text-filter.component.ts | 51 ++++++++++++++++++ .../src/app/services/data.service.ts | 10 ++++ .../text-filter/text-filter.component.html | 14 ----- .../app/text-filter/text-filter.component.ts | 29 ---------- .../src/app/text-filter/text-filter.module.ts | 22 -------- .../app/tree-nodes/tree-nodes.component.ts | 18 ++++++- .../src/main/user-interface/src/styles.css | 16 ++++-- 25 files changed, 313 insertions(+), 175 deletions(-) delete mode 100644 data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.html delete mode 100644 data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.module.ts rename data-showcase/src/main/user-interface/src/app/{ => filters}/checkbox-filter/checkbox-filter.component.css (71%) create mode 100644 data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html rename data-showcase/src/main/user-interface/src/app/{ => filters}/checkbox-filter/checkbox-filter.component.spec.ts (86%) rename data-showcase/src/main/user-interface/src/app/{ => filters}/checkbox-filter/checkbox-filter.component.ts (65%) create mode 100644 data-showcase/src/main/user-interface/src/app/filters/filters.component.css create mode 100644 data-showcase/src/main/user-interface/src/app/filters/filters.component.html create mode 100644 data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts create mode 100644 data-showcase/src/main/user-interface/src/app/filters/filters.component.ts create mode 100644 data-showcase/src/main/user-interface/src/app/filters/filters.module.ts rename data-showcase/src/main/user-interface/src/app/{ => filters}/text-filter/text-filter.component.css (65%) create mode 100644 data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html rename data-showcase/src/main/user-interface/src/app/{ => filters}/text-filter/text-filter.component.spec.ts (83%) create mode 100644 data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts delete mode 100644 data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.html delete mode 100644 data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.ts delete mode 100644 data-showcase/src/main/user-interface/src/app/text-filter/text-filter.module.ts diff --git a/data-showcase/src/main/user-interface/src/app/app.component.css b/data-showcase/src/main/user-interface/src/app/app.component.css index 81487bc..b107a96 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.css +++ b/data-showcase/src/main/user-interface/src/app/app.component.css @@ -66,13 +66,6 @@ color: #00889C; } -.ds-text-filter { - padding-right: 10px; - padding-left: 10px; - border-left: 2px solid darkgrey; - float: right; -} - .ds-shopping-cart { padding-right: 20px; padding-left: 20px; @@ -82,8 +75,7 @@ } .ds-logos, -.ds-shopping-cart, -.ds-text-filter{ +.ds-shopping-cart{ height: 100px; display:inline-block; } diff --git a/data-showcase/src/main/user-interface/src/app/app.component.html b/data-showcase/src/main/user-interface/src/app/app.component.html index 50d3375..12fc684 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.html +++ b/data-showcase/src/main/user-interface/src/app/app.component.html @@ -18,12 +18,11 @@
-
- +
diff --git a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts index 4bcd43b..9f9d724 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts @@ -8,13 +8,12 @@ import { TestBed, async } from '@angular/core/testing'; import { AppComponent } from './app.component'; import {TreeNodesComponent} from "./tree-nodes/tree-nodes.component"; -import {TextFilterComponent} from "./text-filter/text-filter.component"; import { AutoCompleteModule, DataTableModule, DialogModule, FieldsetModule, PanelModule, TreeModule } from "primeng/primeng"; import {FormsModule} from "@angular/forms"; -import {CheckboxFilterComponent} from "./checkbox-filter/checkbox-filter.component"; +import {CheckboxFilterComponent} from "./filters/checkbox-filter/checkbox-filter.component"; import {ListboxModule} from "primeng/components/listbox/listbox"; import {ItemFilter, ItemTableComponent} from "./item-table/item-table.component"; import {DataService} from "./services/data.service"; @@ -28,6 +27,7 @@ import {AppConfigMock} from "./config/app.config.mock"; import {ItemSummaryComponent} from "./item-summary/item-summary.component"; import {LogosComponent} from "./logos/logos.component"; import {PageRibbonComponent} from "./page-ribbon/page-ribbon.component"; +import {FiltersModule} from "./filters/filters.module"; describe('AppComponent', () => { beforeEach(async(() => { @@ -35,8 +35,6 @@ describe('AppComponent', () => { declarations: [ PageRibbonComponent, TreeNodesComponent, - TextFilterComponent, - CheckboxFilterComponent, ItemTableComponent, ItemFilter, ShoppingCartComponent, @@ -50,6 +48,7 @@ describe('AppComponent', () => { PanelModule, ListboxModule, TreeModule, + FiltersModule, FieldsetModule, DataTableModule, BrowserModule, diff --git a/data-showcase/src/main/user-interface/src/app/app.module.ts b/data-showcase/src/main/user-interface/src/app/app.module.ts index 891e822..12a91c9 100644 --- a/data-showcase/src/main/user-interface/src/app/app.module.ts +++ b/data-showcase/src/main/user-interface/src/app/app.module.ts @@ -14,14 +14,13 @@ import {DataService} from "./services/data.service"; import {ResourceService} from "./services/resource.service"; import {BrowserAnimationsModule} from "@angular/platform-browser/animations"; import {ItemTableModule} from "./item-table/item-table.module"; -import {CheckboxFilterModule} from "./checkbox-filter/checkbox-filter.module"; -import {TextFilterModule} from "./text-filter/text-filter.module"; import {HttpModule} from "@angular/http"; import {AppConfig} from "./config/app.config"; import {ShoppingCartModule} from "./shopping-cart/shopping-cart.module"; import {ItemSummaryModule} from "./item-summary/item-summary.module"; -import { LogosComponent } from './logos/logos.component'; -import { PageRibbonComponent } from './page-ribbon/page-ribbon.component'; +import {LogosComponent} from './logos/logos.component'; +import {PageRibbonComponent} from './page-ribbon/page-ribbon.component'; +import {FiltersModule} from "./filters/filters.module"; export function initConfig(config: AppConfig) { return () => config.load() @@ -38,9 +37,8 @@ export function initConfig(config: AppConfig) { HttpModule, BrowserAnimationsModule, TreeNodesModule, - CheckboxFilterModule, + FiltersModule, FormsModule, - TextFilterModule, ItemTableModule, ShoppingCartModule, ItemSummaryModule @@ -58,4 +56,5 @@ export function initConfig(config: AppConfig) { ], bootstrap: [AppComponent] }) -export class AppModule { } +export class AppModule { +} diff --git a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.html b/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.html deleted file mode 100644 index 1e6a305..0000000 --- a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.html +++ /dev/null @@ -1,32 +0,0 @@ - - -
- - -
-
- - - -
-
- - - -
-
- - - -
-
- - -
diff --git a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.module.ts b/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.module.ts deleted file mode 100644 index 2241c19..0000000 --- a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.module.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2017 The Hyve B.V. - * This file is distributed under the GNU Affero General Public License - * (see accompanying file LICENSE). - */ - -import { NgModule } from '@angular/core'; -import { CommonModule } from '@angular/common'; -import {FormsModule} from "@angular/forms"; -import { - PanelModule, FieldsetModule, AutoCompleteModule, CheckboxModule, DataListModule, - ListboxModule -} from "primeng/primeng"; -import {CheckboxFilterComponent} from "./checkbox-filter.component"; - -@NgModule({ - imports: [ - CommonModule, - FormsModule, - PanelModule, - FieldsetModule, - AutoCompleteModule, - CheckboxModule, - DataListModule, - ListboxModule - ], - declarations: [CheckboxFilterComponent], - exports: [CheckboxFilterComponent] -}) -export class CheckboxFilterModule { } diff --git a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.css b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.css similarity index 71% rename from data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.css rename to data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.css index 07694d4..d9dabcb 100644 --- a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.css +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.css @@ -4,13 +4,6 @@ * (see accompanying file LICENSE). */ -.checkbox-filter-container { - margin-left: 10px; - margin-right: 10px; - padding: 10px; - min-width: 800px; -} - .ds-data-format-item { background-color: transparent; border: none; diff --git a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html new file mode 100644 index 0000000..282f1cd --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html @@ -0,0 +1,30 @@ + +
+ +
+
+
+ + + +
+
+ + + +
+
+ + + +
+
diff --git a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.spec.ts b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.spec.ts similarity index 86% rename from data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.spec.ts rename to data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.spec.ts index 3551a35..166aedb 100644 --- a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.spec.ts @@ -9,12 +9,12 @@ import { CheckboxFilterComponent } from './checkbox-filter.component'; import {AutoCompleteModule, FieldsetModule, ListboxModule, PanelModule} from "primeng/primeng"; import {FormsModule} from "@angular/forms"; import {DataListModule} from "primeng/components/datalist/datalist"; -import {DataService} from "../services/data.service"; -import {ResourceService} from "../services/resource.service"; +import {DataService} from "../../services/data.service"; +import {ResourceService} from "../../services/resource.service"; import {BrowserAnimationsModule} from "@angular/platform-browser/animations"; import {HttpModule} from "@angular/http"; -import {AppConfig} from "../config/app.config"; -import {AppConfigMock} from "../config/app.config.mock"; +import {AppConfig} from "../../config/app.config"; +import {AppConfigMock} from "../../config/app.config.mock"; describe('CheckboxFilterComponent', () => { let component: CheckboxFilterComponent; diff --git a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.ts b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts similarity index 65% rename from data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.ts rename to data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts index 3ab4e4c..5b3f18d 100644 --- a/data-showcase/src/main/user-interface/src/app/checkbox-filter/checkbox-filter.component.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts @@ -5,9 +5,9 @@ */ import {Component, OnInit} from '@angular/core'; -import {DataService} from "../services/data.service"; -import {Item} from "../models/item"; -import { CheckboxOption } from '../models/CheckboxOption'; +import {DataService} from "../../services/data.service"; +import {Item} from "../../models/item"; +import { CheckboxOption } from '../../models/CheckboxOption'; @Component({ selector: 'app-checkbox-filter', @@ -16,6 +16,9 @@ import { CheckboxOption } from '../models/CheckboxOption'; }) export class CheckboxFilterComponent implements OnInit { + rerender: boolean = false; + spinner: boolean = false; + items: Item[]; keywords: CheckboxOption[] = []; projects: CheckboxOption[] = []; @@ -29,6 +32,18 @@ export class CheckboxFilterComponent implements OnInit { this.keywords = this.dataService.getKeywords(); this.projects = this.dataService.getProjects(); this.researchLines = this.dataService.getResearchLines(); + + /* workaround for primeng listbox not giving possibility to clear filters: reload checkboxFilters component */ + this.dataService.rerenderCheckboxFilters$.subscribe( + rerender => { + this.spinner = true; + this.rerender = rerender; + window.setTimeout((function () { + this.rerender = false; + this.spinner = false; + }).bind(this), 10); + } + ); } ngOnInit() { diff --git a/data-showcase/src/main/user-interface/src/app/filters/filters.component.css b/data-showcase/src/main/user-interface/src/app/filters/filters.component.css new file mode 100644 index 0000000..85452fc --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/filters/filters.component.css @@ -0,0 +1,26 @@ + +.filter-container { + margin-left: 10px; + margin-right: 10px; + /* padding: top right bottom left */ + padding: 0 10px 0 10px; + min-width: 800px; +} + +.filter-container-header { + color: white; + margin-top: -0.8em; +} + +.btn-primary-outline { + background-color: transparent; + border: none; + color: white; + height: 40px; + padding: 0; + box-shadow: none; + cursor: pointer; +} +.btn-primary-outline:active { + color: black; +} diff --git a/data-showcase/src/main/user-interface/src/app/filters/filters.component.html b/data-showcase/src/main/user-interface/src/app/filters/filters.component.html new file mode 100644 index 0000000..4da86f0 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/filters/filters.component.html @@ -0,0 +1,16 @@ +
+ + + + + + + + +
diff --git a/data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts b/data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts new file mode 100644 index 0000000..8e30eae --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts @@ -0,0 +1,53 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { FiltersComponent } from './filters.component'; +import {HttpModule} from "@angular/http"; +import {BrowserAnimationsModule} from "@angular/platform-browser/animations"; +import {AutoCompleteModule, DataListModule, FieldsetModule, ListboxModule, PanelModule} from "primeng/primeng"; +import {FormsModule} from "@angular/forms"; +import {TextFilterComponent} from "./text-filter/text-filter.component"; +import {CheckboxFilterComponent} from "./checkbox-filter/checkbox-filter.component"; +import {DataService} from "../services/data.service"; +import {ResourceService} from "../services/resource.service"; +import {AppConfig} from "../config/app.config"; +import {AppConfigMock} from "../config/app.config.mock"; + +describe('FiltersComponent', () => { + let component: FiltersComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [ FiltersComponent, TextFilterComponent, CheckboxFilterComponent ], + imports: [ + FormsModule, + PanelModule, + FieldsetModule, + AutoCompleteModule, + DataListModule, + ListboxModule, + BrowserAnimationsModule, + HttpModule + ], + providers: [ + DataService, + ResourceService, + { + provide: AppConfig, + useClass: AppConfigMock + } + ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(FiltersComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should be created', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/data-showcase/src/main/user-interface/src/app/filters/filters.component.ts b/data-showcase/src/main/user-interface/src/app/filters/filters.component.ts new file mode 100644 index 0000000..db4b0a0 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/filters/filters.component.ts @@ -0,0 +1,20 @@ +import { Component, OnInit } from '@angular/core'; +import {DataService} from "../services/data.service"; + +@Component({ + selector: 'app-filters', + templateUrl: './filters.component.html', + styleUrls: ['./filters.component.css'] +}) +export class FiltersComponent implements OnInit { + + constructor(public dataService: DataService) { + } + + ngOnInit() { + } + + clearFilters() { + this.dataService.clearAllFilters(); + } +} diff --git a/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts b/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts new file mode 100644 index 0000000..76b5186 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts @@ -0,0 +1,22 @@ +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import {FormsModule} from "@angular/forms"; +import {AutoCompleteModule, FieldsetModule, ListboxModule, PanelModule} from "primeng/primeng"; +import {FiltersComponent} from "./filters.component"; +import {CheckboxFilterComponent} from "./checkbox-filter/checkbox-filter.component"; +import {TextFilterComponent} from "./text-filter/text-filter.component"; + +@NgModule({ + imports: [ + CommonModule, + FormsModule, + AutoCompleteModule, + PanelModule, + ListboxModule, + FieldsetModule, + AutoCompleteModule + ], + declarations: [FiltersComponent, TextFilterComponent, CheckboxFilterComponent], + exports: [FiltersComponent] +}) +export class FiltersModule { } diff --git a/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.css b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css similarity index 65% rename from data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.css rename to data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css index c549c2d..fee1f48 100644 --- a/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.css +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css @@ -5,8 +5,9 @@ */ .text-filter-container { - margin-left: 10px; - height: 100px; - line-height: 100px; + text-align: center; + height: 50px; + line-height: 50px; white-space: nowrap; } +.text-filter-containe:focus {outline:none;} diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html new file mode 100644 index 0000000..ed87d3b --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html @@ -0,0 +1,15 @@ + + +
+ + + +
diff --git a/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.spec.ts b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.spec.ts similarity index 83% rename from data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.spec.ts rename to data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.spec.ts index 56cb701..58f632f 100644 --- a/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.spec.ts @@ -9,11 +9,11 @@ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { TextFilterComponent } from './text-filter.component'; import {AutoCompleteModule} from "primeng/primeng"; import {FormsModule} from "@angular/forms"; -import {ResourceService} from "../services/resource.service"; -import {DataService} from "../services/data.service"; +import {ResourceService} from "../../services/resource.service"; +import {DataService} from "../../services/data.service"; import {HttpModule} from "@angular/http"; -import {AppConfig} from "../config/app.config"; -import {AppConfigMock} from "../config/app.config.mock"; +import {AppConfig} from "../../config/app.config"; +import {AppConfigMock} from "../../config/app.config.mock"; describe('TextFilterComponent', () => { let component: TextFilterComponent; diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts new file mode 100644 index 0000000..a158873 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +import {Component, ElementRef, OnInit} from '@angular/core'; +import {DataService} from "../../services/data.service"; + +@Component({ + selector: 'app-text-filter', + templateUrl: './text-filter.component.html', + styleUrls: ['./text-filter.component.css'] +}) +export class TextFilterComponent implements OnInit { + + // value of the main text filter + globalFilter: string; + // the delay before triggering updating methods + delay: number; + + constructor(public dataService: DataService, + private element: ElementRef) { + this.dataService.globalFilter$.subscribe( + filter => { + this.globalFilter = filter; + }); + this.delay = 500; + } + + ngOnInit() { + } + + onFiltering(event) { + this.dataService.setGlobalFilter(this.globalFilter); + this.removePrimeNgAutocompleteLoader(); + } + + /* + PrimeNG library is attaching a spinner (.ui-autocomplete-loader) which is not automatically + removed after search is finished. + */ + removePrimeNgAutocompleteLoader() { + window.setTimeout((function () { + let loaderIcon = this.element.nativeElement.querySelector('.ui-autocomplete-loader'); + if (loaderIcon) { + loaderIcon.remove(); + } + }).bind(this), this.delay); + } +} diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index eb771ad..d52e35b 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -44,6 +44,10 @@ export class DataService { private globalFilterSource = new Subject(); public globalFilter$ = this.globalFilterSource.asObservable(); + // trigger checkboxFilters reload + private rerenderCheckboxFiltersSource = new Subject(); + public rerenderCheckboxFilters$ = this.rerenderCheckboxFiltersSource.asObservable(); + private selectedTreeNode: TreeNode = null; // selected checkboxes for keywords filter private selectedKeywords: string[] = []; @@ -275,6 +279,12 @@ export class DataService { this.setFilteredItems(); } + clearAllFilters() { + this.setGlobalFilter(null); + this.clearFilterValues() + this.rerenderCheckboxFiltersSource.next(true); + } + clearFilterValues() { this.selectedKeywords.length = 0; this.selectedResearchLines.length = 0; diff --git a/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.html b/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.html deleted file mode 100644 index b13180d..0000000 --- a/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.html +++ /dev/null @@ -1,14 +0,0 @@ - - -
- - -
diff --git a/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.ts b/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.ts deleted file mode 100644 index 5d9069f..0000000 --- a/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.component.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2017 The Hyve B.V. - * This file is distributed under the GNU Affero General Public License - * (see accompanying file LICENSE). - */ - -import { Component, OnInit } from '@angular/core'; -import {DataService} from "../services/data.service"; - -@Component({ - selector: 'app-text-filter', - templateUrl: './text-filter.component.html', - styleUrls: ['./text-filter.component.css'] -}) -export class TextFilterComponent implements OnInit { - - globalFilter: string; - - constructor(public dataService: DataService) { - } - - ngOnInit() { - } - - onFiltering(event) { - this.dataService.setGlobalFilter(this.globalFilter); - console.log('global filter: ', this.globalFilter); - } -} diff --git a/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.module.ts b/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.module.ts deleted file mode 100644 index 731a0ad..0000000 --- a/data-showcase/src/main/user-interface/src/app/text-filter/text-filter.module.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2017 The Hyve B.V. - * This file is distributed under the GNU Affero General Public License - * (see accompanying file LICENSE). - */ - -import { NgModule } from '@angular/core'; -import { CommonModule } from '@angular/common'; -import {AutoCompleteModule} from "primeng/primeng"; -import {TextFilterComponent} from "./text-filter.component"; -import {FormsModule} from "@angular/forms"; - -@NgModule({ - imports: [ - CommonModule, - FormsModule, - AutoCompleteModule - ], - declarations: [TextFilterComponent], - exports: [TextFilterComponent] -}) -export class TextFilterModule { } diff --git a/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.ts b/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.ts index 4f3903e..fe946a6 100644 --- a/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.ts +++ b/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.ts @@ -39,8 +39,10 @@ export class TreeNodesComponent implements OnInit, AfterViewInit { expansionStatus: any; // the search term in the text input box to filter the tree searchTerm: string; - + // currently selected node selectedNode: Object; + // the delay before triggering updating methods + delay: number; constructor(private element: ElementRef, public dataService: DataService) { @@ -50,6 +52,7 @@ export class TreeNodesComponent implements OnInit, AfterViewInit { treeNode: null }; this.treeNodes = dataService.treeNodes; + this.delay = 500; } ngOnInit() { @@ -137,6 +140,7 @@ export class TreeNodesComponent implements OnInit, AfterViewInit { this.treeNodes = this.filterTreeNodes(this.dataService.treeNodes, 'label', filterWord).matchingTreeNodes; console.log('found tree nodes: ', this.treeNodes); } + this.removePrimeNgAutocompleteLoader(); } update() { @@ -170,4 +174,16 @@ export class TreeNodesComponent implements OnInit, AfterViewInit { } } + /* + PrimeNG library is attaching a spinner (.ui-autocomplete-loader) which is not automatically + removed after search is finished. + */ + removePrimeNgAutocompleteLoader() { + window.setTimeout((function () { + let loaderIcon = this.element.nativeElement.querySelector('.ui-autocomplete-loader'); + if (loaderIcon) { + loaderIcon.remove(); + } + }).bind(this), this.delay); + } } diff --git a/data-showcase/src/main/user-interface/src/styles.css b/data-showcase/src/main/user-interface/src/styles.css index e67047d..69cc7ca 100644 --- a/data-showcase/src/main/user-interface/src/styles.css +++ b/data-showcase/src/main/user-interface/src/styles.css @@ -73,7 +73,7 @@ body .ds-spinner.loading { * Change the font size of the legend - checkbox-filter */ legend { - font-size: larger; + font-size: medium; } /* @@ -159,6 +159,9 @@ body .ui-panel .ui-panel-titlebar .ui-panel-titlebar-icon { */ body .ui-panel .ui-panel-titlebar { background-color: #00889C; + height: 40px; + line-height: 40px; + padding-top: 0px } body .ui-panel, body .ui-fieldset, @@ -174,7 +177,7 @@ body .ui-datatable { box-shadow: 0 2px 2px 0 rgba(0,0,0,0.14),0 3px 1px -2px rgba(0,0,0,0.12),0 1px 5px 0 rgba(0,0,0,0.2); } -body .input:focus { +body .ui-inputtext:focus { border-color: #00889C !important; } @@ -274,10 +277,15 @@ body .ui-inputgroup .ui-inputtext.ui-widget.ui-state-default.ui-corner-all { height: 36px; } -.text-filter-container .ui-inputtext.ui-widget.ui-state-default.ui-corner-all { - width: 500px; +.text-filter-container .ui-inputtext { + width: 550px; height: 36px; } + +.filter-container .ui-panel .ui-panel-content { + padding-top: 5px; + padding-bottom: 5px; +} /*.text-filter-container .ui-inputtext.ui-widget.ui-state-default.ui-corner-all:focus {*/ /*width: 500px;*/ /*}*/ From 29bcd65b7e07cc922a801b9e518fadf8d98af0cb Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 1 Nov 2017 14:18:55 +0100 Subject: [PATCH 020/104] Fix padding of the search box --- data-showcase/src/main/user-interface/src/styles.css | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/styles.css b/data-showcase/src/main/user-interface/src/styles.css index 69cc7ca..7eb05ad 100644 --- a/data-showcase/src/main/user-interface/src/styles.css +++ b/data-showcase/src/main/user-interface/src/styles.css @@ -277,15 +277,14 @@ body .ui-inputgroup .ui-inputtext.ui-widget.ui-state-default.ui-corner-all { height: 36px; } +/* Global text filter */ .text-filter-container .ui-inputtext { width: 550px; height: 36px; + padding: 0; } .filter-container .ui-panel .ui-panel-content { padding-top: 5px; padding-bottom: 5px; } -/*.text-filter-container .ui-inputtext.ui-widget.ui-state-default.ui-corner-all:focus {*/ - /*width: 500px;*/ -/*}*/ From 0dbe8ca8ea36604533c9675c5eb49cdb4fd3967c Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 1 Nov 2017 14:51:56 +0100 Subject: [PATCH 021/104] Remove 'select all' checkbox and disable filter, when no filter element available --- .../checkbox-filter/checkbox-filter.component.html | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html index 282f1cd..1211d68 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html @@ -11,20 +11,22 @@
+ [checkbox]="keywords.length > 0" [disabled]="keywords.length < 1" + [metaKeySelection]=false (onChange)="updateFilters()">
+ [checkbox]="researchLines.length > 0" [disabled]="researchLines.length < 1" + [metaKeySelection]=false (onChange)="updateFilters(); updateProjects();">
+ [checkbox]="projects.length > 0" [disabled]="projects.length < 1" + [metaKeySelection]=false (onChange)="updateFilters()">
From 510f4b5ef70598ceb37de1d77c9b0f003594ea5c Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 1 Nov 2017 16:12:55 +0100 Subject: [PATCH 022/104] Fix table column wrapping for the shopping-cart and item-table --- .../src/app/item-table/item-table.component.html | 4 ++-- .../src/app/shopping-cart/shopping-cart.component.css | 2 +- .../src/app/shopping-cart/shopping-cart.component.html | 4 ++-- data-showcase/src/main/user-interface/src/styles.css | 10 ++++++++++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html index 9322697..023c233 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html @@ -13,7 +13,7 @@ [resizableColumns]="true" [(selection)]="itemsSelection" [loading]="dataService.loadingItems" loadingIcon="fa fa-spin fa-refresh fa-fw"> - + @@ -35,7 +35,7 @@ [sortable]="true"> - + diff --git a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.css b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.css index da6509e..15ad46e 100644 --- a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.css +++ b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.css @@ -10,7 +10,7 @@ cursor:pointer; } -.table_content { +.table-content { text-align: left !important; } .ui-dialog-footer { diff --git a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html index b80bac5..441fa13 100644 --- a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html +++ b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html @@ -4,7 +4,7 @@ ~ (see accompanying file LICENSE). --> -
Shopping Cart is empty. @@ -13,7 +13,7 @@ diff --git a/data-showcase/src/main/user-interface/src/styles.css b/data-showcase/src/main/user-interface/src/styles.css index 7eb05ad..65761cf 100644 --- a/data-showcase/src/main/user-interface/src/styles.css +++ b/data-showcase/src/main/user-interface/src/styles.css @@ -288,3 +288,13 @@ body .ui-inputgroup .ui-inputtext.ui-widget.ui-state-default.ui-corner-all { padding-top: 5px; padding-bottom: 5px; } + +/* Table column wrapping */ +.ui-datatable, +.ui-datatable-resizable thead th, +.ui-datatable-resizable tbody td, +.ui-datatable-resizable tfoot td { + word-wrap: break-word; + word-break: break-all; + white-space: normal; +} From 000a0a86b273fff0681c6ad2176e3762e783cbfe Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 1 Nov 2017 17:05:47 +0100 Subject: [PATCH 023/104] Change item-table counts. Add page number of pages. --- .../src/app/item-table/item-table.component.html | 6 ++++-- .../src/app/item-table/item-table.component.ts | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html index 023c233..5fe969b 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html @@ -8,7 +8,7 @@
- Items selected: {{itemsSelection? itemsSelection.length : 0}}. Total: {{items? items.length : 0}} + Items selected: {{itemsSelection? itemsSelection.length : 0}}. + Total results in table: {{items? items.length : 0}}. + Number of pages: {{pageCount()}}.
diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts index a1d7c0a..b7bd8a7 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts @@ -32,8 +32,10 @@ export class ItemTableComponent implements OnInit { filterValue: string; items: Item[]; itemsSelection: Item[]; + rowsPerPage: number; constructor(public dataService: DataService) { + this.rowsPerPage = 8; this.items = this.dataService.filteredItems; this.dataService.itemsSelection$.subscribe( selection => { @@ -57,4 +59,8 @@ export class ItemTableComponent implements OnInit { showSummary(item: Item){ this.dataService.displayPopup(item); } + + pageCount(): number { + return Math.ceil(this.items.length / this.rowsPerPage) + } } From 981c9b85b96164e10f7deadd88e66af1dedf6fc1 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 1 Nov 2017 17:06:25 +0100 Subject: [PATCH 024/104] Clear items selection on filter change --- .../src/main/user-interface/src/app/services/data.service.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index d52e35b..1753815 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -276,6 +276,7 @@ export class DataService { this.selectedKeywords = selectedKeywords; this.selectedProjects = selectedProjects; this.selectedResearchLines = selectedResearchLines; + this.clearItemsSelection(); this.setFilteredItems(); } From 0abc52c145e41220663fbcaf9d1d559a7b568984 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 2 Nov 2017 09:41:24 +0100 Subject: [PATCH 025/104] Add information bar --- .../user-interface/src/app/app.component.css | 8 ++++++ .../user-interface/src/app/app.component.html | 1 + .../main/user-interface/src/app/app.module.ts | 4 ++- .../src/app/info/info.component.css | 21 ++++++++++++++++ .../src/app/info/info.component.html | 20 +++++++++++++++ .../src/app/info/info.component.spec.ts | 25 +++++++++++++++++++ .../src/app/info/info.component.ts | 21 ++++++++++++++++ .../src/app/info/info.module.ts | 19 ++++++++++++++ .../shopping-cart/shopping-cart.component.css | 1 + 9 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 data-showcase/src/main/user-interface/src/app/info/info.component.css create mode 100644 data-showcase/src/main/user-interface/src/app/info/info.component.html create mode 100644 data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts create mode 100644 data-showcase/src/main/user-interface/src/app/info/info.component.ts create mode 100644 data-showcase/src/main/user-interface/src/app/info/info.module.ts diff --git a/data-showcase/src/main/user-interface/src/app/app.component.css b/data-showcase/src/main/user-interface/src/app/app.component.css index b107a96..9109ee4 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.css +++ b/data-showcase/src/main/user-interface/src/app/app.component.css @@ -74,6 +74,14 @@ float: right; } +.ds-info { + padding-right: 20px; + padding-left: 20px; + padding-top: 25px; + float: right; +} + +.ds-info, .ds-logos, .ds-shopping-cart{ height: 100px; diff --git a/data-showcase/src/main/user-interface/src/app/app.component.html b/data-showcase/src/main/user-interface/src/app/app.component.html index 12fc684..187181d 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.html +++ b/data-showcase/src/main/user-interface/src/app/app.component.html @@ -18,6 +18,7 @@
+
diff --git a/data-showcase/src/main/user-interface/src/app/app.module.ts b/data-showcase/src/main/user-interface/src/app/app.module.ts index 12a91c9..65e6712 100644 --- a/data-showcase/src/main/user-interface/src/app/app.module.ts +++ b/data-showcase/src/main/user-interface/src/app/app.module.ts @@ -21,6 +21,7 @@ import {ItemSummaryModule} from "./item-summary/item-summary.module"; import {LogosComponent} from './logos/logos.component'; import {PageRibbonComponent} from './page-ribbon/page-ribbon.component'; import {FiltersModule} from "./filters/filters.module"; +import {InfoModule} from "./info/info.module"; export function initConfig(config: AppConfig) { return () => config.load() @@ -41,7 +42,8 @@ export function initConfig(config: AppConfig) { FormsModule, ItemTableModule, ShoppingCartModule, - ItemSummaryModule + ItemSummaryModule, + InfoModule ], providers: [ ResourceService, diff --git a/data-showcase/src/main/user-interface/src/app/info/info.component.css b/data-showcase/src/main/user-interface/src/app/info/info.component.css new file mode 100644 index 0000000..81c6aa1 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/info/info.component.css @@ -0,0 +1,21 @@ +.info-button { + font-size:20px; + padding: 0; + cursor:pointer; + box-shadow: none; +} + +.info-header { + font-weight:normal; + color: #00889C; +} + +.info-bar { + height: 200px; +} + +.info-cancel-button { + position: absolute; + bottom: 40px; + left: 20px; +} diff --git a/data-showcase/src/main/user-interface/src/app/info/info.component.html b/data-showcase/src/main/user-interface/src/app/info/info.component.html new file mode 100644 index 0000000..8db6140 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/info/info.component.html @@ -0,0 +1,20 @@ + + + +

Help information

+ This bar will contain filtering instructions and other necessary information. +
+ + + + + + + + diff --git a/data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts b/data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts new file mode 100644 index 0000000..8e2eb7c --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts @@ -0,0 +1,25 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { InfoComponent } from './info.component'; + +describe('InfoComponent', () => { + let component: InfoComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [ InfoComponent ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(InfoComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should be created', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/data-showcase/src/main/user-interface/src/app/info/info.component.ts b/data-showcase/src/main/user-interface/src/app/info/info.component.ts new file mode 100644 index 0000000..32980a4 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/info/info.component.ts @@ -0,0 +1,21 @@ +import { Component, OnInit } from '@angular/core'; + +@Component({ + selector: 'app-info', + templateUrl: './info.component.html', + styleUrls: ['./info.component.css'] +}) +export class InfoComponent implements OnInit { + + display: boolean = false; + + constructor() { } + + ngOnInit() { + } + + showInfoDialog() { + this.display = true; + } + +} diff --git a/data-showcase/src/main/user-interface/src/app/info/info.module.ts b/data-showcase/src/main/user-interface/src/app/info/info.module.ts new file mode 100644 index 0000000..4e179d2 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/info/info.module.ts @@ -0,0 +1,19 @@ +import { NgModule } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import {InfoComponent} from "./info.component"; +import {FormsModule} from "@angular/forms"; +import {DialogModule, PanelModule} from "primeng/primeng"; +import {SidebarModule} from "primeng/components/sidebar/sidebar"; + +@NgModule({ + imports: [ + CommonModule, + FormsModule, + SidebarModule, + PanelModule, + DialogModule + ], + declarations: [InfoComponent], + exports: [InfoComponent] +}) +export class InfoModule { } diff --git a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.css b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.css index 15ad46e..dc51a15 100644 --- a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.css +++ b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.css @@ -8,6 +8,7 @@ font-size:20px; padding: 0; cursor:pointer; + box-shadow: none; } .table-content { From a173607628d8cce0856bf26609b2c73e96186cb8 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 2 Nov 2017 09:47:09 +0100 Subject: [PATCH 026/104] Fix ui e2e tests --- .../user-interface/src/app/app.component.spec.ts | 4 +++- .../src/app/info/info.component.spec.ts | 14 +++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts index 9f9d724..964c689 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts @@ -28,6 +28,7 @@ import {ItemSummaryComponent} from "./item-summary/item-summary.component"; import {LogosComponent} from "./logos/logos.component"; import {PageRibbonComponent} from "./page-ribbon/page-ribbon.component"; import {FiltersModule} from "./filters/filters.module"; +import {InfoModule} from "./info/info.module"; describe('AppComponent', () => { beforeEach(async(() => { @@ -54,7 +55,8 @@ describe('AppComponent', () => { BrowserModule, BrowserAnimationsModule, DialogModule, - HttpModule + HttpModule, + InfoModule ], providers: [ DataService, diff --git a/data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts b/data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts index 8e2eb7c..d5e98fd 100644 --- a/data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts @@ -1,6 +1,11 @@ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { InfoComponent } from './info.component'; +import {FormsModule} from "@angular/forms"; +import {PanelModule} from "primeng/primeng"; +import {SidebarModule} from "primeng/components/sidebar/sidebar"; +import {BrowserModule} from "@angular/platform-browser"; +import {BrowserAnimationsModule} from "@angular/platform-browser/animations"; describe('InfoComponent', () => { let component: InfoComponent; @@ -8,7 +13,14 @@ describe('InfoComponent', () => { beforeEach(async(() => { TestBed.configureTestingModule({ - declarations: [ InfoComponent ] + declarations: [ InfoComponent ], + imports: [ + FormsModule, + PanelModule, + SidebarModule, + BrowserModule, + BrowserAnimationsModule, + ] }) .compileComponents(); })); From e48e3895a7e5289f749a2e8294946ce610e5b7c0 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 2 Nov 2017 13:01:00 +0100 Subject: [PATCH 027/104] Remove keywords checkbox filter --- .../checkbox-filter/checkbox-filter.component.html | 11 ++--------- .../checkbox-filter/checkbox-filter.component.ts | 4 ---- .../user-interface/src/app/services/data.service.ts | 8 +------- 3 files changed, 3 insertions(+), 20 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html index 1211d68..c0c718c 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html @@ -8,21 +8,14 @@ [class.loading]="spinner">
-
- - - -
-
+
-
+
Date: Sun, 5 Nov 2017 19:33:26 +0100 Subject: [PATCH 028/104] Move filtering of items to back-end --- .../datashowcase/ItemController.groovy | 26 +- .../thehyve/datashowcase/ItemService.groovy | 57 +++- .../representation/ItemRepresentation.groovy | 4 + .../main/user-interface/src/app/app.module.ts | 2 + .../checkbox-filter.component.ts | 6 +- .../src/app/filters/filters.module.ts | 5 +- .../text-filter/text-filter.component.html | 5 +- .../text-filter/text-filter.component.ts | 14 +- .../app/item-table/item-table.component.html | 2 +- .../app/item-table/item-table.component.ts | 2 +- .../user-interface/src/app/models/item.ts | 2 +- .../src/app/services/data.service.ts | 250 ++++++++---------- .../src/app/services/resource.service.ts | 9 +- .../services/search-parser.service.spec.ts | 15 ++ .../src/app/services/search-parser.service.ts | 18 ++ .../main/user-interface/src/tsconfig.app.json | 2 +- .../src/main/user-interface/tsconfig.json | 5 +- 17 files changed, 253 insertions(+), 171 deletions(-) create mode 100644 data-showcase/src/main/user-interface/src/app/services/search-parser.service.spec.ts create mode 100644 data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index 2ebdd22..78c656c 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -7,6 +7,8 @@ package nl.thehyve.datashowcase import com.fasterxml.jackson.databind.ObjectMapper +import grails.converters.JSON +import org.grails.web.json.JSONArray import org.springframework.beans.factory.annotation.Autowired class ItemController { @@ -18,14 +20,26 @@ class ItemController { /** * Fetches all items. - * + * TODO description * @return the list of items as JSON. */ def index() { + + Set concepts = parseParams(params.conceptCodes) + Set projects = parseParams(params.projects) + Set linesOfResearch = parseParams(params.linesOfResearch) + Set searchQuery = parseParams(params.searchQuery) + response.status = 200 response.contentType = 'application/json' response.characterEncoding = 'utf-8' - new ObjectMapper().writeValue(response.outputStream, [items: itemService.items]) + Object value + if (concepts || projects || linesOfResearch || searchQuery){ + value = [items: itemService.getItems(concepts, projects, linesOfResearch, searchQuery)] + } else { + value = [items: itemService.items] + } + new ObjectMapper().writeValue(response.outputStream, value) } /** @@ -40,4 +54,12 @@ class ItemController { respond itemService.getItem(id) } + private static Set parseParams(String jsonString){ + try{ + JSONArray json = JSON.parse(jsonString) + return json.collect{ "'" + it + "'"} as Set + } catch (Exception e) { + return null + } + } } diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index 0351329..d2d221c 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -33,6 +33,8 @@ class ItemService { SessionFactory sessionFactory + private final String EMPTY_CONDITION = '1=1' + @CompileStatic static ItemRepresentation map(Map itemData) { new ItemRepresentation( @@ -50,7 +52,7 @@ class ItemService { @Cacheable('items') @Transactional(readOnly = true) List getItems() { - def stopWatch = new StopWatch('Fetch items') + def stopWatch = new StopWatch('Fetch all items') stopWatch.start('Retrieve from database') def session = sessionFactory.openStatelessSession() def items = session.createQuery( @@ -63,10 +65,12 @@ class ItemService { c.conceptCode as conceptCode, c.labelLong as labelLong, c.variableType as type, - p.name as projectName + p.name as projectName, + rl.name as lineOfResearch from Item as i join i.concept c join i.project p + join p.lineOfResearch rl ${dataShowcaseEnvironment.internalInstance ? '' : 'where i.publicItem = true' } @@ -83,6 +87,51 @@ class ItemService { result } + @Transactional(readOnly = true) + List getItems(Set concepts, Set projects, Set linesOfResearch, Set searchQuery) { + String sqlSearchQueryChunk = toSQL(searchQuery) + def stopWatch = new StopWatch('Fetch filtered items') + stopWatch.start('Retrieve from database') + def session = sessionFactory.openStatelessSession() + + def items = session.createQuery( + """ + select + i.id as id, + i.name as name, + i.publicItem as publicItem, + i.itemPath as itemPath, + c.conceptCode as conceptCode, + c.labelLong as labelLong, + c.variableType as type, + p.name as projectName, + rl.name as lineOfResearch + from Item as i + join i.concept c + join (select p.id, p.name, rl.name from Project as P join p.lineOfResearch rl) on i.project = p + where + ${concepts ? 'c.conceptCode IN ' + concepts : EMPTY_CONDITION} + AND ${projects ? 'p.name IN $projects' : EMPTY_CONDITION} + AND ${linesOfResearch ? 'rl.name IN $linesOfResearch' : EMPTY_CONDITION} + AND ${dataShowcaseEnvironment.internalInstance ? + EMPTY_CONDITION : 'i.publicItem = true' + } + AND $sqlSearchQueryChunk + """ + ).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) + .list() as List + + stopWatch.stop() + stopWatch.start('Map to representations') + def result = items.collect { Map itemData -> + map(itemData) + } + stopWatch.stop() + log.info "Filtered items fetched.\n${stopWatch.prettyPrint()}" + result + + } + @CacheEvict(value = 'items', allEntries = true) void clearItemsCache() { log.info "Clear items cache." @@ -143,4 +192,8 @@ class ItemService { throw new ResourceNotFoundException('Item not found') } + private String toSQL(Set query) { + return EMPTY_CONDITION + } + } diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy index ad2c6ed..0ad5752 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy @@ -68,4 +68,8 @@ class ItemRepresentation { */ String project + /** + * The name of the research line + */ + String lineOfResearch } diff --git a/data-showcase/src/main/user-interface/src/app/app.module.ts b/data-showcase/src/main/user-interface/src/app/app.module.ts index 65e6712..f7f9b76 100644 --- a/data-showcase/src/main/user-interface/src/app/app.module.ts +++ b/data-showcase/src/main/user-interface/src/app/app.module.ts @@ -22,6 +22,7 @@ import {LogosComponent} from './logos/logos.component'; import {PageRibbonComponent} from './page-ribbon/page-ribbon.component'; import {FiltersModule} from "./filters/filters.module"; import {InfoModule} from "./info/info.module"; +import {SearchParserService} from "./services/search-parser.service"; export function initConfig(config: AppConfig) { return () => config.load() @@ -48,6 +49,7 @@ export function initConfig(config: AppConfig) { providers: [ ResourceService, DataService, + SearchParserService, AppConfig, { provide: APP_INITIALIZER, diff --git a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts index 8ad473a..9029694 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts @@ -19,16 +19,14 @@ export class CheckboxFilterComponent implements OnInit { rerender: boolean = false; spinner: boolean = false; - items: Item[]; projects: CheckboxOption[] = []; researchLines: CheckboxOption[] = []; selectedProjects: string[] = []; selectedResearchLines: string[] = []; constructor(public dataService: DataService) { - this.items = this.dataService.getItems(); - this.projects = this.dataService.getProjects(); - this.researchLines = this.dataService.getResearchLines(); + this.projects = this.dataService.projects; + this.researchLines = this.dataService.linesOfResearch; /* workaround for primeng listbox not giving possibility to clear filters: reload checkboxFilters component */ this.dataService.rerenderCheckboxFilters$.subscribe( diff --git a/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts b/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts index 76b5186..c951212 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts @@ -1,7 +1,7 @@ import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; import {FormsModule} from "@angular/forms"; -import {AutoCompleteModule, FieldsetModule, ListboxModule, PanelModule} from "primeng/primeng"; +import {AutoCompleteModule, ButtonModule, FieldsetModule, ListboxModule, PanelModule} from "primeng/primeng"; import {FiltersComponent} from "./filters.component"; import {CheckboxFilterComponent} from "./checkbox-filter/checkbox-filter.component"; import {TextFilterComponent} from "./text-filter/text-filter.component"; @@ -14,7 +14,8 @@ import {TextFilterComponent} from "./text-filter/text-filter.component"; PanelModule, ListboxModule, FieldsetModule, - AutoCompleteModule + AutoCompleteModule, + ButtonModule ], declarations: [FiltersComponent, TextFilterComponent, CheckboxFilterComponent], exports: [FiltersComponent] diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html index ed87d3b..196da1a 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html @@ -6,10 +6,11 @@
- + (completeMethod)="removePrimeNgAutocompleteLoader()"> +
diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts index a158873..d27798a 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts @@ -6,6 +6,7 @@ import {Component, ElementRef, OnInit} from '@angular/core'; import {DataService} from "../../services/data.service"; +import {SearchParserService} from "../../services/search-parser.service"; @Component({ selector: 'app-text-filter', @@ -15,25 +16,26 @@ import {DataService} from "../../services/data.service"; export class TextFilterComponent implements OnInit { // value of the main text filter - globalFilter: string; + textFilter: string; // the delay before triggering updating methods delay: number; constructor(public dataService: DataService, + public searchParserService: SearchParserService, private element: ElementRef) { - this.dataService.globalFilter$.subscribe( + this.dataService.textFilterInput$.subscribe( filter => { - this.globalFilter = filter; + this.textFilter = filter; }); - this.delay = 500; + this.delay = 0; } ngOnInit() { } onFiltering(event) { - this.dataService.setGlobalFilter(this.globalFilter); - this.removePrimeNgAutocompleteLoader(); + let jsonQuery = this.searchParserService.parse(this.textFilter); + this.dataService.setJsonSearchQuery(jsonQuery); } /* diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html index 5fe969b..ac7fe74 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html @@ -27,7 +27,7 @@ [sortable]="true"> - diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts index b7bd8a7..fc2e739 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts @@ -42,7 +42,7 @@ export class ItemTableComponent implements OnInit { this.itemsSelection = selection; } ); - this.dataService.globalFilter$.subscribe( + this.dataService.textFilterInput$.subscribe( filter => { this.filterValue = filter; } diff --git a/data-showcase/src/main/user-interface/src/app/models/item.ts b/data-showcase/src/main/user-interface/src/app/models/item.ts index df0772e..e85ec1c 100644 --- a/data-showcase/src/main/user-interface/src/app/models/item.ts +++ b/data-showcase/src/main/user-interface/src/app/models/item.ts @@ -13,7 +13,7 @@ export class Item { itemPath: string; type: string; project: string; - researchLine: string; + lineOfResearch: string; concept: string; summary: ItemSummary; label: string; diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 9376ce5..0c0f72b 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -32,42 +32,34 @@ export class DataService { public loadingItems: boolean = false; // filtered list of items based on selected node and selected checkbox filters public filteredItems: Item[] = []; - // items available for currently selected node - private itemsPerNode: Item[] = []; // items selected in the itemTable private itemsSelectionSource = new Subject(); public itemsSelection$ = this.itemsSelectionSource.asObservable(); // items added to the shopping cart public shoppingCartItems = new BehaviorSubject([]); - // global text filter - private globalFilterSource = new Subject(); - public globalFilter$ = this.globalFilterSource.asObservable(); + // text filter input + private textFilterInputSource = new Subject(); + public textFilterInput$ = this.textFilterInputSource.asObservable(); + + // JSON search query + private jsonSearchQuery: JSON = null; // trigger checkboxFilters reload private rerenderCheckboxFiltersSource = new Subject(); public rerenderCheckboxFilters$ = this.rerenderCheckboxFiltersSource.asObservable(); private selectedTreeNode: TreeNode = null; - // selected checkboxes for keywords filter - private selectedKeywords: string[] = []; // selected checkboxes for projects filter private selectedProjects: string[] = []; // selected checkboxes for research lines filter private selectedResearchLines: string[] = []; - - // list of keywords available for current item list - private keywords: CheckboxOption[] = []; // list of project names available for current item list - private projects: CheckboxOption[] = []; + public projects: CheckboxOption[] = []; // list of research lines available for current item list - private researchLines: CheckboxOption[] = []; - - // list of all concepts - private concepts: Concept[] = []; - + public linesOfResearch: CheckboxOption[] = []; // list of all projects - private availableProjects: Project[] = []; + private allProjects: Project[] = []; // item summary popup visibility private itemSummaryVisibleSource = new Subject(); @@ -86,29 +78,12 @@ export class DataService { public environment$ = this.environmentSource.asObservable(); constructor(private resourceService: ResourceService) { - this.updateAvailableProjects(); - this.updateConcepts(); - this.updateNodes(); - this.updateItems(); - this.setFilteredItems(); + this.fetchAllProjects(); + this.fetchAllTreeNodes(); + this.fetchItems(); this.setEnvironment(); } - loadLogo(type: string) { - this.resourceService.getLogo(type) - .subscribe( - (blobContent) => { - let urlCreator = window.URL; - if (type == 'NTR') { - this.ntrLogoUrlSummary.next(urlCreator.createObjectURL(blobContent)); - } else { - this.vuLogoUrlSummary.next(urlCreator.createObjectURL(blobContent)); - } - }, - err => console.error(err) - ); - } - private processTreeNodes(nodes: TreeNode[]): TreeNodeLib[] { if (nodes == null) { return []; @@ -162,30 +137,17 @@ export class DataService { return newNode; } - updateAvailableProjects() { + fetchAllProjects() { this.resourceService.getProjects() .subscribe( (projects: Project[]) => { - this.availableProjects = projects; + this.allProjects = projects; }, err => console.error(err) ); } - updateConcepts() { - this.resourceService.getConcepts() - .subscribe((concepts: Concept[]) => { - this.concepts = concepts; - let keywords: string[] = [].concat.apply([], this.concepts.map((concept: Concept) => concept.keywords)); - this.keywords.length = 0; - keywords.forEach(keyword => this.keywords.push({label: keyword, value: keyword} as CheckboxOption)); - console.info(`Loaded ${this.keywords.length} key words.`); - }, - err => console.error(err) - ); - } - - updateNodes() { + fetchAllTreeNodes() { this.loadingTreeNodes = 'loading'; // Retrieve all tree nodes this.resourceService.getTreeNodes() @@ -201,28 +163,31 @@ export class DataService { ); } - updateItems() { + fetchItems() { + let t1 = new Date(); + console.debug(`Fetching items ...`); this.loadingItems = true; - this.itemsPerNode.length = 0; this.items.length = 0; - this.resourceService.getItems() - .subscribe( + let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); + let codes = Array.from(selectedConceptCodes); + this.resourceService.getItems( + codes, this.selectedProjects, this.selectedResearchLines, this.jsonSearchQuery).subscribe( (items: Item[]) => { for (let item of items) { - if (this.availableProjects) { - item['researchLine'] = this.availableProjects.find(p => p.name == item['project']).lineOfResearch; + if (this.allProjects) { + item['lineOfResearch'] = this.allProjects.find(p => p.name == item['project']).lineOfResearch; } - - this.itemsPerNode.push(item); + this.filteredItems.push(item); this.items.push(item); } - this.setFilteredItems(); this.getUniqueFilterValues(); console.info(`Loaded ${items.length} items ...`); - this.loadingItems = false; }, err => console.error(err) ); + let t2 = new Date(); + console.info(`Found ${this.items.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); + this.loadingItems = false; } static treeConceptCodes(treeNode: TreeNode) : Set { @@ -251,22 +216,15 @@ export class DataService { updateItemTable() { this.items.length = 0; this.clearItemsSelection(); - this.clearFilterValues(); - if (this.selectedTreeNode == null) { - this.itemsPerNode.forEach(item => this.items.push(item)) - } else { - let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); - let nodeItems = this.itemsPerNode.filter(item => selectedConceptCodes.has(item.concept)); - nodeItems.forEach(item => this.items.push(item)) - } - this.setFilteredItems(); + this.clearCheckboxFilterValues(); + this.fetchItems(); this.getUniqueFilterValues(); } updateProjectsForResearchLines() { this.selectedProjects.length = 0; - this.setFilteredItems(); this.projects.length = 0; + this.fetchItems(); for (let item of this.filteredItems) { DataService.collectUnique(item.project, this.projects); } @@ -276,16 +234,16 @@ export class DataService { this.selectedProjects = selectedProjects; this.selectedResearchLines = selectedResearchLines; this.clearItemsSelection(); - this.setFilteredItems(); + this.fetchItems(); } clearAllFilters() { - this.setGlobalFilter(null); - this.clearFilterValues() + this.setTextFilterInput(null); + this.clearCheckboxFilterValues(); this.rerenderCheckboxFiltersSource.next(true); } - clearFilterValues() { + clearCheckboxFilterValues() { this.selectedResearchLines.length = 0; this.selectedProjects.length = 0; } @@ -294,67 +252,67 @@ export class DataService { this.itemsSelectionSource.next(null); } - getItems() { - return this.items; + // findConceptCodesByKeywords(keywords: string[]): Set { + // return new Set(this.concepts.filter(concept => + // concept.keywords != null && concept.keywords.some((keyword: string) => + // keywords.includes(keyword)) + // ).map(concept => concept.conceptCode)); + // } + // + // static intersection(a: Set, b: Set): Set { + // return new Set( + // Array.from(a).filter(item => b.has(item))); + // } + + // getItemFilter(): (Item) => boolean { + // let conceptCodesFromTree = DataService.treeConceptCodes(this.selectedTreeNode); + // let conceptCodesFromKeywords = this.findConceptCodesByKeywords(this.selectedKeywords); + // let selectedConceptCodes: Set; + // if (conceptCodesFromKeywords.size > 0 && conceptCodesFromTree.size > 0) { + // selectedConceptCodes = DataService.intersection(conceptCodesFromTree, conceptCodesFromKeywords); + // } else if (conceptCodesFromKeywords.size > 0) { + // selectedConceptCodes = conceptCodesFromKeywords; + // } else { + // selectedConceptCodes = conceptCodesFromTree; + // } + // + // return (item: Item) => { + // return ((selectedConceptCodes.size == 0 || selectedConceptCodes.has(item.concept)) + // && (this.selectedProjects.length == 0 || this.selectedProjects.includes(item.project)) + // && (this.selectedResearchLines.length == 0 || this.selectedResearchLines.includes(item.lineOfResearch)) + // ); + // }; + // } + + setTextFilterInput(text: string) { + this.textFilterInputSource.next(text); } - getProjects() { - return this.projects; + setJsonSearchQuery(query: JSON){ + this.jsonSearchQuery = query; } - getResearchLines() { - return this.researchLines; - } - - findConceptCodesByKeywords(keywords: string[]): Set { - return new Set(this.concepts.filter(concept => - concept.keywords != null && concept.keywords.some((keyword: string) => - keywords.includes(keyword)) - ).map(concept => concept.conceptCode)); - } - - static intersection(a: Set, b: Set): Set { - return new Set( - Array.from(a).filter(item => b.has(item))); - } - - getItemFilter(): (Item) => boolean { - let conceptCodesFromTree = DataService.treeConceptCodes(this.selectedTreeNode); - let conceptCodesFromKeywords = this.findConceptCodesByKeywords(this.selectedKeywords); - let selectedConceptCodes: Set; - if (conceptCodesFromKeywords.size > 0 && conceptCodesFromTree.size > 0) { - selectedConceptCodes = DataService.intersection(conceptCodesFromTree, conceptCodesFromKeywords); - } else if (conceptCodesFromKeywords.size > 0) { - selectedConceptCodes = conceptCodesFromKeywords; - } else { - selectedConceptCodes = conceptCodesFromTree; - } + private getUniqueFilterValues() { + this.projects.length = 0; + this.linesOfResearch.length = 0; - return (item: Item) => { - return ((selectedConceptCodes.size == 0 || selectedConceptCodes.has(item.concept)) - && (this.selectedProjects.length == 0 || this.selectedProjects.includes(item.project)) - && (this.selectedResearchLines.length == 0 || this.selectedResearchLines.includes(item.researchLine)) - ); - }; + for (let item of this.items) { + DataService.collectUnique(item.project, this.projects); + DataService.collectUnique(item.lineOfResearch, this.linesOfResearch); + } } - setFilteredItems() { - let t1 = new Date(); - console.debug(`Filtering items ...`); - this.filteredItems.length = 0; - let filter = this.getItemFilter(); - this.items.forEach(item => { - if (filter(item)) { - this.filteredItems.push(item) - } + private static collectUnique(element, list: CheckboxOption[]) { + let values = list.map(function (a) { + return a.value; }); - let t2 = new Date(); - console.info(`Selected ${this.filteredItems.length} / ${this.items.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); + if (element && !values.includes(element)) { + list.push({label: element, value: element} as CheckboxOption); + } } - setGlobalFilter(globalFilter: string) { - this.globalFilterSource.next(globalFilter); - } + + // ------------------------- shopping cart ------------------------- addToShoppingCart(newItemSelection: Item[]) { let newItems: Item[] = this.shoppingCartItems.getValue(); @@ -370,24 +328,7 @@ export class DataService { this.shoppingCartItems.next(items); } - private getUniqueFilterValues() { - this.projects.length = 0; - this.researchLines.length = 0; - - for (let item of this.items) { - DataService.collectUnique(item.project, this.projects); - DataService.collectUnique(item.researchLine, this.researchLines); - } - } - - private static collectUnique(element, list: CheckboxOption[]) { - let values = list.map(function (a) { - return a.value; - }); - if (element && !values.includes(element)) { - list.push({label: element, value: element} as CheckboxOption); - } - } + // ------------------------- item summary ------------------------- displayPopup(item: Item) { this.resourceService.getItem(item.id).subscribe(extendedItem => @@ -395,6 +336,7 @@ export class DataService { ) } + // ------------------------- environment label ------------------------- setEnvironment() { this.resourceService.getEnvironment().subscribe( (env: Environment) => { @@ -402,4 +344,20 @@ export class DataService { }); } + // ------------------------- header logos ------------------------- + loadLogo(type: string) { + this.resourceService.getLogo(type) + .subscribe( + (blobContent) => { + let urlCreator = window.URL; + if (type == 'NTR') { + this.ntrLogoUrlSummary.next(urlCreator.createObjectURL(blobContent)); + } else { + this.vuLogoUrlSummary.next(urlCreator.createObjectURL(blobContent)); + } + }, + err => console.error(err) + ); + } + } diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index 997bea6..f1f297b 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -70,11 +70,16 @@ export class ResourceService { .catch(this.handleError.bind(this)); } - getItems(): Observable { + getItems(conceptCodes?: string[], projects?: string[], linesOfResearch?: string[], jsonSearchQuery?: JSON): Observable { let headers = new Headers(); let url = this.endpoint.apiUrl + PATH_ITEMS; + let urlParams = ""; - return this.http.get(url, { + if(projects || linesOfResearch || jsonSearchQuery) { + urlParams = `?conceptCodes=(${conceptCodes})&projects=(${projects})&linesOfResearch=(${linesOfResearch})&searchQuery=${jsonSearchQuery}` + } + + return this.http.get(url + urlParams, { headers: headers }) .map((response: Response) => response.json().items as Item[]) diff --git a/data-showcase/src/main/user-interface/src/app/services/search-parser.service.spec.ts b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.spec.ts new file mode 100644 index 0000000..4421a69 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.spec.ts @@ -0,0 +1,15 @@ +import { TestBed, inject } from '@angular/core/testing'; + +import { SearchParserService } from './search-parser.service'; + +describe('SearchParserService', () => { + beforeEach(() => { + TestBed.configureTestingModule({ + providers: [SearchParserService] + }); + }); + + it('should be created', inject([SearchParserService], (service: SearchParserService) => { + expect(service).toBeTruthy(); + })); +}); diff --git a/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts new file mode 100644 index 0000000..e301a0e --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts @@ -0,0 +1,18 @@ +import { Injectable } from '@angular/core'; + +@Injectable() +export class SearchParserService { + + constructor() { + } + + /* Generate binary tree with a logic query string as input + * and parse it to JSON object, + * using logic-query-parser library*/ + parse(text: string) { + var parser = require('logic-query-parser'); + let binaryTree = parser.parse(text); + return parser.utils.binaryTreeToQueryJson(binaryTree); + } + +} diff --git a/data-showcase/src/main/user-interface/src/tsconfig.app.json b/data-showcase/src/main/user-interface/src/tsconfig.app.json index 39ba8db..2ac34b0 100644 --- a/data-showcase/src/main/user-interface/src/tsconfig.app.json +++ b/data-showcase/src/main/user-interface/src/tsconfig.app.json @@ -4,7 +4,7 @@ "outDir": "../out-tsc/app", "baseUrl": "./", "module": "es2015", - "types": [] + "types": ["node"] }, "exclude": [ "test.ts", diff --git a/data-showcase/src/main/user-interface/tsconfig.json b/data-showcase/src/main/user-interface/tsconfig.json index aaec839..ccfa574 100644 --- a/data-showcase/src/main/user-interface/tsconfig.json +++ b/data-showcase/src/main/user-interface/tsconfig.json @@ -9,8 +9,11 @@ "experimentalDecorators": true, "target": "es5", "module": "commonjs", + "types": [ + "node" + ], "typeRoots": [ - "node_modules/@types" + "./node_modules/@types" ], "lib": [ "es2016", From 5a9664ad49d26f1987759aa655bc78d8c5230a0d Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 6 Nov 2017 09:31:59 +0100 Subject: [PATCH 029/104] Fix sql filter query --- .../nl/thehyve/datashowcase/ItemService.groovy | 14 +++++++++----- .../src/app/services/resource.service.ts | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index d2d221c..b2f93cc 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -89,6 +89,9 @@ class ItemService { @Transactional(readOnly = true) List getItems(Set concepts, Set projects, Set linesOfResearch, Set searchQuery) { + String conceptArray = concepts.toString().replaceAll("\\[", "\\(").replaceAll("\\]","\\)") + String projectArray = projects.toString().replaceAll("\\[", "\\(").replaceAll("\\]","\\)") + String lineOfResearchArray = linesOfResearch.toString().replaceAll("\\[", "\\(").replaceAll("\\]","\\)") String sqlSearchQueryChunk = toSQL(searchQuery) def stopWatch = new StopWatch('Fetch filtered items') stopWatch.start('Retrieve from database') @@ -108,13 +111,14 @@ class ItemService { rl.name as lineOfResearch from Item as i join i.concept c - join (select p.id, p.name, rl.name from Project as P join p.lineOfResearch rl) on i.project = p + join i.project p + join p.lineOfResearch rl where - ${concepts ? 'c.conceptCode IN ' + concepts : EMPTY_CONDITION} - AND ${projects ? 'p.name IN $projects' : EMPTY_CONDITION} - AND ${linesOfResearch ? 'rl.name IN $linesOfResearch' : EMPTY_CONDITION} + ${concepts ? 'c.conceptCode IN ' + conceptArray : EMPTY_CONDITION} + AND ${projects ? 'p.name IN ' + projectArray : EMPTY_CONDITION} + AND ${linesOfResearch ? 'rl.name IN ' + lineOfResearchArray : EMPTY_CONDITION} AND ${dataShowcaseEnvironment.internalInstance ? - EMPTY_CONDITION : 'i.publicItem = true' + EMPTY_CONDITION : 'i.publicItem=true' } AND $sqlSearchQueryChunk """ diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index f1f297b..931499b 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -76,7 +76,7 @@ export class ResourceService { let urlParams = ""; if(projects || linesOfResearch || jsonSearchQuery) { - urlParams = `?conceptCodes=(${conceptCodes})&projects=(${projects})&linesOfResearch=(${linesOfResearch})&searchQuery=${jsonSearchQuery}` + urlParams = `?conceptCodes=[${conceptCodes}]&projects=[${projects}]&linesOfResearch=[${linesOfResearch}]&searchQuery=${jsonSearchQuery}` } return this.http.get(url + urlParams, { From da3feb598877300ec4d676df44144d7cb42f8937 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 6 Nov 2017 23:06:02 +0100 Subject: [PATCH 030/104] Fix updating filters --- .../thehyve/datashowcase/ItemService.groovy | 25 +++-- .../checkbox-filter.component.html | 4 +- .../checkbox-filter.component.ts | 12 +- .../src/app/services/data.service.ts | 104 +++++++----------- 4 files changed, 64 insertions(+), 81 deletions(-) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index b2f93cc..474c844 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -16,6 +16,7 @@ import nl.thehyve.datashowcase.representation.InternalItemRepresentation import nl.thehyve.datashowcase.representation.ItemRepresentation import nl.thehyve.datashowcase.representation.PublicItemRepresentation import org.grails.core.util.StopWatch +import org.hibernate.Query import org.hibernate.Session import org.hibernate.SessionFactory import org.hibernate.transform.Transformers @@ -89,15 +90,17 @@ class ItemService { @Transactional(readOnly = true) List getItems(Set concepts, Set projects, Set linesOfResearch, Set searchQuery) { - String conceptArray = concepts.toString().replaceAll("\\[", "\\(").replaceAll("\\]","\\)") - String projectArray = projects.toString().replaceAll("\\[", "\\(").replaceAll("\\]","\\)") - String lineOfResearchArray = linesOfResearch.toString().replaceAll("\\[", "\\(").replaceAll("\\]","\\)") + + String conceptArray = concepts ? toSqlArray(concepts) : '' + String projectArray = projects ? toSqlArray(projects) : '' + String lineOfResearchArray = linesOfResearch ? toSqlArray(linesOfResearch) : '' + String sqlSearchQueryChunk = toSQL(searchQuery) def stopWatch = new StopWatch('Fetch filtered items') stopWatch.start('Retrieve from database') def session = sessionFactory.openStatelessSession() - def items = session.createQuery( + Query query = session.createQuery( """ select i.id as id, @@ -114,7 +117,7 @@ class ItemService { join i.project p join p.lineOfResearch rl where - ${concepts ? 'c.conceptCode IN ' + conceptArray : EMPTY_CONDITION} + ${concepts ? 'c.conceptCode IN ' + conceptArray : EMPTY_CONDITION} AND ${projects ? 'p.name IN ' + projectArray : EMPTY_CONDITION} AND ${linesOfResearch ? 'rl.name IN ' + lineOfResearchArray : EMPTY_CONDITION} AND ${dataShowcaseEnvironment.internalInstance ? @@ -122,8 +125,12 @@ class ItemService { } AND $sqlSearchQueryChunk """ - ).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) - .list() as List + ) + query.setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) + //query.setParameterList('concepts', concepts) + + + def items = query.list() as List stopWatch.stop() stopWatch.start('Map to representations') @@ -200,4 +207,8 @@ class ItemService { return EMPTY_CONDITION } + private String toSqlArray(Set set) { + return set.toString().replaceAll("\\[", "\\(").replaceAll("\\]","\\)") + } + } diff --git a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html index c0c718c..5b58497 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html @@ -12,14 +12,14 @@ + [metaKeySelection]=false (onChange)="onResearchLineSelect()">
+ [metaKeySelection]=false (onChange)="onProjectSelect()">
diff --git a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts index 9029694..7e3d1fd 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts @@ -44,14 +44,12 @@ export class CheckboxFilterComponent implements OnInit { ngOnInit() { } - updateFilters() { - this.dataService.updateFilterValues( - this.selectedProjects, - this.selectedResearchLines - ); + onResearchLineSelect() { + this.dataService.filterOnResearchLines(this.selectedResearchLines) } - updateProjects() { - this.dataService.updateProjectsForResearchLines(); + onProjectSelect(){ + this.dataService.filterOnProjects(this.selectedProjects) } + } diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 0c0f72b..86c176d 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -13,8 +13,8 @@ import {Project} from "../models/project"; import {Subject} from "rxjs/Subject"; import {BehaviorSubject} from "rxjs/BehaviorSubject"; import {Environment} from "../models/environment"; -import { Concept } from '../models/concept'; -import { CheckboxOption } from '../models/CheckboxOption'; +import {Concept} from '../models/concept'; +import {CheckboxOption} from '../models/CheckboxOption'; type LoadingState = 'loading' | 'complete'; @@ -167,30 +167,30 @@ export class DataService { let t1 = new Date(); console.debug(`Fetching items ...`); this.loadingItems = true; + this.filteredItems.length = 0; this.items.length = 0; let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); let codes = Array.from(selectedConceptCodes); this.resourceService.getItems( codes, this.selectedProjects, this.selectedResearchLines, this.jsonSearchQuery).subscribe( - (items: Item[]) => { - for (let item of items) { - if (this.allProjects) { - item['lineOfResearch'] = this.allProjects.find(p => p.name == item['project']).lineOfResearch; - } - this.filteredItems.push(item); - this.items.push(item); + (items: Item[]) => { + for (let item of items) { + if (this.allProjects) { + item['lineOfResearch'] = this.allProjects.find(p => p.name == item['project']).lineOfResearch; } - this.getUniqueFilterValues(); - console.info(`Loaded ${items.length} items ...`); - }, - err => console.error(err) - ); + this.filteredItems.push(item); + this.items.push(item); + } + this.getUniqueFilterValues(); + }, + err => console.error(err) + ); let t2 = new Date(); console.info(`Found ${this.items.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); this.loadingItems = false; } - static treeConceptCodes(treeNode: TreeNode) : Set { + static treeConceptCodes(treeNode: TreeNode): Set { if (treeNode == null) { return new Set(); } @@ -200,9 +200,9 @@ export class DataService { } if (treeNode.children != null) { treeNode.children.forEach((node: TreeNode) => - DataService.treeConceptCodes(node).forEach((conceptCode: string) => - conceptCodes.add(conceptCode) - ) + DataService.treeConceptCodes(node).forEach((conceptCode: string) => + conceptCodes.add(conceptCode) + ) ) } return conceptCodes; @@ -215,24 +215,23 @@ export class DataService { updateItemTable() { this.items.length = 0; + this.linesOfResearch.length =0; + this.projects.length = 0; this.clearItemsSelection(); this.clearCheckboxFilterValues(); this.fetchItems(); - this.getUniqueFilterValues(); } - updateProjectsForResearchLines() { - this.selectedProjects.length = 0; + filterOnResearchLines(selectedResearchLines) { this.projects.length = 0; + this.selectedResearchLines = selectedResearchLines; + this.clearItemsSelection(); this.fetchItems(); - for (let item of this.filteredItems) { - DataService.collectUnique(item.project, this.projects); - } } - updateFilterValues(selectedProjects: string[], selectedResearchLines: string[]) { + filterOnProjects(selectedProjects) { + this.linesOfResearch.length = 0; this.selectedProjects = selectedProjects; - this.selectedResearchLines = selectedResearchLines; this.clearItemsSelection(); this.fetchItems(); } @@ -252,53 +251,28 @@ export class DataService { this.itemsSelectionSource.next(null); } - // findConceptCodesByKeywords(keywords: string[]): Set { - // return new Set(this.concepts.filter(concept => - // concept.keywords != null && concept.keywords.some((keyword: string) => - // keywords.includes(keyword)) - // ).map(concept => concept.conceptCode)); - // } - // - // static intersection(a: Set, b: Set): Set { - // return new Set( - // Array.from(a).filter(item => b.has(item))); - // } - - // getItemFilter(): (Item) => boolean { - // let conceptCodesFromTree = DataService.treeConceptCodes(this.selectedTreeNode); - // let conceptCodesFromKeywords = this.findConceptCodesByKeywords(this.selectedKeywords); - // let selectedConceptCodes: Set; - // if (conceptCodesFromKeywords.size > 0 && conceptCodesFromTree.size > 0) { - // selectedConceptCodes = DataService.intersection(conceptCodesFromTree, conceptCodesFromKeywords); - // } else if (conceptCodesFromKeywords.size > 0) { - // selectedConceptCodes = conceptCodesFromKeywords; - // } else { - // selectedConceptCodes = conceptCodesFromTree; - // } - // - // return (item: Item) => { - // return ((selectedConceptCodes.size == 0 || selectedConceptCodes.has(item.concept)) - // && (this.selectedProjects.length == 0 || this.selectedProjects.includes(item.project)) - // && (this.selectedResearchLines.length == 0 || this.selectedResearchLines.includes(item.lineOfResearch)) - // ); - // }; - // } - setTextFilterInput(text: string) { this.textFilterInputSource.next(text); } - setJsonSearchQuery(query: JSON){ + setJsonSearchQuery(query: JSON) { this.jsonSearchQuery = query; } private getUniqueFilterValues() { - this.projects.length = 0; - this.linesOfResearch.length = 0; - - for (let item of this.items) { - DataService.collectUnique(item.project, this.projects); - DataService.collectUnique(item.lineOfResearch, this.linesOfResearch); + if (!this.projects.length && !this.selectedResearchLines.length) { + for (let item of this.filteredItems) { + DataService.collectUnique(item.project, this.projects); + DataService.collectUnique(item.lineOfResearch, this.linesOfResearch); + } + } else if (!this.linesOfResearch.length) { + for (let item of this.filteredItems) { + DataService.collectUnique(item.lineOfResearch, this.linesOfResearch); + } + } else if (!this.projects.length) { + for (let item of this.filteredItems) { + DataService.collectUnique(item.project, this.projects); + } } } From 97fa3bea5b9eaf0d0cec8d1943beb482b8ecc4c6 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 8 Nov 2017 08:45:23 +0100 Subject: [PATCH 031/104] Support POST for /items call --- .../thehyve/datashowcase/UrlMappings.groovy | 5 +++- .../src/app/services/resource.service.ts | 28 +++++++++++-------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy index f543b71..7f3f184 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy @@ -15,7 +15,10 @@ class UrlMappings { "/"(uri: '/index.html') "/api/environment"(controller: 'environment', includes: ['index']) - "/api/items"(resources: 'item', includes: ['index', 'show']) + "/api/items/$id"(method: 'GET', controller: 'item', action: 'show') + "/api/items"(controller: 'item') { + action = [GET: 'index', POST: 'index'] + } "/api/keywords"(controller: 'keyword', includes: ['index']) "/api/projects"(controller: 'project', includes: ['index']) "/api/concepts"(controller: 'concept', includes: ['index']) diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index 931499b..a35193c 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -9,7 +9,7 @@ import {Observable} from 'rxjs/Rx'; import 'rxjs/add/operator/map'; import 'rxjs/add/operator/catch'; import {TreeNode} from "../models/tree-node"; -import {Http, Response, Headers, ResponseContentType} from '@angular/http'; +import {Http, Response, Headers, ResponseContentType, RequestOptions} from '@angular/http'; import {Endpoint} from "../models/endpoint"; import {AppConfig} from "../config/app.config"; import { @@ -70,19 +70,25 @@ export class ResourceService { .catch(this.handleError.bind(this)); } - getItems(conceptCodes?: string[], projects?: string[], linesOfResearch?: string[], jsonSearchQuery?: JSON): Observable { + getItems(conceptCodes?: string[], projects?: string[], jsonSearchQuery?: JSON): Observable { let headers = new Headers(); - let url = this.endpoint.apiUrl + PATH_ITEMS; - let urlParams = ""; - - if(projects || linesOfResearch || jsonSearchQuery) { - urlParams = `?conceptCodes=[${conceptCodes}]&projects=[${projects}]&linesOfResearch=[${linesOfResearch}]&searchQuery=${jsonSearchQuery}` + headers.append('Content-Type', 'application/json'); + + const options = new RequestOptions({headers: headers}); + const url = this.endpoint.apiUrl + PATH_ITEMS; + let body = null; + + if(projects || jsonSearchQuery) { + body = { + conceptCodes: conceptCodes, + projects: projects, + searchQuery: JSON.stringify(jsonSearchQuery) + } } - return this.http.get(url + urlParams, { - headers: headers - }) - .map((response: Response) => response.json().items as Item[]) + // use POST because of url length limits in some of the browsers (limit of characters) + return this.http.post(url, body, options) + .map((res: Response) => res.json().items as Item[]) .catch(this.handleError.bind(this)); } From fad0f953d8cd76bcbc40a83f7eb5e3159f40d92b Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 8 Nov 2017 08:51:00 +0100 Subject: [PATCH 032/104] Remove researchLine from /items call parameters, search improvements --- .../datashowcase/ItemController.groovy | 30 +++++--- .../thehyve/datashowcase/ItemService.groovy | 77 ++++++++----------- .../representation/ItemRepresentation.groovy | 4 - .../src/app/services/data.service.ts | 22 +++++- 4 files changed, 72 insertions(+), 61 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index 78c656c..7009b5a 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -24,18 +24,17 @@ class ItemController { * @return the list of items as JSON. */ def index() { - - Set concepts = parseParams(params.conceptCodes) - Set projects = parseParams(params.projects) - Set linesOfResearch = parseParams(params.linesOfResearch) - Set searchQuery = parseParams(params.searchQuery) + def args = getGetOrPostParams() + Set concepts = args.conceptCodes as Set + Set projects = args.projects as Set + def searchQuery = args.searchQuery response.status = 200 response.contentType = 'application/json' response.characterEncoding = 'utf-8' Object value - if (concepts || projects || linesOfResearch || searchQuery){ - value = [items: itemService.getItems(concepts, projects, linesOfResearch, searchQuery)] + if (concepts || projects || searchQuery){ + value = [items: itemService.getItems(concepts, projects, searchQuery)] } else { value = [items: itemService.items] } @@ -54,12 +53,25 @@ class ItemController { respond itemService.getItem(id) } - private static Set parseParams(String jsonString){ + private static Set parseParams(JSONArray json){ try{ - JSONArray json = JSON.parse(jsonString) return json.collect{ "'" + it + "'"} as Set } catch (Exception e) { return null } } + + protected Map getGetOrPostParams() { + if (request.method == "POST") { + //return ((Map)request.JSON).collectEntries{ String k, v -> if(v) parseParams(v)} as Map + return (Map)request.JSON + } + return params.collectEntries { String k, v -> + if (v instanceof Object[] || v instanceof List) { + [k, v.collect { URLDecoder.decode(it, 'UTF-8') }] + } else { + [k, URLDecoder.decode(v, 'UTF-8')] + } + } + } } diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index 474c844..69536f8 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -16,9 +16,11 @@ import nl.thehyve.datashowcase.representation.InternalItemRepresentation import nl.thehyve.datashowcase.representation.ItemRepresentation import nl.thehyve.datashowcase.representation.PublicItemRepresentation import org.grails.core.util.StopWatch -import org.hibernate.Query +import org.hibernate.Criteria import org.hibernate.Session import org.hibernate.SessionFactory +import org.hibernate.criterion.Projections +import org.hibernate.criterion.Restrictions import org.hibernate.transform.Transformers import org.modelmapper.ModelMapper import org.springframework.beans.factory.annotation.Autowired @@ -34,8 +36,6 @@ class ItemService { SessionFactory sessionFactory - private final String EMPTY_CONDITION = '1=1' - @CompileStatic static ItemRepresentation map(Map itemData) { new ItemRepresentation( @@ -67,11 +67,9 @@ class ItemService { c.labelLong as labelLong, c.variableType as type, p.name as projectName, - rl.name as lineOfResearch from Item as i join i.concept c join i.project p - join p.lineOfResearch rl ${dataShowcaseEnvironment.internalInstance ? '' : 'where i.publicItem = true' } @@ -89,50 +87,39 @@ class ItemService { } @Transactional(readOnly = true) - List getItems(Set concepts, Set projects, Set linesOfResearch, Set searchQuery) { - - String conceptArray = concepts ? toSqlArray(concepts) : '' - String projectArray = projects ? toSqlArray(projects) : '' - String lineOfResearchArray = linesOfResearch ? toSqlArray(linesOfResearch) : '' + List getItems(Set concepts, Set projects, def searchQuery) { String sqlSearchQueryChunk = toSQL(searchQuery) def stopWatch = new StopWatch('Fetch filtered items') stopWatch.start('Retrieve from database') def session = sessionFactory.openStatelessSession() - Query query = session.createQuery( - """ - select - i.id as id, - i.name as name, - i.publicItem as publicItem, - i.itemPath as itemPath, - c.conceptCode as conceptCode, - c.labelLong as labelLong, - c.variableType as type, - p.name as projectName, - rl.name as lineOfResearch - from Item as i - join i.concept c - join i.project p - join p.lineOfResearch rl - where - ${concepts ? 'c.conceptCode IN ' + conceptArray : EMPTY_CONDITION} - AND ${projects ? 'p.name IN ' + projectArray : EMPTY_CONDITION} - AND ${linesOfResearch ? 'rl.name IN ' + lineOfResearchArray : EMPTY_CONDITION} - AND ${dataShowcaseEnvironment.internalInstance ? - EMPTY_CONDITION : 'i.publicItem=true' - } - AND $sqlSearchQueryChunk - """ - ) - query.setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) - //query.setParameterList('concepts', concepts) - - - def items = query.list() as List + Criteria criteria = session.createCriteria(Item, "i") + .createAlias("i.concept", "c") + .createAlias("i.project", "p") + .setProjection(Projections.projectionList() + .add(Projections.property("i.id").as("id")) + .add(Projections.property("i.name").as("name")) + .add(Projections.property("i.publicItem").as("publicItem")) + .add(Projections.property("i.itemPath").as("itemPath")) + .add(Projections.property("c.conceptCode").as("conceptCode")) + .add(Projections.property("c.labelLong").as("labelLong")) + .add(Projections.property("c.variableType").as("variableType")) + .add(Projections.property("p.name").as("projectName"))) + if(concepts) { + criteria.add( Restrictions.in('c.conceptCode', concepts)) + } + if(projects) { + criteria.add( Restrictions.in('p.name', projects)) + } + if(dataShowcaseEnvironment.internalInstance) { + criteria.add( Restrictions.eq('i.publicItem',true)) + } + criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) + def items = criteria.list() as List stopWatch.stop() + stopWatch.start('Map to representations') def result = items.collect { Map itemData -> map(itemData) @@ -203,12 +190,8 @@ class ItemService { throw new ResourceNotFoundException('Item not found') } - private String toSQL(Set query) { - return EMPTY_CONDITION - } - - private String toSqlArray(Set set) { - return set.toString().replaceAll("\\[", "\\(").replaceAll("\\]","\\)") + private String toSQL(def query) { + return "1=1" } } diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy index 0ad5752..ad2c6ed 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/ItemRepresentation.groovy @@ -68,8 +68,4 @@ class ItemRepresentation { */ String project - /** - * The name of the research line - */ - String lineOfResearch } diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 86c176d..bd59b07 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -169,10 +169,14 @@ export class DataService { this.loadingItems = true; this.filteredItems.length = 0; this.items.length = 0; + let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); let codes = Array.from(selectedConceptCodes); + let projects = this.getProjectsForSelectedResearchLines(); + let searchQuery = JSON.parse(JSON.stringify(this.jsonSearchQuery)); + this.resourceService.getItems( - codes, this.selectedProjects, this.selectedResearchLines, this.jsonSearchQuery).subscribe( + codes, projects, searchQuery).subscribe( (items: Item[]) => { for (let item of items) { if (this.allProjects) { @@ -236,6 +240,21 @@ export class DataService { this.fetchItems(); } + getProjectsForSelectedResearchLines(): string[] { + if(this.selectedResearchLines.length && !this.selectedProjects.length) { + let projects: string[] = []; + this.allProjects.forEach( p => { + if(this.selectedResearchLines.includes(p.lineOfResearch)) { + projects.push(p.name); + } + }); + return projects; + } else { + return this.selectedProjects; + } + } + + clearAllFilters() { this.setTextFilterInput(null); this.clearCheckboxFilterValues(); @@ -257,6 +276,7 @@ export class DataService { setJsonSearchQuery(query: JSON) { this.jsonSearchQuery = query; + this.fetchItems(); } private getUniqueFilterValues() { From 2ca631a94466494290e3217f72ebac92242a4668 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 9 Nov 2017 11:32:00 +0100 Subject: [PATCH 033/104] Add search criteria builder --- .../grails-app/conf/spring/resources.groovy | 2 + .../datashowcase/ItemController.groovy | 18 +- .../thehyve/datashowcase/ItemService.groovy | 26 ++- .../datashowcase/search/Operator.groovy | 46 +++++ .../search/SearchCriteriaBuilder.groovy | 185 ++++++++++++++++++ .../datashowcase/search/SearchField.groovy | 66 +++++++ .../src/app/services/data.service.ts | 5 +- .../datashowcase/PublicItemServiceSpec.groovy | 14 ++ 8 files changed, 336 insertions(+), 26 deletions(-) create mode 100644 data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy create mode 100644 data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy create mode 100644 data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchField.groovy diff --git a/data-showcase/grails-app/conf/spring/resources.groovy b/data-showcase/grails-app/conf/spring/resources.groovy index ce610e2..595eb8c 100644 --- a/data-showcase/grails-app/conf/spring/resources.groovy +++ b/data-showcase/grails-app/conf/spring/resources.groovy @@ -6,6 +6,7 @@ import nl.thehyve.datashowcase.Environment import nl.thehyve.datashowcase.StartupMessage +import nl.thehyve.datashowcase.search.SearchCriteriaBuilder import org.modelmapper.ModelMapper import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder @@ -15,4 +16,5 @@ beans = { modelMapper(ModelMapper) {} startupMessage(StartupMessage) {} bcryptEncoder(BCryptPasswordEncoder) {} + searchCriteriaBuilder(SearchCriteriaBuilder) {} } diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index 7009b5a..07b22a8 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -9,6 +9,7 @@ package nl.thehyve.datashowcase import com.fasterxml.jackson.databind.ObjectMapper import grails.converters.JSON import org.grails.web.json.JSONArray +import org.grails.web.json.JSONObject import org.springframework.beans.factory.annotation.Autowired class ItemController { @@ -19,21 +20,21 @@ class ItemController { ItemService itemService /** - * Fetches all items. - * TODO description + * Fetches all items with filter criteria. + * Supported criteria: conceptCodes, projects, free text search query. * @return the list of items as JSON. */ def index() { def args = getGetOrPostParams() Set concepts = args.conceptCodes as Set Set projects = args.projects as Set - def searchQuery = args.searchQuery + JSONObject searchQuery = args.searchQuery ? JSON.parse((String)args.searchQuery) : {} response.status = 200 response.contentType = 'application/json' response.characterEncoding = 'utf-8' Object value - if (concepts || projects || searchQuery){ + if (concepts || projects || searchQuery.length()){ value = [items: itemService.getItems(concepts, projects, searchQuery)] } else { value = [items: itemService.items] @@ -53,17 +54,8 @@ class ItemController { respond itemService.getItem(id) } - private static Set parseParams(JSONArray json){ - try{ - return json.collect{ "'" + it + "'"} as Set - } catch (Exception e) { - return null - } - } - protected Map getGetOrPostParams() { if (request.method == "POST") { - //return ((Map)request.JSON).collectEntries{ String k, v -> if(v) parseParams(v)} as Map return (Map)request.JSON } return params.collectEntries { String k, v -> diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index 69536f8..483f29a 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -15,10 +15,13 @@ import nl.thehyve.datashowcase.exception.ResourceNotFoundException import nl.thehyve.datashowcase.representation.InternalItemRepresentation import nl.thehyve.datashowcase.representation.ItemRepresentation import nl.thehyve.datashowcase.representation.PublicItemRepresentation +import nl.thehyve.datashowcase.search.SearchCriteriaBuilder import org.grails.core.util.StopWatch +import org.grails.web.json.JSONObject import org.hibernate.Criteria import org.hibernate.Session import org.hibernate.SessionFactory +import org.hibernate.criterion.Criterion import org.hibernate.criterion.Projections import org.hibernate.criterion.Restrictions import org.hibernate.transform.Transformers @@ -34,6 +37,9 @@ class ItemService { @Autowired ModelMapper modelMapper + @Autowired + SearchCriteriaBuilder searchCriteriaBuilder + SessionFactory sessionFactory @CompileStatic @@ -53,7 +59,7 @@ class ItemService { @Cacheable('items') @Transactional(readOnly = true) List getItems() { - def stopWatch = new StopWatch('Fetch all items') + def stopWatch = new StopWatch('Fetch items') stopWatch.start('Retrieve from database') def session = sessionFactory.openStatelessSession() def items = session.createQuery( @@ -66,7 +72,7 @@ class ItemService { c.conceptCode as conceptCode, c.labelLong as labelLong, c.variableType as type, - p.name as projectName, + p.name as projectName from Item as i join i.concept c join i.project p @@ -75,7 +81,7 @@ class ItemService { } """ ).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) - .list() as List + .list() as List stopWatch.stop() stopWatch.start('Map to representations') def result = items.collect { Map itemData -> @@ -87,9 +93,9 @@ class ItemService { } @Transactional(readOnly = true) - List getItems(Set concepts, Set projects, def searchQuery) { + List getItems(Set concepts, Set projects, JSONObject searchQuery) { - String sqlSearchQueryChunk = toSQL(searchQuery) + Criterion searchQueryCriterion = searchQuery.length() ? searchCriteriaBuilder.buildCriteria(searchQuery) : null def stopWatch = new StopWatch('Fetch filtered items') stopWatch.start('Retrieve from database') def session = sessionFactory.openStatelessSession() @@ -97,8 +103,9 @@ class ItemService { Criteria criteria = session.createCriteria(Item, "i") .createAlias("i.concept", "c") .createAlias("i.project", "p") + .createAlias("c.keywords", "k") .setProjection(Projections.projectionList() - .add(Projections.property("i.id").as("id")) + .add(Projections.distinct(Projections.property("i.id").as("id"))) .add(Projections.property("i.name").as("name")) .add(Projections.property("i.publicItem").as("publicItem")) .add(Projections.property("i.itemPath").as("itemPath")) @@ -115,6 +122,9 @@ class ItemService { if(dataShowcaseEnvironment.internalInstance) { criteria.add( Restrictions.eq('i.publicItem',true)) } + if(searchQueryCriterion) { + criteria.add(searchQueryCriterion) + } criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) def items = criteria.list() as List @@ -190,8 +200,4 @@ class ItemService { throw new ResourceNotFoundException('Item not found') } - private String toSQL(def query) { - return "1=1" - } - } diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy new file mode 100644 index 0000000..d169e24 --- /dev/null +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy @@ -0,0 +1,46 @@ +package nl.thehyve.datashowcase.search + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.springframework.stereotype.Component + +/** + * Operator types supported by the criteria builder. + */ +@CompileStatic +@Component +@Slf4j +enum Operator { + + AND('and'), + OR('or'), + EQUALS('='), + NOT_EQUALS("!="), + LIKE('like'), + IN('in'), + NOT('not'), + NONE('none') + + String symbol + + Operator(String symbol) { + this.symbol = symbol + } + + private static final Map mapping = new HashMap<>() + + static { + for (Operator op : Operator.values()) { + mapping.put(op.symbol, op) + } + } + + static Operator forSymbol(String symbol) { + if (mapping.containsKey(symbol)) { + return mapping[symbol] + } else { + log.error "Unknown operator: ${symbol}" + return NONE + } + } +} diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy new file mode 100644 index 0000000..b8826de --- /dev/null +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy @@ -0,0 +1,185 @@ +package nl.thehyve.datashowcase.search + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.grails.web.json.JSONObject +import org.hibernate.criterion.Criterion +import org.hibernate.criterion.Restrictions +import sun.reflect.generics.reflectiveObjects.NotImplementedException + +/** + * The class for parsing search criteria from free text filter, serialized as JSON + * into Hibernate Criteria + */ +@CompileStatic +@Slf4j +class SearchCriteriaBuilder { + + private final static String ITEM_ALIAS = "i" + private final static String CONCEPT_ALIAS = "c" + private final static String KEYWORDS_ALIAS = "k" + + private final Operator defaultOperator = Operator.EQUALS + + /** + * Construnt criteria from JSON query + * @param query + * @return + */ + Criterion buildCriteria(JSONObject query) { + def operator = Operator.forSymbol(query.type as String) + if (isJunctionOperator(operator)) { + List values = [] + List criteria = [] + + query.values.each { JSONObject c -> + if (c.type == "string") { + values.add(c.value) + } else { + criteria.add(buildCriteria(c)) + } + } + criteria.addAll(buildCriteriaFromChunks(values)) + Criterion[] criteriaArray = criteria.collect { it } + return expressionToCriteria(operator, criteriaArray) + + } else { + List values = [] + values.addAll(query.value) + List criteria = buildCriteriaFromChunks(values) + if(criteria.size() > 1) { + // throw exception + } + return criteria.first() + } + } + + /** + * Build criteria for each { "type": "string", "value": ""} element (chunk) + * Where chunk can be a representation of search field, operator or values + * @param values - list of "" from all elements (chunks) + * @return + */ + private List buildCriteriaFromChunks(List values) { + List criteria = [] + int size = values.size() + if (size == 2) { + criteria.add(applyToAllSearchFields(Operator.forSymbol(values[0]), values[1])) + return criteria + } else if (size > 0) { + if(size == 1) { + criteria.add(applyToAllSearchFields(defaultOperator, values[0])) + return criteria + } else if (size > 1) { + Operator op = Operator.forSymbol((String) values[1]) + if (op != Operator.NONE) { + def chunks = values.collate(3) + chunks.each { criteria.add(triplesChunksToCriteria(it)) } + return criteria + } + } + criteria.add(applyToAllSearchFields(Operator.IN, values)) + } + + return criteria + } + + /** + * Create criteria from triple ["property", "operator", "value(s)"] + * @param chunks + * @return + */ + private static Criterion triplesChunksToCriteria(List chunks) { + String nameElementString = chunks[0] + String operatorSymbol = chunks[1] + String[] values = chunks[2.. criteria = [] + SearchField.values().each { SearchField field -> + if(field != SearchField.NONE) { + criteria.add(buildSingleCriteria(operator, searchFieldToPropertyName(field), value)) + } + } + Criterion[] criteriaArray = criteria.collect { it } + return expressionToCriteria(Operator.OR, criteriaArray) + } + + /** + * Parse specified field to supported property name + * @param field + * @return + */ + private static String searchFieldToPropertyName(SearchField field) { + switch (field) { + case SearchField.NAME: + return ITEM_ALIAS + "." + SearchField.NAME.value + case SearchField.KEYWORDS: + return KEYWORDS_ALIAS + "." + "keyword" + case SearchField.LABEL: + return CONCEPT_ALIAS + "." + SearchField.LABEL.value + case SearchField.LABEL_LONG: + return CONCEPT_ALIAS + "." + SearchField.LABEL_LONG.value + case SearchField.LABEL_NL: + return CONCEPT_ALIAS + "." + SearchField.LABEL_NL.value + case SearchField.LABEL_NL_LONG: + return CONCEPT_ALIAS + "." + SearchField.LABEL_NL_LONG.value + default: + return SearchField.NONE.value + } + } +} diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchField.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchField.groovy new file mode 100644 index 0000000..d143e05 --- /dev/null +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchField.groovy @@ -0,0 +1,66 @@ +package nl.thehyve.datashowcase.search + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.springframework.stereotype.Component + +/** + * The fields on which it is possible to search (a free text filter). + */ +@CompileStatic +@Component +@Slf4j +enum SearchField { + /** + * Name of the item. + */ + NAME('name'), + /** + * Key words associated with concept. + */ + KEYWORDS('keywords'), + /** + * The short name of the variable in English associated with concept. + */ + LABEL('label'), + /** + * A textual description of the variable in English associated with concept. + */ + LABEL_LONG('labelLong'), + /** + * The short name of the variable in Dutch associated with concept. + */ + LABEL_NL('labelNl'), + /** + * A textual description of the variable in Dutch associated with concept. + */ + LABEL_NL_LONG('labelNlLong'), + /** + * Unknown field. + */ + NONE('none') + + String value + + SearchField(String value) { + this.value = value + } + + private static final Map mapping = new HashMap<>() + + static { + for (SearchField type : values()) { + mapping.put(type.value, type) + } + } + + static SearchField forName(String name) { + if (mapping.containsKey(name)) { + return mapping[name] + } else { + log.error "Unknown search field: ${name}" + return NONE + } + } + +} diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index bd59b07..5fb46bd 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -175,8 +175,7 @@ export class DataService { let projects = this.getProjectsForSelectedResearchLines(); let searchQuery = JSON.parse(JSON.stringify(this.jsonSearchQuery)); - this.resourceService.getItems( - codes, projects, searchQuery).subscribe( + this.resourceService.getItems(codes, projects, searchQuery).subscribe( (items: Item[]) => { for (let item of items) { if (this.allProjects) { @@ -187,7 +186,7 @@ export class DataService { } this.getUniqueFilterValues(); }, - err => console.error(err) + err => { this.clearCheckboxFilterValues(); console.error(err);} ); let t2 = new Date(); console.info(`Found ${this.items.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); diff --git a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy index 8575244..f4e68b0 100644 --- a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy +++ b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy @@ -8,6 +8,7 @@ package nl.thehyve.datashowcase import grails.testing.mixin.integration.Integration import groovy.util.logging.Slf4j +import org.grails.web.json.JSONObject import org.springframework.beans.factory.annotation.Autowired import org.springframework.transaction.annotation.Transactional import spock.lang.Requires @@ -53,4 +54,17 @@ class PublicItemServiceSpec extends Specification { that(items*.concept, hasItem('age')) } + @Requires({ -> Environment.grailsEnvironmentIn(Constants.PUBLIC_ENVIRONMENTS)}) + void "test free text filter"() { + given: + setupData() + when: + JSONObject searchQuery = ["type":"string","value":"ageA"] + def items = itemService.getItems([] as Set, [] as Set, searchQuery) + + then: + items.size() == 1 + items*.name == ['ageA'] + } + } From 6a55f846dd22fccbda2a652ac2109af9ea7d0329 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 9 Nov 2017 16:43:27 +0100 Subject: [PATCH 034/104] Extend the width of a search input field --- data-showcase/src/main/user-interface/src/styles.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-showcase/src/main/user-interface/src/styles.css b/data-showcase/src/main/user-interface/src/styles.css index 65761cf..bca6d3e 100644 --- a/data-showcase/src/main/user-interface/src/styles.css +++ b/data-showcase/src/main/user-interface/src/styles.css @@ -279,7 +279,7 @@ body .ui-inputgroup .ui-inputtext.ui-widget.ui-state-default.ui-corner-all { /* Global text filter */ .text-filter-container .ui-inputtext { - width: 550px; + width: 900px; height: 36px; padding: 0; } From 17dc2b93b942ac9c33e905ba698c0dafbf96b68c Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 9 Nov 2017 18:07:15 +0100 Subject: [PATCH 035/104] Fix search query builder logic, add tests --- .../datashowcase/ItemController.groovy | 19 +++++-- .../datashowcase/search/Operator.groovy | 1 + .../search/SearchCriteriaBuilder.groovy | 49 ++++++++++++------- .../datashowcase/PublicItemServiceSpec.groovy | 35 ++++++++++++- 4 files changed, 81 insertions(+), 23 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index 07b22a8..48f4317 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -34,10 +34,16 @@ class ItemController { response.contentType = 'application/json' response.characterEncoding = 'utf-8' Object value - if (concepts || projects || searchQuery.length()){ - value = [items: itemService.getItems(concepts, projects, searchQuery)] - } else { - value = [items: itemService.items] + try { + if (concepts || projects || searchQuery.length()) { + value = [items: itemService.getItems(concepts, projects, searchQuery)] + } else { + value = [items: itemService.items] + } + } catch (Exception e) { + response.status = 400 + respond error: "An error occured when fetching items. Error: $e.message" + return } new ObjectMapper().writeValue(response.outputStream, value) } @@ -54,6 +60,11 @@ class ItemController { respond itemService.getItem(id) } + /** + * Both GET and POST are supported for items filtering + * Parameters can be either passed as request params or request body (JSON) + * @return a map of query parameters. + */ protected Map getGetOrPostParams() { if (request.method == "POST") { return (Map)request.JSON diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy index d169e24..cffaac1 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy @@ -36,6 +36,7 @@ enum Operator { } static Operator forSymbol(String symbol) { + symbol = symbol.toLowerCase() if (mapping.containsKey(symbol)) { return mapping[symbol] } else { diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy index b8826de..acd4d43 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy @@ -2,7 +2,6 @@ package nl.thehyve.datashowcase.search import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.grails.web.json.JSONObject import org.hibernate.criterion.Criterion import org.hibernate.criterion.Restrictions import sun.reflect.generics.reflectiveObjects.NotImplementedException @@ -26,15 +25,17 @@ class SearchCriteriaBuilder { * @param query * @return */ - Criterion buildCriteria(JSONObject query) { + Criterion buildCriteria(Map query) { def operator = Operator.forSymbol(query.type as String) if (isJunctionOperator(operator)) { List values = [] List criteria = [] - query.values.each { JSONObject c -> + query.values.each { Map c -> if (c.type == "string") { - values.add(c.value) + if (c.value != ",") { + values.add(c.value) + } } else { criteria.add(buildCriteria(c)) } @@ -42,13 +43,12 @@ class SearchCriteriaBuilder { criteria.addAll(buildCriteriaFromChunks(values)) Criterion[] criteriaArray = criteria.collect { it } return expressionToCriteria(operator, criteriaArray) - } else { List values = [] values.addAll(query.value) List criteria = buildCriteriaFromChunks(values) - if(criteria.size() > 1) { - // throw exception + if (criteria.size() > 1) { + throw new IllegalArgumentException("Specified search query is invalid.") } return criteria.first() } @@ -63,18 +63,25 @@ class SearchCriteriaBuilder { private List buildCriteriaFromChunks(List values) { List criteria = [] int size = values.size() - if (size == 2) { - criteria.add(applyToAllSearchFields(Operator.forSymbol(values[0]), values[1])) - return criteria - } else if (size > 0) { - if(size == 1) { + if (size > 0) { + if (size == 1) { criteria.add(applyToAllSearchFields(defaultOperator, values[0])) return criteria - } else if (size > 1) { + } else if (size == 2) { + if (Operator.forSymbol((String) values[0]) != Operator.NONE) { + def criteriaForAllFields = applyToAllSearchFields(Operator.forSymbol((String) values[0]), values[1]) + criteria.add(criteriaForAllFields) + return criteria + } + } else { Operator op = Operator.forSymbol((String) values[1]) if (op != Operator.NONE) { - def chunks = values.collate(3) - chunks.each { criteria.add(triplesChunksToCriteria(it)) } + if (op == Operator.IN) { + criteria.add(triplesChunksToCriteria(values)) + } else { + def chunks = values.collate(3) + chunks.each { criteria.add(triplesChunksToCriteria(it)) } + } return criteria } } @@ -94,6 +101,9 @@ class SearchCriteriaBuilder { String operatorSymbol = chunks[1] String[] values = chunks[2.. valueList = new ArrayList() + value.each { valueList.add(it.toString()) } + return Restrictions.in(propertyName, valueList) } } @@ -151,8 +163,9 @@ class SearchCriteriaBuilder { private static Criterion applyToAllSearchFields(Operator operator, Object value) { List criteria = [] SearchField.values().each { SearchField field -> - if(field != SearchField.NONE) { - criteria.add(buildSingleCriteria(operator, searchFieldToPropertyName(field), value)) + if (field != SearchField.NONE) { + def singleCriteria = buildSingleCriteria(operator, searchFieldToPropertyName(field), value) + criteria.add(singleCriteria) } } Criterion[] criteriaArray = criteria.collect { it } diff --git a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy index f4e68b0..47752ec 100644 --- a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy +++ b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy @@ -58,13 +58,46 @@ class PublicItemServiceSpec extends Specification { void "test free text filter"() { given: setupData() - when: + when: "Filter on single word without field and operator specified" JSONObject searchQuery = ["type":"string","value":"ageA"] def items = itemService.getItems([] as Set, [] as Set, searchQuery) + then: + items.size() == 1 + items*.name == ['ageA'] + when: "Filter on words conjunction (OR) without field and operator specified" + searchQuery = ["type": "or", "values": [ + ["type": "string", "value": "ageA"], + ["type": "string", "value": "heightB"] + ]] + items = itemService.getItems([] as Set, [] as Set, searchQuery) + then: + items.size() == 2 + items*.name == ['ageA', 'heightB'] + + when: "Filter on single word without field, operator is specified" + searchQuery = ["type":"and", + "values":[ + ["type":"string","value":"LIKE"], + ["type":"string","value":"age%"] + ]] + items = itemService.getItems([] as Set, [] as Set, searchQuery) then: items.size() == 1 items*.name == ['ageA'] + + when: "Filter on single word with specified list of fields and operator ('keyword' IN '[]')" + searchQuery = ["type":"and","values":[ + ["type":"string","value":"keywords"], + ["type":"string","value":"IN"], + ["type":"string","value":"Personal information"], + ["type":"string","value":","], + ["type":"string","value":"Family related"]] + ] + items = itemService.getItems([] as Set, [] as Set, searchQuery) + then: + items.size() == 2 + items*.name == ['ageA', 'heightB'] } } From 08f5e33cdf9a1b7b7f0ff7ac4e8c67760f55e267 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Fri, 10 Nov 2017 09:30:20 +0100 Subject: [PATCH 036/104] Minor front end refactoring --- .../src/app/services/data.service.ts | 50 +++++++++---------- .../src/app/services/search-parser.service.ts | 4 +- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 5fb46bd..373fdb0 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -26,8 +26,6 @@ export class DataService { // the status indicating the when the tree is being loaded or finished loading public loadingTreeNodes: LoadingState = 'complete'; - // list of all items - private items: Item[] = []; // the flag indicating if Items are still being loaded public loadingItems: boolean = false; // filtered list of items based on selected node and selected checkbox filters @@ -61,10 +59,6 @@ export class DataService { // list of all projects private allProjects: Project[] = []; - // item summary popup visibility - private itemSummaryVisibleSource = new Subject(); - public itemSummaryVisible$ = this.itemSummaryVisibleSource.asObservable(); - // NTR logo private ntrLogoUrlSummary = new Subject(); public ntrLogoUrl$ = this.ntrLogoUrlSummary.asObservable(); @@ -74,6 +68,10 @@ export class DataService { public vuLogoUrl$ = this.vuLogoUrlSummary.asObservable(); // item summary popup visibility + private itemSummaryVisibleSource = new Subject(); + public itemSummaryVisible$ = this.itemSummaryVisibleSource.asObservable(); + + // environment label visibility private environmentSource = new Subject(); public environment$ = this.environmentSource.asObservable(); @@ -84,6 +82,8 @@ export class DataService { this.setEnvironment(); } + // ------------------------- tree nodes ------------------------- + private processTreeNodes(nodes: TreeNode[]): TreeNodeLib[] { if (nodes == null) { return []; @@ -137,16 +137,6 @@ export class DataService { return newNode; } - fetchAllProjects() { - this.resourceService.getProjects() - .subscribe( - (projects: Project[]) => { - this.allProjects = projects; - }, - err => console.error(err) - ); - } - fetchAllTreeNodes() { this.loadingTreeNodes = 'loading'; // Retrieve all tree nodes @@ -163,12 +153,28 @@ export class DataService { ); } + selectTreeNode(treeNode: TreeNode) { + this.selectedTreeNode = treeNode; + this.updateItemTable(); + } + + // ------------------------- filters and item table ------------------------- + + fetchAllProjects() { + this.resourceService.getProjects() + .subscribe( + (projects: Project[]) => { + this.allProjects = projects; + }, + err => console.error(err) + ); + } + fetchItems() { let t1 = new Date(); console.debug(`Fetching items ...`); this.loadingItems = true; this.filteredItems.length = 0; - this.items.length = 0; let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); let codes = Array.from(selectedConceptCodes); @@ -182,14 +188,13 @@ export class DataService { item['lineOfResearch'] = this.allProjects.find(p => p.name == item['project']).lineOfResearch; } this.filteredItems.push(item); - this.items.push(item); } this.getUniqueFilterValues(); }, err => { this.clearCheckboxFilterValues(); console.error(err);} ); let t2 = new Date(); - console.info(`Found ${this.items.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); + console.info(`Found ${this.filteredItems.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); this.loadingItems = false; } @@ -211,13 +216,7 @@ export class DataService { return conceptCodes; } - selectTreeNode(treeNode: TreeNode) { - this.selectedTreeNode = treeNode; - this.updateItemTable(); - } - updateItemTable() { - this.items.length = 0; this.linesOfResearch.length =0; this.projects.length = 0; this.clearItemsSelection(); @@ -253,7 +252,6 @@ export class DataService { } } - clearAllFilters() { this.setTextFilterInput(null); this.clearCheckboxFilterValues(); diff --git a/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts index e301a0e..598170a 100644 --- a/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts @@ -9,8 +9,8 @@ export class SearchParserService { /* Generate binary tree with a logic query string as input * and parse it to JSON object, * using logic-query-parser library*/ - parse(text: string) { - var parser = require('logic-query-parser'); + static parse(text: string) { + let parser = require('logic-query-parser'); let binaryTree = parser.parse(text); return parser.utils.binaryTreeToQueryJson(binaryTree); } From 1f12db5e02ab1dd910702010e19bc75065cd55cd Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Fri, 10 Nov 2017 11:28:00 +0100 Subject: [PATCH 037/104] Add displaying of search error message --- .../text-filter/text-filter.component.css | 7 ++++++- .../text-filter/text-filter.component.html | 5 +++++ .../text-filter/text-filter.component.ts | 9 +++++++- .../src/app/services/data.service.ts | 21 +++++++++++++++++-- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css index fee1f48..e6d2196 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css @@ -10,4 +10,9 @@ line-height: 50px; white-space: nowrap; } -.text-filter-containe:focus {outline:none;} +.text-filter-container:focus {outline:none;} + +.search-error-message { + color: red; + text-align: center; +} diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html index 196da1a..06eb20e 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html @@ -14,3 +14,8 @@
+ diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts index d27798a..c89dbc0 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts @@ -17,12 +17,18 @@ export class TextFilterComponent implements OnInit { // value of the main text filter textFilter: string; + // search error message + searchErrorMessage: string = ''; // the delay before triggering updating methods delay: number; constructor(public dataService: DataService, public searchParserService: SearchParserService, private element: ElementRef) { + this.dataService.searchErrorMessage$.subscribe( + message => { + this.searchErrorMessage = message; + }); this.dataService.textFilterInput$.subscribe( filter => { this.textFilter = filter; @@ -34,7 +40,8 @@ export class TextFilterComponent implements OnInit { } onFiltering(event) { - let jsonQuery = this.searchParserService.parse(this.textFilter); + this.dataService.clearErrorSearchMessage(); + let jsonQuery = SearchParserService.parse(this.textFilter); this.dataService.setJsonSearchQuery(jsonQuery); } diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 373fdb0..94c1c9a 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -46,7 +46,7 @@ export class DataService { // trigger checkboxFilters reload private rerenderCheckboxFiltersSource = new Subject(); public rerenderCheckboxFilters$ = this.rerenderCheckboxFiltersSource.asObservable(); - + // currently selected tree node private selectedTreeNode: TreeNode = null; // selected checkboxes for projects filter private selectedProjects: string[] = []; @@ -71,6 +71,10 @@ export class DataService { private itemSummaryVisibleSource = new Subject(); public itemSummaryVisible$ = this.itemSummaryVisibleSource.asObservable(); + // search error message + private searchErrorMessageSource = new Subject(); + public searchErrorMessage$ = this.searchErrorMessageSource.asObservable(); + // environment label visibility private environmentSource = new Subject(); public environment$ = this.environmentSource.asObservable(); @@ -175,6 +179,7 @@ export class DataService { console.debug(`Fetching items ...`); this.loadingItems = true; this.filteredItems.length = 0; + this.clearErrorSearchMessage(); let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); let codes = Array.from(selectedConceptCodes); @@ -191,13 +196,23 @@ export class DataService { } this.getUniqueFilterValues(); }, - err => { this.clearCheckboxFilterValues(); console.error(err);} + err => { + if(err.startsWith('400')){ + this.searchErrorMessageSource.next(err); + } + console.error(err); + this.clearCheckboxFilterValues(); + } ); let t2 = new Date(); console.info(`Found ${this.filteredItems.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); this.loadingItems = false; } + clearErrorSearchMessage(){ + this.searchErrorMessageSource.next(''); + } + static treeConceptCodes(treeNode: TreeNode): Set { if (treeNode == null) { return new Set(); @@ -259,6 +274,8 @@ export class DataService { } clearCheckboxFilterValues() { + this.linesOfResearch.length = 0; + this.projects.length = 0; this.selectedResearchLines.length = 0; this.selectedProjects.length = 0; } From 75712c7cce11cb1f7d8b8a01d5818311e6029cfa Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Fri, 10 Nov 2017 11:58:04 +0100 Subject: [PATCH 038/104] Fix bugs with updating filters --- .../text-filter/text-filter.component.ts | 3 +++ .../src/app/services/data.service.ts | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts index c89dbc0..e817d09 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts @@ -32,6 +32,9 @@ export class TextFilterComponent implements OnInit { this.dataService.textFilterInput$.subscribe( filter => { this.textFilter = filter; + if(this.textFilter == ''){ + this.onFiltering(null); + } }); this.delay = 0; } diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 94c1c9a..5e83f0d 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -201,7 +201,7 @@ export class DataService { this.searchErrorMessageSource.next(err); } console.error(err); - this.clearCheckboxFilterValues(); + this.clearCheckboxFilters(); } ); let t2 = new Date(); @@ -235,7 +235,7 @@ export class DataService { this.linesOfResearch.length =0; this.projects.length = 0; this.clearItemsSelection(); - this.clearCheckboxFilterValues(); + this.clearCheckboxFilterSelection(); this.fetchItems(); } @@ -268,18 +268,23 @@ export class DataService { } clearAllFilters() { - this.setTextFilterInput(null); - this.clearCheckboxFilterValues(); + this.setTextFilterInput(''); + this.clearCheckboxFilterSelection(); this.rerenderCheckboxFiltersSource.next(true); } - clearCheckboxFilterValues() { + clearCheckboxFilters() { this.linesOfResearch.length = 0; this.projects.length = 0; this.selectedResearchLines.length = 0; this.selectedProjects.length = 0; } + clearCheckboxFilterSelection() { + this.selectedResearchLines.length = 0; + this.selectedProjects.length = 0; + } + clearItemsSelection() { this.itemsSelectionSource.next(null); } @@ -294,7 +299,9 @@ export class DataService { } private getUniqueFilterValues() { - if (!this.projects.length && !this.selectedResearchLines.length) { + if (!this.projects.length && !this.selectedResearchLines.length + || !this.selectedProjects.length && !this.selectedResearchLines.length ) { + this.clearCheckboxFilters(); for (let item of this.filteredItems) { DataService.collectUnique(item.project, this.projects); DataService.collectUnique(item.lineOfResearch, this.linesOfResearch); From 16fbf5ef678a77492fee15a2043f1aefb33ef2d4 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Fri, 10 Nov 2017 14:57:15 +0100 Subject: [PATCH 039/104] Add instruction of how to create a search query --- .../src/app/filters/filters.component.html | 2 +- .../src/app/info/info.component.css | 6 - .../src/app/info/info.component.html | 128 +++++++++++++++++- .../src/app/info/info.module.ts | 5 +- 4 files changed, 128 insertions(+), 13 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/filters/filters.component.html b/data-showcase/src/main/user-interface/src/app/filters/filters.component.html index 4da86f0..c08ac2a 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/filters.component.html +++ b/data-showcase/src/main/user-interface/src/app/filters/filters.component.html @@ -1,5 +1,5 @@
- + +

General:

+ General information about the application. +

+

Searching:

+ + + General structure of the query is: +
< FIELD OPERATOR value(s) >, +

+ where FIELD is a name of the property to which the condition is applied (see: Supported fields). +
+ If FIELD is not specified, condition will be applied to all the supported fields. + If condition is satisfied for at least one of the fields of an item, this item will be added to returned item list. +

+ Combination of multiple query criterion can be composed using 'AND' and 'OR' operators + (see: Supported operators). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
'name' Name of the item.
'keywords' - Key words associated with concept.
'label' - The short name of the variable in English associated with concept.
'labelLong' - A textual description of the variable in English associated with concept.
'labelNl' - The short name of the variable in Dutch associated with concept.
'labelNlLong' - A textual description of the variable in Dutch associated with concept.
+
If any other field is used, an exception will be returned. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
'=' - equals,
+ e.g. name = "value1" +
'!=' - not equals,
+ e.g. labelEn != "value1" +
'IN' - allows to specify multiple values. Values have to be placed between round brackets.
+ e.g. name IN ("value1", "value2") +
'LIKE' - like, search for a specified pattern. There are two wildcards used:
+
    +
  • % - The percent sign represents zero, one, or multiple characters;
    • +
  • _ - The underscore represents a single character;/n + e.g. labelNl LIKE "va_ue%"
+
'AND' - logical conjunction,
+ e.g. (name = "value1") AND (labelLong != "value2") +
'OR' - logical disjunction,
+ e.g. (name = "value1") OR (name = "value2") +
+
All the combinations of junction operators are possible, when using round brackets. +
e.g. ((name = "value1") OR (name = "value2")) AND (labelLong = "value3") + + +
+ diff --git a/data-showcase/src/main/user-interface/src/app/info/info.module.ts b/data-showcase/src/main/user-interface/src/app/info/info.module.ts index 4e179d2..7b3e2ad 100644 --- a/data-showcase/src/main/user-interface/src/app/info/info.module.ts +++ b/data-showcase/src/main/user-interface/src/app/info/info.module.ts @@ -2,7 +2,7 @@ import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; import {InfoComponent} from "./info.component"; import {FormsModule} from "@angular/forms"; -import {DialogModule, PanelModule} from "primeng/primeng"; +import {AccordionModule, DialogModule, PanelModule} from "primeng/primeng"; import {SidebarModule} from "primeng/components/sidebar/sidebar"; @NgModule({ @@ -11,7 +11,8 @@ import {SidebarModule} from "primeng/components/sidebar/sidebar"; FormsModule, SidebarModule, PanelModule, - DialogModule + DialogModule, + AccordionModule ], declarations: [InfoComponent], exports: [InfoComponent] From 379b7880084fed163a891f1e3cac147e513cfc7e Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Fri, 10 Nov 2017 16:51:11 +0100 Subject: [PATCH 040/104] Extend test cases for search query builder --- .../datashowcase/PublicItemServiceSpec.groovy | 99 ++++++++++++++++--- 1 file changed, 84 insertions(+), 15 deletions(-) diff --git a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy index 47752ec..514763b 100644 --- a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy +++ b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy @@ -54,12 +54,13 @@ class PublicItemServiceSpec extends Specification { that(items*.concept, hasItem('age')) } - @Requires({ -> Environment.grailsEnvironmentIn(Constants.PUBLIC_ENVIRONMENTS)}) + @Requires({ -> Environment.grailsEnvironmentIn(Constants.PUBLIC_ENVIRONMENTS) }) void "test free text filter"() { given: - setupData() + setupData() + when: "Filter on single word without field and operator specified" - JSONObject searchQuery = ["type":"string","value":"ageA"] + JSONObject searchQuery = ["type": "string", "value": "ageA"] def items = itemService.getItems([] as Set, [] as Set, searchQuery) then: items.size() == 1 @@ -75,11 +76,11 @@ class PublicItemServiceSpec extends Specification { items.size() == 2 items*.name == ['ageA', 'heightB'] - when: "Filter on single word without field, operator is specified" - searchQuery = ["type":"and", - "values":[ - ["type":"string","value":"LIKE"], - ["type":"string","value":"age%"] + when: "Filter on single word without field, operator (LIKE) is specified" + searchQuery = ["type" : "and", + "values": [ + ["type": "string", "value": "LIKE"], + ["type": "string", "value": "a_e%"] ]] items = itemService.getItems([] as Set, [] as Set, searchQuery) then: @@ -87,17 +88,85 @@ class PublicItemServiceSpec extends Specification { items*.name == ['ageA'] when: "Filter on single word with specified list of fields and operator ('keyword' IN '[]')" - searchQuery = ["type":"and","values":[ - ["type":"string","value":"keywords"], - ["type":"string","value":"IN"], - ["type":"string","value":"Personal information"], - ["type":"string","value":","], - ["type":"string","value":"Family related"]] + searchQuery = ["type": "and", "values": [ + ["type": "string", "value": "keywords"], + ["type": "string", "value": "IN"], + ["type": "string", "value": "Personal information"], + ["type": "string", "value": ","], + ["type": "string", "value": "Family related"]] ] items = itemService.getItems([] as Set, [] as Set, searchQuery) - then: + then: items.size() == 2 items*.name == ['ageA', 'heightB'] + + when: "Filter on words disjunction (AND) and '=' operator ('field1=val1 OR field2=val2')" + searchQuery = ["type": "and", "values": [ + ["type": "string", "value": "name"], + ["type": "string", "value": "="], + ["type": "string", "value": "ageA"], + ["type": "string", "value": "label"], + ["type": "string", "value": "="], + ["type": "string", "value": "ageB"] + ]] + items = itemService.getItems([] as Set, [] as Set, searchQuery) + then: + items.size() == 0 + + when: "Invalid field name specified" + def invalidProperty = "test_field" + searchQuery = ["type": "and", "values": [ + ["type": "string", "value": invalidProperty], + ["type": "string", "value": "="], + ["type": "string", "value": "value"] + ]] + itemService.getItems([] as Set, [] as Set, searchQuery) + then: "Exception is thrown" + IllegalArgumentException ex = thrown() + ex.message == "Specified property name: $invalidProperty is not supported." + + when: "Brackets are used in junction query" + // 'name = "ageA" OR (name = "heightA" AND label = "height")' + JSONObject searchQuery1 = ["type": "or", "values": [ + ["type": "and", "values": [ + ["type": "string", "value": "name"], + ["type": "string", "value": "="], + ["type": "string", "value": "ageA"] + ]], + ["type": "and", "values": [ + ["type": "string", "value": "name"], + ["type": "string", "value": "="], + ["type": "string", "value": "heightB"], + ["type": "string", "value": "labelNl"], + ["type": "string", "value": "="], + ["type": "string", "value": "hoogte"] + ]] + ]] + + // '(name = "ageA" OR name = "heightA") AND label = "height"' + JSONObject searchQuery2 = ["type": "and", "values": [ + ["type": "or", "values": [ + ["type": "and", "values": [ + ["type": "string", "value": "name"], + ["type": "string", "value": "="], + ["type": "string", "value": "ageA"] + ]], + ["type": "and", "values": [ + ["type": "string", "value": "name"], + ["type": "string", "value": "="], + ["type": "string", "value": "heightB"] + ]] + ]], + ["type": "string", "value": "labelNl"], + ["type": "string", "value": "="], + ["type": "string", "value": "hoogte"] + ]] + def itemsForQuery1 = itemService.getItems([] as Set, [] as Set, searchQuery1) + def itemsForQuery2 = itemService.getItems([] as Set, [] as Set, searchQuery2) + then: "Results are different, depending on the distribution of brackets" + itemsForQuery1 != itemsForQuery2 + itemsForQuery1.size() == 2 + itemsForQuery2.size() == 1 } } From f3436fc8be3a92efbb12d311f0c0f06d1fe473cf Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Fri, 10 Nov 2017 17:04:05 +0100 Subject: [PATCH 041/104] Fix ng Karma tests --- .../src/main/user-interface/src/app/app.component.spec.ts | 4 +++- .../user-interface/src/app/filters/filters.component.spec.ts | 4 +++- .../src/main/user-interface/src/app/filters/filters.module.ts | 4 +++- .../src/app/filters/text-filter/text-filter.component.spec.ts | 4 +++- .../main/user-interface/src/app/info/info.component.spec.ts | 3 ++- .../src/main/user-interface/src/app/info/info.module.ts | 2 +- .../src/main/user-interface/src/app/services/data.service.ts | 2 +- 7 files changed, 16 insertions(+), 7 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts index 964c689..aad8c44 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts @@ -29,6 +29,7 @@ import {LogosComponent} from "./logos/logos.component"; import {PageRibbonComponent} from "./page-ribbon/page-ribbon.component"; import {FiltersModule} from "./filters/filters.module"; import {InfoModule} from "./info/info.module"; +import {SearchParserService} from "./services/search-parser.service"; describe('AppComponent', () => { beforeEach(async(() => { @@ -64,7 +65,8 @@ describe('AppComponent', () => { provide: AppConfig, useClass: AppConfigMock }, - ResourceService + ResourceService, + SearchParserService ] }).compileComponents(); })); diff --git a/data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts b/data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts index 8e30eae..8b718ef 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts @@ -11,6 +11,7 @@ import {DataService} from "../services/data.service"; import {ResourceService} from "../services/resource.service"; import {AppConfig} from "../config/app.config"; import {AppConfigMock} from "../config/app.config.mock"; +import {SearchParserService} from "../services/search-parser.service"; describe('FiltersComponent', () => { let component: FiltersComponent; @@ -35,7 +36,8 @@ describe('FiltersComponent', () => { { provide: AppConfig, useClass: AppConfigMock - } + }, + SearchParserService ] }) .compileComponents(); diff --git a/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts b/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts index c951212..be1c8fb 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts @@ -5,6 +5,7 @@ import {AutoCompleteModule, ButtonModule, FieldsetModule, ListboxModule, PanelMo import {FiltersComponent} from "./filters.component"; import {CheckboxFilterComponent} from "./checkbox-filter/checkbox-filter.component"; import {TextFilterComponent} from "./text-filter/text-filter.component"; +import {SearchParserService} from "../services/search-parser.service"; @NgModule({ imports: [ @@ -18,6 +19,7 @@ import {TextFilterComponent} from "./text-filter/text-filter.component"; ButtonModule ], declarations: [FiltersComponent, TextFilterComponent, CheckboxFilterComponent], - exports: [FiltersComponent] + exports: [FiltersComponent], + providers: [SearchParserService] }) export class FiltersModule { } diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.spec.ts b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.spec.ts index 58f632f..3086711 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.spec.ts @@ -14,6 +14,7 @@ import {DataService} from "../../services/data.service"; import {HttpModule} from "@angular/http"; import {AppConfig} from "../../config/app.config"; import {AppConfigMock} from "../../config/app.config.mock"; +import {SearchParserService} from "../../services/search-parser.service"; describe('TextFilterComponent', () => { let component: TextFilterComponent; @@ -33,7 +34,8 @@ describe('TextFilterComponent', () => { { provide: AppConfig, useClass: AppConfigMock - } + }, + SearchParserService ] }) .compileComponents(); diff --git a/data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts b/data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts index d5e98fd..33ef1e5 100644 --- a/data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/info/info.component.spec.ts @@ -2,7 +2,7 @@ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { InfoComponent } from './info.component'; import {FormsModule} from "@angular/forms"; -import {PanelModule} from "primeng/primeng"; +import {AccordionModule, PanelModule} from "primeng/primeng"; import {SidebarModule} from "primeng/components/sidebar/sidebar"; import {BrowserModule} from "@angular/platform-browser"; import {BrowserAnimationsModule} from "@angular/platform-browser/animations"; @@ -20,6 +20,7 @@ describe('InfoComponent', () => { SidebarModule, BrowserModule, BrowserAnimationsModule, + AccordionModule ] }) .compileComponents(); diff --git a/data-showcase/src/main/user-interface/src/app/info/info.module.ts b/data-showcase/src/main/user-interface/src/app/info/info.module.ts index 7b3e2ad..e035591 100644 --- a/data-showcase/src/main/user-interface/src/app/info/info.module.ts +++ b/data-showcase/src/main/user-interface/src/app/info/info.module.ts @@ -12,7 +12,7 @@ import {SidebarModule} from "primeng/components/sidebar/sidebar"; SidebarModule, PanelModule, DialogModule, - AccordionModule + AccordionModule, ], declarations: [InfoComponent], exports: [InfoComponent] diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 5e83f0d..13ba7ad 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -197,7 +197,7 @@ export class DataService { this.getUniqueFilterValues(); }, err => { - if(err.startsWith('400')){ + if(err != String(undefined) && err.startsWith("400")){ this.searchErrorMessageSource.next(err); } console.error(err); From b832b3e775125205d40c95037cb23f6515a0c286 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 13 Nov 2017 10:38:04 +0100 Subject: [PATCH 042/104] Move logic-query-parser import to package.json. --- data-showcase/src/main/user-interface/package.json | 1 + .../user-interface/src/app/services/search-parser.service.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/data-showcase/src/main/user-interface/package.json b/data-showcase/src/main/user-interface/package.json index fddbb4e..8bb32a1 100644 --- a/data-showcase/src/main/user-interface/package.json +++ b/data-showcase/src/main/user-interface/package.json @@ -28,6 +28,7 @@ "core-js": "^2.4.1", "file-saver": "^1.3.3", "font-awesome": "^4.7.0", + "logic-query-parser": "0.0.5", "primeng": "4.2.2", "roboto-fontface": "^0.8.0", "rxjs": "^5.5.2", diff --git a/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts index 598170a..0a21a51 100644 --- a/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts @@ -1,4 +1,5 @@ import { Injectable } from '@angular/core'; +import parser from 'logic-query-parser'; @Injectable() export class SearchParserService { @@ -10,7 +11,6 @@ export class SearchParserService { * and parse it to JSON object, * using logic-query-parser library*/ static parse(text: string) { - let parser = require('logic-query-parser'); let binaryTree = parser.parse(text); return parser.utils.binaryTreeToQueryJson(binaryTree); } From 4a2fc25cd282cc2ac80789128d1f9d9d98121142 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 13 Nov 2017 13:06:27 +0100 Subject: [PATCH 043/104] Remove unneeded explicit JSON parsing. --- .../datashowcase/ItemController.groovy | 7 +-- .../thehyve/datashowcase/ItemService.groovy | 4 +- .../datashowcase/search/Operator.groovy | 3 ++ .../search/SearchCriteriaBuilder.groovy | 50 +++++++++++-------- .../src/app/services/data.service.ts | 2 +- .../src/app/services/resource.service.ts | 2 +- 6 files changed, 40 insertions(+), 28 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index 48f4317..dc345eb 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -28,21 +28,22 @@ class ItemController { def args = getGetOrPostParams() Set concepts = args.conceptCodes as Set Set projects = args.projects as Set - JSONObject searchQuery = args.searchQuery ? JSON.parse((String)args.searchQuery) : {} + def searchQuery = args.searchQuery as Map response.status = 200 response.contentType = 'application/json' response.characterEncoding = 'utf-8' Object value try { - if (concepts || projects || searchQuery.length()) { + if (concepts || projects || searchQuery) { value = [items: itemService.getItems(concepts, projects, searchQuery)] } else { value = [items: itemService.items] } } catch (Exception e) { response.status = 400 - respond error: "An error occured when fetching items. Error: $e.message" + log.error 'An error occurred when fetching items.', e + respond error: "An error occurred when fetching items. Error: $e.message" return } new ObjectMapper().writeValue(response.outputStream, value) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index 483f29a..c39d9ae 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -93,9 +93,9 @@ class ItemService { } @Transactional(readOnly = true) - List getItems(Set concepts, Set projects, JSONObject searchQuery) { + List getItems(Set concepts, Set projects, Map searchQuery) { - Criterion searchQueryCriterion = searchQuery.length() ? searchCriteriaBuilder.buildCriteria(searchQuery) : null + Criterion searchQueryCriterion = searchQuery ? searchCriteriaBuilder.buildCriteria(searchQuery) : null def stopWatch = new StopWatch('Fetch filtered items') stopWatch.start('Retrieve from database') def session = sessionFactory.openStatelessSession() diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy index cffaac1..0115353 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy @@ -36,6 +36,9 @@ enum Operator { } static Operator forSymbol(String symbol) { + if (symbol == null) { + return null + } symbol = symbol.toLowerCase() if (mapping.containsKey(symbol)) { return mapping[symbol] diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy index acd4d43..5019df3 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy @@ -21,29 +21,16 @@ class SearchCriteriaBuilder { private final Operator defaultOperator = Operator.EQUALS /** - * Construnt criteria from JSON query + * Construct criteria from JSON query * @param query * @return */ Criterion buildCriteria(Map query) { - def operator = Operator.forSymbol(query.type as String) - if (isJunctionOperator(operator)) { - List values = [] - List criteria = [] - - query.values.each { Map c -> - if (c.type == "string") { - if (c.value != ",") { - values.add(c.value) - } - } else { - criteria.add(buildCriteria(c)) - } - } - criteria.addAll(buildCriteriaFromChunks(values)) - Criterion[] criteriaArray = criteria.collect { it } - return expressionToCriteria(operator, criteriaArray) - } else { + if (query == null) { + return null + } + def type = query.type as String + if (type == 'string') { List values = [] values.addAll(query.value) List criteria = buildCriteriaFromChunks(values) @@ -51,6 +38,27 @@ class SearchCriteriaBuilder { throw new IllegalArgumentException("Specified search query is invalid.") } return criteria.first() + } else { + def operator = Operator.forSymbol(type) + if (isJunctionOperator(operator)) { + List values = [] + List criteria = [] + + query.values.each { Map c -> + if (c.type == "string") { + if (c.value != ",") { + values.add(c.value) + } + } else { + criteria.add(buildCriteria(c)) + } + } + criteria.addAll(buildCriteriaFromChunks(values)) + Criterion[] criteriaArray = criteria.collect { it } + return expressionToCriteria(operator, criteriaArray) + } else { + throw new IllegalArgumentException("Unsupported search type: ${type}.") + } } } @@ -149,13 +157,13 @@ class SearchCriteriaBuilder { } // TODO implement negation criterion - private static Criterion nagateExpression(Criterion c) { + private static Criterion negateExpression(Criterion c) { throw new NotImplementedException() // return Restrictions.not(c) } /** - * If searchFiled is not specified, search query is applied to all supported properties + * If searchField is not specified, search query is applied to all supported properties * @param operator * @param value * @return diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 13ba7ad..5f695da 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -184,7 +184,7 @@ export class DataService { let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); let codes = Array.from(selectedConceptCodes); let projects = this.getProjectsForSelectedResearchLines(); - let searchQuery = JSON.parse(JSON.stringify(this.jsonSearchQuery)); + let searchQuery = this.jsonSearchQuery; //JSON.parse(JSON.stringify(this.jsonSearchQuery)); this.resourceService.getItems(codes, projects, searchQuery).subscribe( (items: Item[]) => { diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index a35193c..37f000b 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -82,7 +82,7 @@ export class ResourceService { body = { conceptCodes: conceptCodes, projects: projects, - searchQuery: JSON.stringify(jsonSearchQuery) + searchQuery: jsonSearchQuery } } From 695307e707c7a50a42021768b63cdd44174afe12 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 13 Nov 2017 13:06:43 +0100 Subject: [PATCH 044/104] Make 'like' queries case insensitive. --- .../nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy index 5019df3..fb1a8b9 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy @@ -130,7 +130,7 @@ class SearchCriteriaBuilder { case Operator.NOT_EQUALS: return Restrictions.eq(propertyName, value) case Operator.LIKE: - return Restrictions.like(propertyName, value) + return Restrictions.ilike(propertyName, value) case Operator.IN: List valueList = new ArrayList() value.each { valueList.add(it.toString()) } From 3e052648802a3f27c0b3bcb7a94255f82b232229 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 13 Nov 2017 13:17:42 +0100 Subject: [PATCH 045/104] Write test output after test failure on Travis. --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1e97978..0cabaa4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,8 @@ cache: - $HOME/.m2 before_install: + - sudo apt-get -qq update + - sudo apt-get install -y html2text - pushd data-showcase - gradle wrapper - GRADLE_SCRIPT="$(pwd)/gradlew" @@ -33,6 +35,11 @@ script: - ${GRADLE_SCRIPT} test - popd +after_failure: + - echo "Writing reports..." + - html2text data-showcase/build/reports/tests/test/index.html + - for f in data-showcase/build/reports/tests/test/classes/*.html; do echo "$f"; html2text "$f"; done + notifications: webhooks: on_success: change From 3c48b53d33f2a78b64e702250bd0aa60ed300a38 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 13 Nov 2017 14:30:03 +0100 Subject: [PATCH 046/104] Fix item service tests. --- .../nl/thehyve/datashowcase/PublicItemServiceSpec.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy index 514763b..4559715 100644 --- a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy +++ b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy @@ -74,7 +74,7 @@ class PublicItemServiceSpec extends Specification { items = itemService.getItems([] as Set, [] as Set, searchQuery) then: items.size() == 2 - items*.name == ['ageA', 'heightB'] + items*.name as Set == ['ageA', 'heightB'] as Set when: "Filter on single word without field, operator (LIKE) is specified" searchQuery = ["type" : "and", @@ -98,7 +98,7 @@ class PublicItemServiceSpec extends Specification { items = itemService.getItems([] as Set, [] as Set, searchQuery) then: items.size() == 2 - items*.name == ['ageA', 'heightB'] + items*.name as Set == ['ageA', 'heightB'] as Set when: "Filter on words disjunction (AND) and '=' operator ('field1=val1 OR field2=val2')" searchQuery = ["type": "and", "values": [ From 1d37a7beccc0643335bfacdbee9ec24b106339ee Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 13 Nov 2017 14:52:56 +0100 Subject: [PATCH 047/104] Support 'contains' operator. 'contains' and case insensitive as default. --- .../nl/thehyve/datashowcase/search/Operator.groovy | 1 + .../search/SearchCriteriaBuilder.groovy | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy index 0115353..f5b0016 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy @@ -14,6 +14,7 @@ enum Operator { AND('and'), OR('or'), + CONTAINS('contains'), EQUALS('='), NOT_EQUALS("!="), LIKE('like'), diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy index fb1a8b9..512e08b 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy @@ -3,6 +3,7 @@ package nl.thehyve.datashowcase.search import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.hibernate.criterion.Criterion +import org.hibernate.criterion.MatchMode import org.hibernate.criterion.Restrictions import sun.reflect.generics.reflectiveObjects.NotImplementedException @@ -18,7 +19,7 @@ class SearchCriteriaBuilder { private final static String CONCEPT_ALIAS = "c" private final static String KEYWORDS_ALIAS = "k" - private final Operator defaultOperator = Operator.EQUALS + private final Operator defaultOperator = Operator.CONTAINS /** * Construct criteria from JSON query @@ -125,16 +126,20 @@ class SearchCriteriaBuilder { */ private static Criterion buildSingleCriteria(Operator operator, String propertyName, Object value) { switch (operator) { + case Operator.CONTAINS: + return Restrictions.ilike(propertyName, value as String, MatchMode.ANYWHERE) case Operator.EQUALS: - return Restrictions.eq(propertyName, value) + return Restrictions.ilike(propertyName, value as String, MatchMode.EXACT) case Operator.NOT_EQUALS: - return Restrictions.eq(propertyName, value) + return Restrictions.not(Restrictions.ilike(propertyName, value as String, MatchMode.EXACT)) case Operator.LIKE: - return Restrictions.ilike(propertyName, value) + return Restrictions.ilike(propertyName, value as String) case Operator.IN: List valueList = new ArrayList() value.each { valueList.add(it.toString()) } return Restrictions.in(propertyName, valueList) + default: + throw new IllegalArgumentException("Unsupported operator: ${operator}.") } } From ad26874a3c45c0155b9429725b9dac56ad6e64d3 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 13 Nov 2017 16:27:24 +0100 Subject: [PATCH 048/104] Clear text filter on node change --- .../src/main/user-interface/src/app/services/data.service.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 5f695da..f0c3d02 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -232,10 +232,8 @@ export class DataService { } updateItemTable() { - this.linesOfResearch.length =0; - this.projects.length = 0; this.clearItemsSelection(); - this.clearCheckboxFilterSelection(); + this.clearAllFilters(); this.fetchItems(); } From c5dc9432966f10049071e598b1af3498c85d4887 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 13 Nov 2017 16:31:31 +0100 Subject: [PATCH 049/104] Fix "Clear filters" button --- .../src/main/user-interface/src/app/services/data.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index f0c3d02..cf6de7e 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -266,8 +266,8 @@ export class DataService { } clearAllFilters() { - this.setTextFilterInput(''); this.clearCheckboxFilterSelection(); + this.setTextFilterInput(''); this.rerenderCheckboxFiltersSource.next(true); } From 8cf9845fb5ce2f27bb7aa89efcf8f13f827d571d Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 13 Nov 2017 16:57:00 +0100 Subject: [PATCH 050/104] Add search when pressing the key --- .../src/app/filters/text-filter/text-filter.component.html | 3 ++- .../src/app/filters/text-filter/text-filter.component.ts | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html index 06eb20e..f9e0fbe 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html @@ -10,7 +10,8 @@ placeholder="Enter search term" class="text-filter" (onClear)="onFiltering($event)" - (completeMethod)="removePrimeNgAutocompleteLoader()"> + (completeMethod)="removePrimeNgAutocompleteLoader()" + (onKeyUp)="onKeyUp($event)">
diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts index e817d09..2fc87fc 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts @@ -42,6 +42,13 @@ export class TextFilterComponent implements OnInit { ngOnInit() { } + onKeyUp(event) { + // "enter" key code = 13 + if (event.keyCode == 13) { + this.onFiltering(event) + } + } + onFiltering(event) { this.dataService.clearErrorSearchMessage(); let jsonQuery = SearchParserService.parse(this.textFilter); From 0aa66855a0797c74dc70986c05dc93d5fd87ceaf Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 13 Nov 2017 17:32:13 +0100 Subject: [PATCH 051/104] Add item-table highlight on items loading completion --- .../app/item-table/item-table.component.html | 4 ++-- .../src/app/item-table/item-table.component.ts | 17 ++++++++++++++++- .../src/app/services/data.service.ts | 7 ++++--- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html index ac7fe74..872a064 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html @@ -4,7 +4,7 @@ ~ (see accompanying file LICENSE). --> -
+
+ [loading]="dataService.loadingItems == 'loading'" loadingIcon="fa fa-spin fa-refresh fa-fw"> complete', [ + style({ + background: 'rgba(51, 156, 144, 0.5)' + }), + animate('1000ms ease-out', style({ + background: 'rgba(255, 255, 255, 0.0)' + })) + ]) + ]) + ] }) export class ItemTableComponent implements OnInit { diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index cf6de7e..44cd57d 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -27,7 +27,7 @@ export class DataService { public loadingTreeNodes: LoadingState = 'complete'; // the flag indicating if Items are still being loaded - public loadingItems: boolean = false; + public loadingItems: LoadingState = 'complete'; // filtered list of items based on selected node and selected checkbox filters public filteredItems: Item[] = []; // items selected in the itemTable @@ -177,7 +177,7 @@ export class DataService { fetchItems() { let t1 = new Date(); console.debug(`Fetching items ...`); - this.loadingItems = true; + this.loadingItems = 'loading'; this.filteredItems.length = 0; this.clearErrorSearchMessage(); @@ -194,6 +194,7 @@ export class DataService { } this.filteredItems.push(item); } + this.loadingItems = "complete"; this.getUniqueFilterValues(); }, err => { @@ -206,7 +207,7 @@ export class DataService { ); let t2 = new Date(); console.info(`Found ${this.filteredItems.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); - this.loadingItems = false; + } clearErrorSearchMessage(){ From c7852bfdf51e881736e655592bf78bafb8fd478e Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Tue, 14 Nov 2017 22:04:27 +0100 Subject: [PATCH 052/104] Refactor text filter: parse text in front end. --- .../datashowcase/ItemController.groovy | 71 +++--- .../thehyve/datashowcase/UrlMappings.groovy | 2 +- .../thehyve/datashowcase/ItemService.groovy | 15 +- .../{search => enumeration}/Operator.groovy | 6 +- .../SearchField.groovy | 6 +- .../SearchQueryRepresentation.groovy | 10 + .../search/SearchCriteriaBuilder.groovy | 230 +++++++++--------- .../src/app/filters/filters.module.ts | 12 +- .../text-filter/text-filter.component.css | 6 + .../text-filter/text-filter.component.html | 11 +- .../text-filter/text-filter.component.ts | 13 +- .../app/item-table/item-table.component.html | 2 +- .../src/app/models/binary-tree.ts | 32 +++ .../src/app/models/search-query.ts | 28 +++ .../src/app/services/data.service.ts | 32 +-- .../src/app/services/resource.service.ts | 12 +- .../src/app/services/search-parser.service.ts | 121 ++++++++- .../shopping-cart.component.html | 2 +- 18 files changed, 412 insertions(+), 199 deletions(-) rename data-showcase/src/main/groovy/nl/thehyve/datashowcase/{search => enumeration}/Operator.groovy (85%) rename data-showcase/src/main/groovy/nl/thehyve/datashowcase/{search => enumeration}/SearchField.groovy (89%) create mode 100644 data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/SearchQueryRepresentation.groovy create mode 100644 data-showcase/src/main/user-interface/src/app/models/binary-tree.ts create mode 100644 data-showcase/src/main/user-interface/src/app/models/search-query.ts diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index dc345eb..eba92f4 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -7,9 +7,7 @@ package nl.thehyve.datashowcase import com.fasterxml.jackson.databind.ObjectMapper -import grails.converters.JSON -import org.grails.web.json.JSONArray -import org.grails.web.json.JSONObject +import nl.thehyve.datashowcase.representation.SearchQueryRepresentation import org.springframework.beans.factory.annotation.Autowired class ItemController { @@ -20,33 +18,49 @@ class ItemController { ItemService itemService /** - * Fetches all items with filter criteria. - * Supported criteria: conceptCodes, projects, free text search query. + * Fetches all items. * @return the list of items as JSON. */ def index() { - def args = getGetOrPostParams() - Set concepts = args.conceptCodes as Set - Set projects = args.projects as Set - def searchQuery = args.searchQuery as Map + try { + response.status = 200 + response.contentType = 'application/json' + response.characterEncoding = 'utf-8' + def data = [items: itemService.items] + log.info "Writing ${data.items.size()} items ..." + new ObjectMapper().writeValue(response.outputStream, data) + } catch (Exception e) { + response.status = 400 + log.error 'An error occurred when fetching items.', e + respond error: "An error occurred when fetching items. Error: $e.message" + } + } - response.status = 200 - response.contentType = 'application/json' - response.characterEncoding = 'utf-8' - Object value + /** + * Fetches all items with filter criteria. + * Supported criteria: conceptCodes, projects, free text search query. + * @return the list of items as JSON. + */ + def search() { try { - if (concepts || projects || searchQuery) { - value = [items: itemService.getItems(concepts, projects, searchQuery)] - } else { - value = [items: itemService.items] - } + def args = request.JSON as Map + Set concepts = args.conceptCodes as Set + Set projects = args.projects as Set + log.info "Query input: ${args.searchQuery}" + def searchQuery = new SearchQueryRepresentation() + bindData(searchQuery, args.searchQuery) + + response.status = 200 + response.contentType = 'application/json' + response.characterEncoding = 'utf-8' + + def data = [items: itemService.getItems(concepts, projects, searchQuery)] + new ObjectMapper().writeValue(response.outputStream, data) } catch (Exception e) { response.status = 400 log.error 'An error occurred when fetching items.', e respond error: "An error occurred when fetching items. Error: $e.message" - return } - new ObjectMapper().writeValue(response.outputStream, value) } /** @@ -61,21 +75,4 @@ class ItemController { respond itemService.getItem(id) } - /** - * Both GET and POST are supported for items filtering - * Parameters can be either passed as request params or request body (JSON) - * @return a map of query parameters. - */ - protected Map getGetOrPostParams() { - if (request.method == "POST") { - return (Map)request.JSON - } - return params.collectEntries { String k, v -> - if (v instanceof Object[] || v instanceof List) { - [k, v.collect { URLDecoder.decode(it, 'UTF-8') }] - } else { - [k, URLDecoder.decode(v, 'UTF-8')] - } - } - } } diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy index 7f3f184..8261e47 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy @@ -17,7 +17,7 @@ class UrlMappings { "/api/environment"(controller: 'environment', includes: ['index']) "/api/items/$id"(method: 'GET', controller: 'item', action: 'show') "/api/items"(controller: 'item') { - action = [GET: 'index', POST: 'index'] + action = [GET: 'index', POST: 'search'] } "/api/keywords"(controller: 'keyword', includes: ['index']) "/api/projects"(controller: 'project', includes: ['index']) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index c39d9ae..4eb4bdd 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -15,13 +15,14 @@ import nl.thehyve.datashowcase.exception.ResourceNotFoundException import nl.thehyve.datashowcase.representation.InternalItemRepresentation import nl.thehyve.datashowcase.representation.ItemRepresentation import nl.thehyve.datashowcase.representation.PublicItemRepresentation +import nl.thehyve.datashowcase.representation.SearchQueryRepresentation import nl.thehyve.datashowcase.search.SearchCriteriaBuilder import org.grails.core.util.StopWatch -import org.grails.web.json.JSONObject import org.hibernate.Criteria import org.hibernate.Session import org.hibernate.SessionFactory import org.hibernate.criterion.Criterion +import org.hibernate.criterion.Order import org.hibernate.criterion.Projections import org.hibernate.criterion.Restrictions import org.hibernate.transform.Transformers @@ -47,6 +48,7 @@ class ItemService { new ItemRepresentation( id: itemData.id as Long, name: itemData.name as String, + label: itemData.label as String, labelLong: itemData.labelLong as String, project: itemData.projectName as String, concept: itemData.conceptCode as String, @@ -70,6 +72,7 @@ class ItemService { i.publicItem as publicItem, i.itemPath as itemPath, c.conceptCode as conceptCode, + c.label as label, c.labelLong as labelLong, c.variableType as type, p.name as projectName @@ -79,6 +82,7 @@ class ItemService { ${dataShowcaseEnvironment.internalInstance ? '' : 'where i.publicItem = true' } + order by p.name asc """ ).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) .list() as List @@ -93,7 +97,7 @@ class ItemService { } @Transactional(readOnly = true) - List getItems(Set concepts, Set projects, Map searchQuery) { + List getItems(Set concepts, Set projects, SearchQueryRepresentation searchQuery) { Criterion searchQueryCriterion = searchQuery ? searchCriteriaBuilder.buildCriteria(searchQuery) : null def stopWatch = new StopWatch('Fetch filtered items') @@ -110,6 +114,7 @@ class ItemService { .add(Projections.property("i.publicItem").as("publicItem")) .add(Projections.property("i.itemPath").as("itemPath")) .add(Projections.property("c.conceptCode").as("conceptCode")) + .add(Projections.property("c.label").as("label")) .add(Projections.property("c.labelLong").as("labelLong")) .add(Projections.property("c.variableType").as("variableType")) .add(Projections.property("p.name").as("projectName"))) @@ -119,12 +124,13 @@ class ItemService { if(projects) { criteria.add( Restrictions.in('p.name', projects)) } - if(dataShowcaseEnvironment.internalInstance) { + if(!dataShowcaseEnvironment.internalInstance) { criteria.add( Restrictions.eq('i.publicItem',true)) } if(searchQueryCriterion) { criteria.add(searchQueryCriterion) } + criteria.addOrder(Order.asc('i.name')) criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) def items = criteria.list() as List @@ -135,9 +141,8 @@ class ItemService { map(itemData) } stopWatch.stop() - log.info "Filtered items fetched.\n${stopWatch.prettyPrint()}" + log.info "${result.size()} filtered items fetched.\n${stopWatch.prettyPrint()}" result - } @CacheEvict(value = 'items', allEntries = true) diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/Operator.groovy similarity index 85% rename from data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy rename to data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/Operator.groovy index f5b0016..4c338b0 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/Operator.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/Operator.groovy @@ -1,14 +1,12 @@ -package nl.thehyve.datashowcase.search +package nl.thehyve.datashowcase.enumeration import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.springframework.stereotype.Component /** * Operator types supported by the criteria builder. */ @CompileStatic -@Component @Slf4j enum Operator { @@ -44,7 +42,7 @@ enum Operator { if (mapping.containsKey(symbol)) { return mapping[symbol] } else { - log.error "Unknown operator: ${symbol}" + log.debug "Unknown operator: ${symbol}" return NONE } } diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchField.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/SearchField.groovy similarity index 89% rename from data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchField.groovy rename to data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/SearchField.groovy index d143e05..541096c 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchField.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/SearchField.groovy @@ -1,14 +1,12 @@ -package nl.thehyve.datashowcase.search +package nl.thehyve.datashowcase.enumeration import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import org.springframework.stereotype.Component /** * The fields on which it is possible to search (a free text filter). */ @CompileStatic -@Component @Slf4j enum SearchField { /** @@ -58,7 +56,7 @@ enum SearchField { if (mapping.containsKey(name)) { return mapping[name] } else { - log.error "Unknown search field: ${name}" + log.debug "Unknown search field: ${name}" return NONE } } diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/SearchQueryRepresentation.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/SearchQueryRepresentation.groovy new file mode 100644 index 0000000..c62d437 --- /dev/null +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/SearchQueryRepresentation.groovy @@ -0,0 +1,10 @@ +package nl.thehyve.datashowcase.representation + +import groovy.transform.CompileStatic + +@CompileStatic +class SearchQueryRepresentation { + String type + String value + List values +} diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy index 512e08b..660a58e 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy @@ -1,11 +1,14 @@ package nl.thehyve.datashowcase.search +import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nl.thehyve.datashowcase.enumeration.Operator +import nl.thehyve.datashowcase.enumeration.SearchField +import nl.thehyve.datashowcase.representation.SearchQueryRepresentation import org.hibernate.criterion.Criterion import org.hibernate.criterion.MatchMode import org.hibernate.criterion.Restrictions -import sun.reflect.generics.reflectiveObjects.NotImplementedException /** * The class for parsing search criteria from free text filter, serialized as JSON @@ -21,110 +24,34 @@ class SearchCriteriaBuilder { private final Operator defaultOperator = Operator.CONTAINS - /** - * Construct criteria from JSON query - * @param query - * @return - */ - Criterion buildCriteria(Map query) { - if (query == null) { - return null - } - def type = query.type as String - if (type == 'string') { - List values = [] - values.addAll(query.value) - List criteria = buildCriteriaFromChunks(values) - if (criteria.size() > 1) { - throw new IllegalArgumentException("Specified search query is invalid.") - } - return criteria.first() - } else { - def operator = Operator.forSymbol(type) - if (isJunctionOperator(operator)) { - List values = [] - List criteria = [] - - query.values.each { Map c -> - if (c.type == "string") { - if (c.value != ",") { - values.add(c.value) - } - } else { - criteria.add(buildCriteria(c)) - } - } - criteria.addAll(buildCriteriaFromChunks(values)) - Criterion[] criteriaArray = criteria.collect { it } - return expressionToCriteria(operator, criteriaArray) - } else { - throw new IllegalArgumentException("Unsupported search type: ${type}.") - } - } - } + private static final EnumSet booleanOperators = EnumSet.of(Operator.NOT, Operator.AND, Operator.OR) + private static final EnumSet valueOperators = EnumSet.of( + Operator.EQUALS, + Operator.NOT_EQUALS, + Operator.CONTAINS, + Operator.LIKE, + Operator.IN + ) /** - * Build criteria for each { "type": "string", "value": ""} element (chunk) - * Where chunk can be a representation of search field, operator or values - * @param values - list of "" from all elements (chunks) + * Returns true if operator equals AND or OR + * @param operator * @return */ - private List buildCriteriaFromChunks(List values) { - List criteria = [] - int size = values.size() - if (size > 0) { - if (size == 1) { - criteria.add(applyToAllSearchFields(defaultOperator, values[0])) - return criteria - } else if (size == 2) { - if (Operator.forSymbol((String) values[0]) != Operator.NONE) { - def criteriaForAllFields = applyToAllSearchFields(Operator.forSymbol((String) values[0]), values[1]) - criteria.add(criteriaForAllFields) - return criteria - } - } else { - Operator op = Operator.forSymbol((String) values[1]) - if (op != Operator.NONE) { - if (op == Operator.IN) { - criteria.add(triplesChunksToCriteria(values)) - } else { - def chunks = values.collate(3) - chunks.each { criteria.add(triplesChunksToCriteria(it)) } - } - return criteria - } - } - criteria.add(applyToAllSearchFields(Operator.IN, values)) - } - - return criteria + private static boolean isBooleanOperator(Operator operator) { + booleanOperators.contains(operator) } /** - * Create criteria from triple ["property", "operator", "value(s)"] - * @param chunks + * Returns true if the operator is a value operator + * @param operator * @return */ - private static Criterion triplesChunksToCriteria(List chunks) { - String nameElementString = chunks[0] - String operatorSymbol = chunks[1] - String[] values = chunks[2.. valueList = new ArrayList() - value.each { valueList.add(it.toString()) } - return Restrictions.in(propertyName, valueList) default: throw new IllegalArgumentException("Unsupported operator: ${operator}.") } } /** - * Returns true if operator equals AND or OR + * Create single Restriction criterion for a specified operator * @param operator + * @param propertyName + * @param value * @return */ - private static boolean isJunctionOperator(Operator operator) { - return operator == Operator.AND || operator == Operator.OR + private static Criterion buildSingleCriteria(Operator operator, SearchField field, List values) { + String propertyName = searchFieldToPropertyName(field) + switch (operator) { + case Operator.CONTAINS: + case Operator.EQUALS: + case Operator.LIKE: + def criteria = values.collect { applyOperator(operator, propertyName, it) } + if (criteria.size() == 1) { + return criteria[0] + } else { + return Restrictions.or(criteria.toArray() as Criterion[]) + } + case Operator.NOT_EQUALS: + def criteria = values.collect { applyOperator(operator, propertyName, it) } + def arg + if (criteria.size() == 1) { + arg = criteria[0] + } else { + arg = Restrictions.or(criteria.toArray() as Criterion[]) + } + return Restrictions.not(arg) + case Operator.IN: + return Restrictions.in(propertyName, values) + default: + throw new IllegalArgumentException("Unsupported operator: ${operator}.") + } } - static Criterion expressionToCriteria(Operator operator, Criterion[] criteria) { + static Criterion expressionToCriteria(Operator operator, List criteria) { + log.info "Applying ${operator} to ${criteria.size()} arguments." switch (operator) { + case Operator.NOT: + if (criteria.size() != 1) { + throw new IllegalArgumentException("Not can only be applied to a single argument.") + } + log.info "Applying NOT to ${criteria.size()} argument." + return Restrictions.not(criteria[0]) case Operator.AND: - return Restrictions.and(criteria) + log.info "Applying AND to ${criteria.size()} arguments." + return Restrictions.and(criteria.toArray() as Criterion[]) case Operator.OR: - return Restrictions.or(criteria) + log.info "Applying OR to ${criteria.size()} arguments." + return Restrictions.or(criteria.toArray() as Criterion[]) + default: + throw new IllegalArgumentException("Unsupported operator: ${operator}.") } } - // TODO implement negation criterion - private static Criterion negateExpression(Criterion c) { - throw new NotImplementedException() - // return Restrictions.not(c) - } - /** * If searchField is not specified, search query is applied to all supported properties * @param operator * @param value * @return */ - private static Criterion applyToAllSearchFields(Operator operator, Object value) { + private static Criterion applyToAllSearchFields(Operator operator, List values) { List criteria = [] SearchField.values().each { SearchField field -> if (field != SearchField.NONE) { - def singleCriteria = buildSingleCriteria(operator, searchFieldToPropertyName(field), value) + def singleCriteria = buildSingleCriteria(operator, field, values) criteria.add(singleCriteria) } } - Criterion[] criteriaArray = criteria.collect { it } - return expressionToCriteria(Operator.OR, criteriaArray) + return expressionToCriteria(Operator.OR, criteria) } /** @@ -208,4 +161,53 @@ class SearchCriteriaBuilder { return SearchField.NONE.value } } + + /** + * Construct criteria from JSON query + * @param query + * @return + */ + Criterion buildCriteria(SearchQueryRepresentation query) { + log.info "Build criteria: ${query}" + if (query == null) { + return null + } + def type = query.type + if (type == null) { + log.error "Unexpected null type in object: ${JsonOutput.toJson(query)}" + return null + } + else if (type == 'string') { + def values = [query.value as String] + log.info "Applying default operator ${defaultOperator} on args: ${values}" + return applyToAllSearchFields(defaultOperator, values) + } else { + def operator = Operator.forSymbol(type) + if (isBooleanOperator(operator)) { + return expressionToCriteria(operator, query.values.collect { buildCriteria(it) }) + } else if (isValueOperator(operator)) { + def propertyName = query.value + def property = SearchField.NONE + def args = query.values.collect { it.value } + if (propertyName) { + property = SearchField.forName(propertyName) + if (property == SearchField.NONE) { + throw new IllegalArgumentException("Unsupported property: ${propertyName}.") + } + } + if (property != SearchField.NONE) { + log.info "Applying ${operator} on field ${property} with args: ${args}" + // applying an operator to a field with a list of values + return buildSingleCriteria(operator, property, args) + } else { + log.info "Applying unary ${operator} on args: ${args}" + // applying a unary operator to all fields with a list of values + return applyToAllSearchFields(operator, args) + } + } else { + throw new IllegalArgumentException("Unsupported type: ${type}.") + } + } + } + } diff --git a/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts b/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts index be1c8fb..a9004a7 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/filters.module.ts @@ -1,7 +1,7 @@ import { NgModule } from '@angular/core'; import { CommonModule } from '@angular/common'; import {FormsModule} from "@angular/forms"; -import {AutoCompleteModule, ButtonModule, FieldsetModule, ListboxModule, PanelModule} from "primeng/primeng"; +import {AutoCompleteModule, ButtonModule, FieldsetModule, ListboxModule, PanelModule, TooltipModule} from "primeng/primeng"; import {FiltersComponent} from "./filters.component"; import {CheckboxFilterComponent} from "./checkbox-filter/checkbox-filter.component"; import {TextFilterComponent} from "./text-filter/text-filter.component"; @@ -9,14 +9,14 @@ import {SearchParserService} from "../services/search-parser.service"; @NgModule({ imports: [ + AutoCompleteModule, + ButtonModule, CommonModule, + FieldsetModule, FormsModule, - AutoCompleteModule, - PanelModule, ListboxModule, - FieldsetModule, - AutoCompleteModule, - ButtonModule + PanelModule, + TooltipModule ], declarations: [FiltersComponent, TextFilterComponent, CheckboxFilterComponent], exports: [FiltersComponent], diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css index e6d2196..2481ad8 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css @@ -16,3 +16,9 @@ color: red; text-align: center; } + +.active-search-filter { + opacity: .8; + background-color: #eee; + padding: 1px 5px; +} diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html index f9e0fbe..311a974 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html @@ -15,8 +15,13 @@
+ diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts index 2fc87fc..bafe736 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts @@ -19,6 +19,8 @@ export class TextFilterComponent implements OnInit { textFilter: string; // search error message searchErrorMessage: string = ''; + // search query as html + searchQueryHtml: string = ''; // the delay before triggering updating methods delay: number; @@ -51,8 +53,15 @@ export class TextFilterComponent implements OnInit { onFiltering(event) { this.dataService.clearErrorSearchMessage(); - let jsonQuery = SearchParserService.parse(this.textFilter); - this.dataService.setJsonSearchQuery(jsonQuery); + try { + this.searchQueryHtml = ''; + let query = SearchParserService.parse(this.textFilter); + this.searchQueryHtml = SearchParserService.toHtml(query); + this.dataService.setSearchQuery(query); + } catch(e) { + console.error(`${e}`, e); + this.searchErrorMessage = `${e}`; + } } /* diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html index 872a064..dc657ed 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html @@ -27,7 +27,7 @@ [sortable]="true"> - diff --git a/data-showcase/src/main/user-interface/src/app/models/binary-tree.ts b/data-showcase/src/main/user-interface/src/app/models/binary-tree.ts new file mode 100644 index 0000000..d1eda71 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/models/binary-tree.ts @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +export type LexemeType = 'and' | 'or' | 'string' | 'not'; + +export class Lexeme { + type: LexemeType; + value: string; + + static forType(type: LexemeType): Lexeme { + let result = new Lexeme(); + result.type = type; + return result; + } +} + +export class BinaryTree { + lexeme: Lexeme; + left: BinaryTree; + right: BinaryTree; + + static forBranches(type: LexemeType, left: BinaryTree, right: BinaryTree) : BinaryTree { + let result = new BinaryTree(); + result.lexeme = Lexeme.forType(type); + result.left = left; + result.right = right; + return result; + } +} diff --git a/data-showcase/src/main/user-interface/src/app/models/search-query.ts b/data-showcase/src/main/user-interface/src/app/models/search-query.ts new file mode 100644 index 0000000..08dfe41 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/models/search-query.ts @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +export type QueryType = 'and' | 'or' | 'string' | 'not' | '=' | '!=' | 'contains' | 'like' | 'in'; + +export class SearchQuery { + type: QueryType; + value: string; + values: SearchQuery[]; + + static forValues(type: QueryType, values: SearchQuery[]) : SearchQuery { + let result = new SearchQuery(); + result.type = type; + result.values = values; + return result; + } + + static forValue(value: string) : SearchQuery { + let result = new SearchQuery(); + result.type = 'string'; + result.value = value; + return result; + } + +} diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 44cd57d..101d2ed 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -13,7 +13,6 @@ import {Project} from "../models/project"; import {Subject} from "rxjs/Subject"; import {BehaviorSubject} from "rxjs/BehaviorSubject"; import {Environment} from "../models/environment"; -import {Concept} from '../models/concept'; import {CheckboxOption} from '../models/CheckboxOption'; type LoadingState = 'loading' | 'complete'; @@ -40,8 +39,8 @@ export class DataService { private textFilterInputSource = new Subject(); public textFilterInput$ = this.textFilterInputSource.asObservable(); - // JSON search query - private jsonSearchQuery: JSON = null; + // Search query + private searchQuery: Object = null; // trigger checkboxFilters reload private rerenderCheckboxFiltersSource = new Subject(); @@ -80,9 +79,8 @@ export class DataService { public environment$ = this.environmentSource.asObservable(); constructor(private resourceService: ResourceService) { - this.fetchAllProjects(); + this.fetchAllProjectsAndItems(); this.fetchAllTreeNodes(); - this.fetchItems(); this.setEnvironment(); } @@ -164,16 +162,25 @@ export class DataService { // ------------------------- filters and item table ------------------------- - fetchAllProjects() { + fetchAllProjectsAndItems() { this.resourceService.getProjects() .subscribe( (projects: Project[]) => { this.allProjects = projects; + this.fetchItems(); }, err => console.error(err) ); } + projectToResearchLine(projectName: string): string { + if (this.allProjects) { + return this.allProjects.find(p => p.name == projectName).lineOfResearch; + } else { + return null; + } + } + fetchItems() { let t1 = new Date(); console.debug(`Fetching items ...`); @@ -184,21 +191,18 @@ export class DataService { let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); let codes = Array.from(selectedConceptCodes); let projects = this.getProjectsForSelectedResearchLines(); - let searchQuery = this.jsonSearchQuery; //JSON.parse(JSON.stringify(this.jsonSearchQuery)); - this.resourceService.getItems(codes, projects, searchQuery).subscribe( + this.resourceService.getItems(codes, projects, this.searchQuery).subscribe( (items: Item[]) => { for (let item of items) { - if (this.allProjects) { - item['lineOfResearch'] = this.allProjects.find(p => p.name == item['project']).lineOfResearch; - } + item.lineOfResearch = this.projectToResearchLine(item.project); this.filteredItems.push(item); } this.loadingItems = "complete"; this.getUniqueFilterValues(); }, err => { - if(err != String(undefined) && err.startsWith("400")){ + if (err != String(undefined)) { this.searchErrorMessageSource.next(err); } console.error(err); @@ -292,8 +296,8 @@ export class DataService { this.textFilterInputSource.next(text); } - setJsonSearchQuery(query: JSON) { - this.jsonSearchQuery = query; + setSearchQuery(query: Object) { + this.searchQuery = query; this.fetchItems(); } diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index 37f000b..b88e273 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -38,9 +38,13 @@ export class ResourceService { private handleError(error: Response | any) { let errMsg: string; if (error instanceof Response) { - const body = error.json() || ''; - const err = body.error || JSON.stringify(body); - errMsg = `${error.status} - ${error.statusText || ''} ${err}`; + let contentType = error.headers.get('Content-Type') || ''; + if (contentType == 'application/json') { + const body = error.json() || ''; + errMsg = body.error || JSON.stringify(body); + } else { + errMsg = `Error: ${error.statusText}`; + } } else { errMsg = error.message ? error.message : error.toString(); } @@ -70,7 +74,7 @@ export class ResourceService { .catch(this.handleError.bind(this)); } - getItems(conceptCodes?: string[], projects?: string[], jsonSearchQuery?: JSON): Observable { + getItems(conceptCodes?: string[], projects?: string[], jsonSearchQuery?: Object): Observable { let headers = new Headers(); headers.append('Content-Type', 'application/json'); diff --git a/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts index 0a21a51..8d648d9 100644 --- a/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts @@ -1,5 +1,11 @@ import { Injectable } from '@angular/core'; import parser from 'logic-query-parser'; +import { QueryType, SearchQuery } from '../models/search-query'; +import { BinaryTree } from '../models/binary-tree'; + +const booleanOperators: Set = new Set(['and', 'or', 'not']); +const valueOperators: Set = new Set(['=', '!=', 'like', 'contains']); +const fields: Set = new Set(['name', 'keywords', 'label', 'labelLong', 'labelNl', 'labelNlLong']); @Injectable() export class SearchParserService { @@ -7,12 +13,121 @@ export class SearchParserService { constructor() { } + static toHtml(query: SearchQuery): string { + if (query == null) { + return ''; + } else if (query.type == 'string') { + return query.value; + } else if (query.type == 'not') { + let args = query.values.map(SearchParserService.toHtml).join(' '); + return `not (${args})`; + } if (query.type == 'and' || query.type == 'or') { + return query.values.map(SearchParserService.toHtml).join(` ${query.type} `); + } else if (valueOperators.has(query.type)) { + let property = query.value ? `${query.value} `: ''; + let args = query.values.map(SearchParserService.toHtml).join(' '); + return `${property}${query.type} ${args}`; + } else { + return query.values.map(SearchParserService.toHtml).join(' '); + } + } + + static parseQueryExpression(junctionOperator: QueryType, parts: string[]): SearchQuery { + if (parts.length == 0) { + return null + } + if (parts.length == 1) { + // single value, apply CONTAINS operator + return SearchQuery.forValue(parts[0]); + } + console.debug(`Parse query expression. parts = ${parts}`); + let property: string; + let operator: QueryType; + if (parts.length > 1) { + if (valueOperators.has(parts[0])) { + // unary operator applies to a list of values + operator = parts.shift() as QueryType; + console.debug(`Apply unary operator ${operator} to parts: ${parts}`); + return SearchQuery.forValues(operator, parts.map(value => SearchQuery.forValue(value))); + } + } + if (parts.length > 2) { + if (fields.has(parts[0]) && valueOperators.has(parts[1])) { + // binary operator applies to a field and a list of values + property = parts.shift(); + operator = parts.shift() as QueryType; + console.debug(`Apply operator ${operator} to field ${property} with values: ${parts}`); + let result = new SearchQuery(); + result.type = operator; + result.value = property; + result.values = parts.map(value => SearchQuery.forValue(value)); + return result; + } + } + return SearchQuery.forValues(junctionOperator, parts.map(value => SearchQuery.forValue(value))); + } + + + static processQuery(query: SearchQuery): SearchQuery { + if (query == null) { + return null; + } + let type = query.type; + if (!booleanOperators.has(type)) { + return query; + } + let values = query.values; + let valueTypes: Set = new Set(values.map(obj => obj.type)); + if (valueTypes.size == 1 && valueTypes.has('string')) { + // Only string values. Will parse the string expression. + return SearchParserService.parseQueryExpression(type, values.map(obj => obj.value)); + } else { + let result = new SearchQuery(); + result.type = type; + result.values = values.map(SearchParserService.processQuery); + return result; + } + } + + static parseNegation(binaryTree: BinaryTree): BinaryTree { + if (binaryTree == null) { + return null; + } + console.debug(`Parsing binary tree of type ${binaryTree.lexeme.type}...`, binaryTree); + if (binaryTree.lexeme.type == 'string') { + return binaryTree; + } else if (binaryTree.lexeme.type == 'and' || binaryTree.lexeme.type == 'or') { + if (binaryTree.left != null) { + let leftLexeme = binaryTree.left.lexeme; + if (leftLexeme.type == 'string' && leftLexeme.value.toLowerCase() == 'not') { + // apply 'not' operator to the right + return BinaryTree.forBranches('not', null, SearchParserService.parseNegation(binaryTree.right)); + } else { + // recursion + return BinaryTree.forBranches(binaryTree.lexeme.type, SearchParserService.parseNegation(binaryTree.left), SearchParserService.parseNegation(binaryTree.right)); + } + } else { + // left is null + return BinaryTree.forBranches(binaryTree.lexeme.type, null, SearchParserService.parseNegation(binaryTree.right)); + } + } else { + throw `Unexpected type: ${binaryTree.lexeme.type}`; + } + } + /* Generate binary tree with a logic query string as input * and parse it to JSON object, * using logic-query-parser library*/ - static parse(text: string) { - let binaryTree = parser.parse(text); - return parser.utils.binaryTreeToQueryJson(binaryTree); + static parse(text: string) : SearchQuery { + if (text == null || text.length == 0) { + return null; + } + let binaryTree = parser.parse(text) as BinaryTree; + binaryTree = SearchParserService.parseNegation(binaryTree); + let query = parser.utils.binaryTreeToQueryJson(binaryTree) as SearchQuery; + query = SearchParserService.processQuery(query); + console.debug(`Query`, query); + return query; } } diff --git a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html index 441fa13..b42df6f 100644 --- a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html +++ b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html @@ -35,7 +35,7 @@ [sortable]="true"> - From dd1b5f5efb6e01e962e316cb84da8197425041b1 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Tue, 14 Nov 2017 22:06:10 +0100 Subject: [PATCH 053/104] Fix tests. --- data-showcase/grails-app/conf/application.yml | 8 ++ .../datashowcase/TestController.groovy | 6 + .../e2e/features/search.feature | 28 ++--- .../e2e/features/treeview.feature | 2 +- .../e2e/stepdefinitions/cart.ts | 12 +- .../e2e/stepdefinitions/search.ts | 4 +- .../e2e/stepdefinitions/treeview.ts | 46 +++++--- .../main/user-interface/protractor.conf.js | 2 +- .../src/app/config/app.config.mock.ts | 2 +- .../datashowcase/PublicItemServiceSpec.groovy | 104 ++++++++++-------- 10 files changed, 130 insertions(+), 84 deletions(-) diff --git a/data-showcase/grails-app/conf/application.yml b/data-showcase/grails-app/conf/application.yml index 4676864..be2a6f6 100644 --- a/data-showcase/grails-app/conf/application.yml +++ b/data-showcase/grails-app/conf/application.yml @@ -103,6 +103,14 @@ environments: grails: cors: enabled: true + test: + grails: + cors: + enabled: true + testInternal: + grails: + cors: + enabled: true --- grails: diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TestController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TestController.groovy index 45e76b3..66d7c71 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TestController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/TestController.groovy @@ -20,14 +20,20 @@ class TestController { def clearDatabase() { dataService.clearDatabase() + dataService.clearCaches() + response.status = 200 } def createInternalData() { testService.createRandomInternalTestData() + dataService.clearCaches() + response.status = 200 } def createPublicData() { testService.createPublicTestData() + dataService.clearCaches() + response.status = 200 } } diff --git a/data-showcase/src/main/user-interface/e2e/features/search.feature b/data-showcase/src/main/user-interface/e2e/features/search.feature index dc8b3c6..8975a94 100644 --- a/data-showcase/src/main/user-interface/e2e/features/search.feature +++ b/data-showcase/src/main/user-interface/e2e/features/search.feature @@ -13,20 +13,20 @@ Feature: Data Showcase provides search functionality. (NTRREQ-41) [["heightB", "Height at time of survey", "Project B", "Research line 2", "height", "ui-btn"]] """ - Scenario: Search and filter based on keyword - Given I select keywords 'Family related' - Then the data table contains - """ - [["heightB", "Height at time of survey", "Project B", "Research line 2", "height", "ui-btn"]] - """ - - Scenario: Search and filter based on multiple keywords - Given I select keywords 'Family related, Administration' - Then the data table contains - """ - [["ageA", "Age at time of survey", "Project A", "Research line 1", "age", "ui-btn"], - ["heightB", "Height at time of survey", "Project B", "Research line 2", "height", "ui-btn"]] - """ +# Scenario: Search and filter based on keyword +# Given I select keywords 'Family related' +# Then the data table contains +# """ +# [["heightB", "Height at time of survey", "Project B", "Research line 2", "height", "ui-btn"]] +# """ + +# Scenario: Search and filter based on multiple keywords +# Given I select keywords 'Family related, Administration' +# Then the data table contains +# """ +# [["ageA", "Age at time of survey", "Project A", "Research line 1", "age", "ui-btn"], +# ["heightB", "Height at time of survey", "Project B", "Research line 2", "height", "ui-btn"]] +# """ Scenario: Search and filter based on Research line Given I select Research lines 'Research line 2' diff --git a/data-showcase/src/main/user-interface/e2e/features/treeview.feature b/data-showcase/src/main/user-interface/e2e/features/treeview.feature index b4acbbd..e631c8e 100644 --- a/data-showcase/src/main/user-interface/e2e/features/treeview.feature +++ b/data-showcase/src/main/user-interface/e2e/features/treeview.feature @@ -22,4 +22,4 @@ Feature: As a researcher I want to view the data showcase catalogue from a tree. Given I open all tree nodes When I select all data in the data table When I select 'Age' - Then I see the counters Items selected '2' and total '1' + Then I see the counters Items selected '0', total '1' and number of pages '1' diff --git a/data-showcase/src/main/user-interface/e2e/stepdefinitions/cart.ts b/data-showcase/src/main/user-interface/e2e/stepdefinitions/cart.ts index a842191..a211b7d 100644 --- a/data-showcase/src/main/user-interface/e2e/stepdefinitions/cart.ts +++ b/data-showcase/src/main/user-interface/e2e/stepdefinitions/cart.ts @@ -26,13 +26,15 @@ defineSupportCode(({ Given, When, Then }) => { Then('the cart contains', function (string) { let tableRows: [string] = JSON.parse(string); - return countIs($('.ds-shopping-cart').$$(".ui-datatable-data > tr"), tableRows.length).then(() => { - return $('.ds-shopping-cart').$$(".ui-datatable-data > tr").map((row, rowIndex) => { // get all data rows - return row.$$('.ui-cell-data').map((cell, cellIndex) => { - return checkTextElement(cell, tableRows[rowIndex][cellIndex]); + return browser.driver.wait(function () { + return countIs($('.ds-shopping-cart').$$(".ui-datatable-data > tr"), tableRows.length).then(() => { + return $('.ds-shopping-cart').$$(".ui-datatable-data > tr").map((row, rowIndex) => { // get all data rows + return row.$$('.ui-cell-data').map((cell, cellIndex) => { + return checkTextElement(cell, tableRows[rowIndex][cellIndex]); + }) }) }) - }) + }, 1000000) }); Then('the cart is empty', function () { diff --git a/data-showcase/src/main/user-interface/e2e/stepdefinitions/search.ts b/data-showcase/src/main/user-interface/e2e/stepdefinitions/search.ts index 91a9da5..351eabd 100644 --- a/data-showcase/src/main/user-interface/e2e/stepdefinitions/search.ts +++ b/data-showcase/src/main/user-interface/e2e/stepdefinitions/search.ts @@ -12,7 +12,9 @@ let { defineSupportCode } = require('cucumber'); defineSupportCode(({ Given, When, Then }) => { Given(/^In the main search bar I type '(.*)'$/, function (searchText): Promise { - return $('.text-filter-container >p-autocomplete > span > input').sendKeys(searchText); + return $('.text-filter-container >p-autocomplete > span > input').sendKeys(searchText).then(()=> + $('.text-filter-container >button.export-button').click() + ); }); Given(/^I select keywords '(.*)'$/, function (keywordSting) { diff --git a/data-showcase/src/main/user-interface/e2e/stepdefinitions/treeview.ts b/data-showcase/src/main/user-interface/e2e/stepdefinitions/treeview.ts index 51aadab..6f4c5e6 100644 --- a/data-showcase/src/main/user-interface/e2e/stepdefinitions/treeview.ts +++ b/data-showcase/src/main/user-interface/e2e/stepdefinitions/treeview.ts @@ -5,7 +5,7 @@ */ import { Promise } from 'es6-promise'; -import { $, $$, by } from 'protractor'; +import { $, $$, by, browser, promise } from 'protractor'; import { checkTextElement, countIs, promiseTrue } from './support/util'; let { defineSupportCode } = require('cucumber'); @@ -20,7 +20,9 @@ defineSupportCode(({ Given, When, Then }) => { }); Given('I open all tree nodes', function (): Promise { - return toggleNode() + return browser.driver.wait(function () { + return openNodes() + }, 5000000) }); Then(/^I see the following nodes in the tree: '(.*)'$/, function (nodeText: string): Promise { @@ -46,21 +48,23 @@ defineSupportCode(({ Given, When, Then }) => { Then('the data table contains', function (string): Promise { let tableRows: [string] = JSON.parse(string); - return countIs($$(".ui-datatable-data > tr"), tableRows.length).then(() => { - return $$(".ui-datatable-data > tr").map((row, rowIndex) => { // get all data rows - return row.$$('.ui-cell-data').map((cell, cellIndex) => { - return checkTextElement(cell, tableRows[rowIndex][cellIndex]) + return browser.driver.wait(function () { + return countIs($$(".ui-datatable-data > tr"), tableRows.length).then(() => { + return $$(".ui-datatable-data > tr").map((row, rowIndex) => { // get all data rows + return row.$$('.ui-cell-data').map((cell, cellIndex) => { + return checkTextElement(cell, tableRows[rowIndex][cellIndex]) + }) + }) }) - }) - }) + }, 30000) }); When('I select all data in the data table', function (): Promise { return $('.ui-datatable-thead').$$('.ui-chkbox').click(); }); - Then('I see the counters Items selected \'{int}\' and total \'{int}\'', function (int, int2) { - let counts = [int, int2]; + Then('I see the counters Items selected \'{int}\', total \'{int}\' and number of pages \'{int}\'', function (int, int2, int3) { + let counts = [int, int2, int3]; return $$('.item-count-container > b').map((counter, index) => { return checkTextElement(counter, counts[index]); }) @@ -68,8 +72,22 @@ defineSupportCode(({ Given, When, Then }) => { }); -export function toggleNode() { - return $$(".fa-caret-right").click().then(() => { - return $$(".fa-caret-right").click() - }) +export function openNodes(): Promise { + return new Promise(function(resolve, reject) { + let elements = $$('.fa-caret-right').filter((element) => element.isDisplayed()); + elements.count().then((n) => { + if (n > 0) { + return elements.reduce((res, el) => { + el.click(); + return ++res; + }, 0).then((res) => { + openNodes() + .then(() => resolve(true)) + .catch((err) => reject(err)) + }) + } else { + resolve(false); + } + }); + }); } diff --git a/data-showcase/src/main/user-interface/protractor.conf.js b/data-showcase/src/main/user-interface/protractor.conf.js index 7b3d093..8ef4032 100644 --- a/data-showcase/src/main/user-interface/protractor.conf.js +++ b/data-showcase/src/main/user-interface/protractor.conf.js @@ -13,7 +13,7 @@ exports.config = { 'browserName': 'chrome', chromeOptions: { // Get rid of --ignore-certificate yellow warning - args: ['--no-sandbox', '--test-type=browser'], + args: ['--no-sandbox', '--test-type=browser', '--window-size=1400,1024'], // Set download path and avoid prompting for download even though // this is already the default on Chrome but for completeness prefs: { diff --git a/data-showcase/src/main/user-interface/src/app/config/app.config.mock.ts b/data-showcase/src/main/user-interface/src/app/config/app.config.mock.ts index bdd76a2..88cd181 100644 --- a/data-showcase/src/main/user-interface/src/app/config/app.config.mock.ts +++ b/data-showcase/src/main/user-interface/src/app/config/app.config.mock.ts @@ -9,7 +9,7 @@ export class AppConfigMock { private env: Object = null; constructor() { this.config = { - 'api-url': '', + 'api-url': 'http://localhost:8080', 'api-version': '', 'app-url': '' }; diff --git a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy index 4559715..6014486 100644 --- a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy +++ b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy @@ -7,8 +7,9 @@ package nl.thehyve.datashowcase import grails.testing.mixin.integration.Integration +import grails.web.databinding.DataBinder import groovy.util.logging.Slf4j -import org.grails.web.json.JSONObject +import nl.thehyve.datashowcase.representation.SearchQueryRepresentation import org.springframework.beans.factory.annotation.Autowired import org.springframework.transaction.annotation.Transactional import spock.lang.Requires @@ -31,6 +32,16 @@ class PublicItemServiceSpec extends Specification { @Autowired TestService testService + static class SearchQueryBinder implements DataBinder {} + + static final searchQueryBinder = new SearchQueryBinder() + + static SearchQueryRepresentation parseQuery(final Map values) { + def searchQuery = new SearchQueryRepresentation() + searchQueryBinder.bindData(searchQuery, values) + return searchQuery + } + def setupData() { log.info "Clear database ..." dataService.clearDatabase() @@ -58,109 +69,108 @@ class PublicItemServiceSpec extends Specification { void "test free text filter"() { given: setupData() + SearchQueryRepresentation searchQuery when: "Filter on single word without field and operator specified" - JSONObject searchQuery = ["type": "string", "value": "ageA"] + searchQuery = parseQuery(["type": "string", "value": "ageA"]) def items = itemService.getItems([] as Set, [] as Set, searchQuery) then: items.size() == 1 items*.name == ['ageA'] when: "Filter on words conjunction (OR) without field and operator specified" - searchQuery = ["type": "or", "values": [ + searchQuery = parseQuery(["type": "or", "values": [ ["type": "string", "value": "ageA"], ["type": "string", "value": "heightB"] - ]] + ]]) items = itemService.getItems([] as Set, [] as Set, searchQuery) then: items.size() == 2 items*.name as Set == ['ageA', 'heightB'] as Set when: "Filter on single word without field, operator (LIKE) is specified" - searchQuery = ["type" : "and", + searchQuery = parseQuery(["type" : "like", "values": [ - ["type": "string", "value": "LIKE"], ["type": "string", "value": "a_e%"] - ]] + ]]) items = itemService.getItems([] as Set, [] as Set, searchQuery) then: items.size() == 1 items*.name == ['ageA'] when: "Filter on single word with specified list of fields and operator ('keyword' IN '[]')" - searchQuery = ["type": "and", "values": [ - ["type": "string", "value": "keywords"], - ["type": "string", "value": "IN"], + searchQuery = parseQuery(["type": "in", "value": "keywords", "values": [ ["type": "string", "value": "Personal information"], - ["type": "string", "value": ","], ["type": "string", "value": "Family related"]] - ] + ]) items = itemService.getItems([] as Set, [] as Set, searchQuery) then: items.size() == 2 items*.name as Set == ['ageA', 'heightB'] as Set when: "Filter on words disjunction (AND) and '=' operator ('field1=val1 OR field2=val2')" - searchQuery = ["type": "and", "values": [ - ["type": "string", "value": "name"], - ["type": "string", "value": "="], - ["type": "string", "value": "ageA"], - ["type": "string", "value": "label"], - ["type": "string", "value": "="], - ["type": "string", "value": "ageB"] - ]] + searchQuery = parseQuery(["type": "and", "values": [ + ["type": "=", "value": "name", "values": [ + ["type": "string", "value": "ageA"] + ]], + ["type": "=", "value": "label", "values": [ + ["type": "string", "value": "ageB"] + ]] + ]]) items = itemService.getItems([] as Set, [] as Set, searchQuery) then: items.size() == 0 when: "Invalid field name specified" def invalidProperty = "test_field" - searchQuery = ["type": "and", "values": [ - ["type": "string", "value": invalidProperty], - ["type": "string", "value": "="], + searchQuery = parseQuery(["type": "=", "value": invalidProperty, "values": [ ["type": "string", "value": "value"] - ]] + ]]) itemService.getItems([] as Set, [] as Set, searchQuery) then: "Exception is thrown" IllegalArgumentException ex = thrown() - ex.message == "Specified property name: $invalidProperty is not supported." + ex.message == "Unsupported property: ${invalidProperty}." + + when: "Invalid operator specified" + def invalidOperator = "~" + searchQuery = parseQuery(["type": invalidOperator, "value": "label", "values": [ + ["type": "string", "value": "value"] + ]]) + itemService.getItems([] as Set, [] as Set, searchQuery) + then: "Exception is thrown" + IllegalArgumentException ex2 = thrown() + ex2.message == "Unsupported type: ${invalidOperator}." when: "Brackets are used in junction query" // 'name = "ageA" OR (name = "heightA" AND label = "height")' - JSONObject searchQuery1 = ["type": "or", "values": [ - ["type": "and", "values": [ - ["type": "string", "value": "name"], - ["type": "string", "value": "="], + def searchQuery1 = parseQuery(["type": "or", "values": [ + ["type": "=", "value": "name", "values": [ ["type": "string", "value": "ageA"] ]], ["type": "and", "values": [ - ["type": "string", "value": "name"], - ["type": "string", "value": "="], - ["type": "string", "value": "heightB"], - ["type": "string", "value": "labelNl"], - ["type": "string", "value": "="], - ["type": "string", "value": "hoogte"] + ["type": "=", "value": "name", "values": [ + ["type": "string", "value": "heightB"] + ]], + ["type": "=", "value": "labelNl", "values": [ + ["type": "string", "value": "hoogte"] + ]] ]] - ]] + ]]) // '(name = "ageA" OR name = "heightA") AND label = "height"' - JSONObject searchQuery2 = ["type": "and", "values": [ + def searchQuery2 = parseQuery(["type": "and", "values": [ ["type": "or", "values": [ - ["type": "and", "values": [ - ["type": "string", "value": "name"], - ["type": "string", "value": "="], + ["type": "=", "value": "name", "values": [ ["type": "string", "value": "ageA"] ]], - ["type": "and", "values": [ - ["type": "string", "value": "name"], - ["type": "string", "value": "="], + ["type": "=", "value": "name", "values": [ ["type": "string", "value": "heightB"] ]] ]], - ["type": "string", "value": "labelNl"], - ["type": "string", "value": "="], - ["type": "string", "value": "hoogte"] - ]] + ["type": "=", "value": "labelNl", "values": [ + ["type": "string", "value": "hoogte"] + ]] + ]]) def itemsForQuery1 = itemService.getItems([] as Set, [] as Set, searchQuery1) def itemsForQuery2 = itemService.getItems([] as Set, [] as Set, searchQuery2) then: "Results are different, depending on the distribution of brackets" From 067fdb4c5a7b89c71c41cac5480769470ea0ab42 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Wed, 15 Nov 2017 00:34:40 +0100 Subject: [PATCH 054/104] Prevent duplicates in item table. --- .../src/main/user-interface/src/app/services/data.service.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 101d2ed..248e3a8 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -239,7 +239,6 @@ export class DataService { updateItemTable() { this.clearItemsSelection(); this.clearAllFilters(); - this.fetchItems(); } filterOnResearchLines(selectedResearchLines) { From a4daa1198e589b5d226e50359cff1fe3d36bac68 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Wed, 15 Nov 2017 00:34:51 +0100 Subject: [PATCH 055/104] Prevent duplicates in the shopping cart. --- .../src/main/user-interface/src/app/services/data.service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 248e3a8..0ea63c4 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -333,8 +333,9 @@ export class DataService { addToShoppingCart(newItemSelection: Item[]) { let newItems: Item[] = this.shoppingCartItems.getValue(); + let itemNames = newItems.map((item) => item.name); for (let item of newItemSelection) { - if (!newItems.includes(item)) { + if (!itemNames.includes(item.name)) { newItems.push(item); } } From de85c2d9b29d717f291507f4dbfefffd3d537afb Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Tue, 14 Nov 2017 10:33:54 +0100 Subject: [PATCH 056/104] Fix log message when fetching items --- .../src/main/user-interface/src/app/services/data.service.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 0ea63c4..6fc27d3 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -200,6 +200,8 @@ export class DataService { } this.loadingItems = "complete"; this.getUniqueFilterValues(); + let t2 = new Date(); + console.info(`Found ${this.filteredItems.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); }, err => { if (err != String(undefined)) { @@ -209,9 +211,6 @@ export class DataService { this.clearCheckboxFilters(); } ); - let t2 = new Date(); - console.info(`Found ${this.filteredItems.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); - } clearErrorSearchMessage(){ From cc01d661586ac54413ebf25dc2a777bfae3c7dee Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 15 Nov 2017 14:58:01 +0100 Subject: [PATCH 057/104] Add pagination for items --- .../datashowcase/ItemController.groovy | 19 ++++- .../datashowcase/ProjectController.groovy | 39 ++++++++- .../thehyve/datashowcase/UrlMappings.groovy | 4 +- .../thehyve/datashowcase/ItemService.groovy | 35 ++++++-- .../datashowcase/ProjectService.groovy | 74 ++++++++++++++++ .../app/item-table/item-table.component.html | 8 +- .../app/item-table/item-table.component.ts | 20 ++++- .../src/app/item-table/item-table.module.ts | 5 +- .../src/app/services/data.service.ts | 84 ++++++++++++++----- .../src/app/services/resource.service.ts | 40 ++++++--- 10 files changed, 280 insertions(+), 48 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index eba92f4..b5e751a 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -23,10 +23,17 @@ class ItemController { */ def index() { try { + + def args = request.JSON as Map + int firstResult = args.firstResult + int maxResults = args.maxResults + String order = args.order + String propertyName = args.propertyName + response.status = 200 response.contentType = 'application/json' response.characterEncoding = 'utf-8' - def data = [items: itemService.items] + def data = [items: itemService.getItems(firstResult, maxResults, order, propertyName)] log.info "Writing ${data.items.size()} items ..." new ObjectMapper().writeValue(response.outputStream, data) } catch (Exception e) { @@ -43,9 +50,14 @@ class ItemController { */ def search() { try { + def args = request.JSON as Map Set concepts = args.conceptCodes as Set Set projects = args.projects as Set + int firstResult = args.firstResult + int maxResults = args.maxResults + String order = args.order + String propertyName = args.propertyName log.info "Query input: ${args.searchQuery}" def searchQuery = new SearchQueryRepresentation() bindData(searchQuery, args.searchQuery) @@ -54,8 +66,11 @@ class ItemController { response.contentType = 'application/json' response.characterEncoding = 'utf-8' - def data = [items: itemService.getItems(concepts, projects, searchQuery)] + def data = [items: itemService.getItems( + firstResult, maxResults, order, propertyName, concepts, projects, searchQuery) + ] new ObjectMapper().writeValue(response.outputStream, data) + } catch (Exception e) { response.status = 400 log.error 'An error occurred when fetching items.', e diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy index 98b003a..27156fd 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy @@ -15,8 +15,43 @@ class ProjectController { @Autowired ProjectService projectService + /** + * Fetches all projects for items with filter criteria. + * Supported criteria: conceptCodes, free text search query. + * @return the list of projects as JSON. + */ def index() { - respond projects: projectService.projects - } + def args = getGetOrPostParams() + Set concepts = args.conceptCodes as Set + def searchQuery = args.searchQuery as Map + try { + if (concepts || searchQuery) { + respond projects: projectService.getProjects(concepts, searchQuery) + } else { + respond projects: projectService.projects + } + } catch (Exception e) { + response.status = 400 + log.error 'An error occurred when fetching projects.', e + respond error: "An error occurred when fetching projects. Error: $e.message" + } + } + /** + * Both GET and POST are supported for projects filtering + * Parameters can be either passed as request params or request body (JSON) + * @return a map of query parameters. + */ + protected Map getGetOrPostParams() { + if (request.method == "POST") { + return (Map)request.JSON + } + return params.collectEntries { String k, v -> + if (v instanceof Object[] || v instanceof List) { + [k, v.collect { URLDecoder.decode(it, 'UTF-8') }] + } else { + [k, URLDecoder.decode(v, 'UTF-8')] + } + } + } } diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy index 8261e47..20661eb 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy @@ -19,8 +19,10 @@ class UrlMappings { "/api/items"(controller: 'item') { action = [GET: 'index', POST: 'search'] } + "/api/projects"(controller: 'project') { + action = [GET: 'index', POST: 'index'] + } "/api/keywords"(controller: 'keyword', includes: ['index']) - "/api/projects"(controller: 'project', includes: ['index']) "/api/concepts"(controller: 'concept', includes: ['index']) "/api/lines_of_research"(controller: 'researchLine', includes: ['index']) "/api/tree_nodes"(controller: 'tree', includes: ['index']) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index 4eb4bdd..b9d3cf6 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -58,9 +58,21 @@ class ItemService { ) } + static String propertyNameFromRepresentationName(String name) { + switch (name){ + case "project": return "projectName" + case "concept": return "conceptCode" + case "label": return "label" + case "lineofresearch": return "p.lineOfResearch" + default: return "i.name" + } + } + @Cacheable('items') @Transactional(readOnly = true) - List getItems() { + List getItems(int firstResult, int maxResults, String order, String propertyName) { + def property = propertyNameFromRepresentationName(propertyName) + def stopWatch = new StopWatch('Fetch items') stopWatch.start('Retrieve from database') def session = sessionFactory.openStatelessSession() @@ -82,9 +94,13 @@ class ItemService { ${dataShowcaseEnvironment.internalInstance ? '' : 'where i.publicItem = true' } - order by p.name asc + order by $property ${order == 'desc' ? + 'desc' : 'asc' + } """ ).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) + .setFirstResult(firstResult) + .setMaxResults(maxResults) .list() as List stopWatch.stop() stopWatch.start('Map to representations') @@ -92,13 +108,15 @@ class ItemService { map(itemData) } stopWatch.stop() - log.info "Items fetched.\n${stopWatch.prettyPrint()}" + log.info "${result.size()} items fetched.\n${stopWatch.prettyPrint()}" result } @Transactional(readOnly = true) - List getItems(Set concepts, Set projects, SearchQueryRepresentation searchQuery) { + List getItems(int firstResult, int maxResults, String order, String propertyName, + Set concepts, Set projects, SearchQueryRepresentation searchQuery) { + def property = propertyNameFromRepresentationName(propertyName) Criterion searchQueryCriterion = searchQuery ? searchCriteriaBuilder.buildCriteria(searchQuery) : null def stopWatch = new StopWatch('Fetch filtered items') stopWatch.start('Retrieve from database') @@ -132,7 +150,13 @@ class ItemService { } criteria.addOrder(Order.asc('i.name')) criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) - + .setFirstResult(firstResult) + .setMaxResults(maxResults) + if (order == "desc") { + criteria.addOrder(Order.desc(property)) + } else { + criteria.addOrder(Order.asc(property)) + } def items = criteria.list() as List stopWatch.stop() @@ -204,5 +228,4 @@ class ItemService { } throw new ResourceNotFoundException('Item not found') } - } diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy index 1716555..a558021 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy @@ -7,20 +7,94 @@ package nl.thehyve.datashowcase import grails.gorm.transactions.Transactional +import groovy.transform.CompileStatic import nl.thehyve.datashowcase.representation.ProjectRepresentation +import nl.thehyve.datashowcase.search.SearchCriteriaBuilder +import org.grails.core.util.StopWatch +import org.hibernate.Criteria +import org.hibernate.HibernateException +import org.hibernate.SessionFactory +import org.hibernate.Transaction +import org.hibernate.criterion.Criterion +import org.hibernate.criterion.Projections +import org.hibernate.criterion.Restrictions import org.modelmapper.ModelMapper import org.springframework.beans.factory.annotation.Autowired @Transactional(readOnly = true) class ProjectService { + @Autowired + Environment dataShowcaseEnvironment + + @Autowired + SearchCriteriaBuilder searchCriteriaBuilder + + SessionFactory sessionFactory + @Autowired ModelMapper modelMapper + @CompileStatic + static ProjectRepresentation map(Map projectData) { + new ProjectRepresentation( + name: projectData.name as String, + lineOfResearch: projectData.lineOfResearch as String + ) + } + List getProjects() { Project.findAll().collect({ modelMapper.map(it, ProjectRepresentation) }) } + List getProjects(Set concepts, Map searchQuery) { + + Criterion searchQueryCriterion = searchQuery ? searchCriteriaBuilder.buildCriteria(searchQuery) : null + def stopWatch = new StopWatch('Fetch projects filtered items') + stopWatch.start('Retrieve from database') + // stateless session does not support collections of associated objects + // http://forum.spring.io/forum/spring-projects/batch/37785-collections-cannot-be-fetched-by-a-stateless-session + def session = sessionFactory.openSession() + Transaction tx = null + try { + tx = session.beginTransaction(); + + Criteria criteria = session.createCriteria(Project, "p") + .createAlias("p.items", "i") + .createAlias("i.concept", "c") + .createAlias("c.keywords", "k") + .setProjection(Projections.projectionList() + .add(Projections.distinct(Projections.property("p.name").as("name"))) + .add(Projections.property("p.lineOfResearch").as("lineOfResearch"))) + if (concepts) { + criteria.add(Restrictions.in('c.conceptCode', concepts)) + } + if (dataShowcaseEnvironment.internalInstance) { + criteria.add(Restrictions.eq('i.publicItem', true)) + } + if (searchQueryCriterion) { + criteria.add(searchQueryCriterion) + } + criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) + def projects = criteria.list() as List + stopWatch.stop() + + stopWatch.start('Map to representations') + def result = projects.collect { Map projectData -> + map(projectData) + } + stopWatch.stop() + log.info "Projects fetched.\n${stopWatch.prettyPrint()}" + result + } catch (HibernateException e) { + if (tx != null) + tx.rollback() + e.printStackTrace() + } finally { + session.close() + } + } + } diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html index dc657ed..9f5fc58 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html @@ -8,10 +8,11 @@ @@ -41,12 +42,15 @@ icon="fa fa-question-circle"> + + +
Items selected: {{itemsSelection? itemsSelection.length : 0}}. - Total results in table: {{items? items.length : 0}}. + Total results in table: {{countItems()}}. Number of pages: {{pageCount()}}.
diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts index c8686f4..8fc6163 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts @@ -76,6 +76,24 @@ export class ItemTableComponent implements OnInit { } pageCount(): number { - return Math.ceil(this.items.length / this.rowsPerPage) + return Math.ceil(this.countItems() / this.rowsPerPage) + } + + countItems(): number { + return this.dataService.countItems() + } + + changeSort(event) { + console.log("Sort: " + event.field + ", " + event.order); + this.dataService.itemsOrder = event.order; + this.dataService.itemsPropertyName = event.field; + this.dataService.fetchItems(); + } + + paginate(event) { + console.log("PAGE: " + event.page); + this.dataService.itemsFirstResult = event.page * event.rows; + this.dataService.itemsMaxResults = event.rows; + this.dataService.fetchItems(); } } diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.module.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.module.ts index 2f0a54d..2e444b6 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.module.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.module.ts @@ -6,7 +6,7 @@ import {NgModule} from '@angular/core'; import {CommonModule} from '@angular/common'; -import {ButtonModule, DataTableModule, ListboxModule, PanelModule} from "primeng/primeng"; +import {ButtonModule, DataTableModule, ListboxModule, PaginatorModule, PanelModule} from "primeng/primeng"; import {FormsModule} from "@angular/forms"; import {ItemFilter, ItemTableComponent} from "./item-table.component"; @@ -17,7 +17,8 @@ import {ItemFilter, ItemTableComponent} from "./item-table.component"; PanelModule, DataTableModule, ListboxModule, - ButtonModule + ButtonModule, + PaginatorModule ], declarations: [ItemTableComponent, ItemFilter], exports: [ItemTableComponent, ItemFilter] diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 6fc27d3..dc2c87f 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -16,6 +16,7 @@ import {Environment} from "../models/environment"; import {CheckboxOption} from '../models/CheckboxOption'; type LoadingState = 'loading' | 'complete'; +type Order = 'asc' | 'desc'; @Injectable() export class DataService { @@ -34,6 +35,7 @@ export class DataService { public itemsSelection$ = this.itemsSelectionSource.asObservable(); // items added to the shopping cart public shoppingCartItems = new BehaviorSubject([]); + public totalItemsCount: number = 0; // text filter input private textFilterInputSource = new Subject(); @@ -42,6 +44,16 @@ export class DataService { // Search query private searchQuery: Object = null; + // item table pagination settings + // the first result to retrieve, numbered from '0' + public itemsFirstResult: number = 0; + // the maximum number of results + public itemsMaxResults: number = 8; + // ascending/descending order + public itemsOrder: number = 1; + // the property to order on + public itemsPropertyName: string = ""; + // trigger checkboxFilters reload private rerenderCheckboxFiltersSource = new Subject(); public rerenderCheckboxFilters$ = this.rerenderCheckboxFiltersSource.asObservable(); @@ -146,6 +158,7 @@ export class DataService { .subscribe( (nodes: TreeNode[]) => { this.loadingTreeNodes = 'complete'; + nodes.forEach( node => this.totalItemsCount += node.accumulativeItemCount); let treeNodes = this.processTreeNodes(nodes); treeNodes.forEach((function (node) { this.treeNodes.push(node); // to ensure the treeNodes pointer remains unchanged @@ -168,6 +181,10 @@ export class DataService { (projects: Project[]) => { this.allProjects = projects; this.fetchItems(); + for (let project of projects) { + this.projects.push({label: project.name, value: project.name}); + DataService.collectUnique(project.lineOfResearch, this.linesOfResearch); + } }, err => console.error(err) ); @@ -181,6 +198,28 @@ export class DataService { } } + fetchFilters() { + this.projects.length = 0; + this.projects.length = 0; + this.linesOfResearch.length = 0; + + let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); + let codes = Array.from(selectedConceptCodes); + + this.resourceService.getProjects(codes, this.searchQuery).subscribe( + (projects: Project[]) => { + for (let project of projects) { + this.allProjects.push(project); + this.projects.push({label: project.name, value: project.name}); + DataService.collectUnique(project.lineOfResearch, this.linesOfResearch); + } + }, + err => { + console.error(err); + } + ); + } + fetchItems() { let t1 = new Date(); console.debug(`Fetching items ...`); @@ -192,14 +231,18 @@ export class DataService { let codes = Array.from(selectedConceptCodes); let projects = this.getProjectsForSelectedResearchLines(); - this.resourceService.getItems(codes, projects, this.searchQuery).subscribe( + let order: Order = this.orderFlagToOrderName(this.itemsOrder); + + this.resourceService.getItems(this.itemsFirstResult, this.itemsMaxResults, order, this.itemsPropertyName, + codes, projects, this.searchQuery).subscribe( (items: Item[]) => { for (let item of items) { - item.lineOfResearch = this.projectToResearchLine(item.project); + if (this.allProjects && this.allProjects.length > 0) { + item.lineOfResearch = this.projectToResearchLine(item.project); + } this.filteredItems.push(item); } this.loadingItems = "complete"; - this.getUniqueFilterValues(); let t2 = new Date(); console.info(`Found ${this.filteredItems.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); }, @@ -213,6 +256,11 @@ export class DataService { ); } + + static orderFlagToOrderName(order: number){ + return order == 1 ? "asc" : "desc"; + } + clearErrorSearchMessage(){ this.searchErrorMessageSource.next(''); } @@ -245,6 +293,7 @@ export class DataService { this.selectedResearchLines = selectedResearchLines; this.clearItemsSelection(); this.fetchItems(); + this.getUniqueProjects(); } filterOnProjects(selectedProjects) { @@ -252,6 +301,7 @@ export class DataService { this.selectedProjects = selectedProjects; this.clearItemsSelection(); this.fetchItems(); + this.getUniqueLinesOfResearch(); } getProjectsForSelectedResearchLines(): string[] { @@ -299,22 +349,15 @@ export class DataService { this.fetchItems(); } - private getUniqueFilterValues() { - if (!this.projects.length && !this.selectedResearchLines.length - || !this.selectedProjects.length && !this.selectedResearchLines.length ) { - this.clearCheckboxFilters(); - for (let item of this.filteredItems) { - DataService.collectUnique(item.project, this.projects); - DataService.collectUnique(item.lineOfResearch, this.linesOfResearch); - } - } else if (!this.linesOfResearch.length) { - for (let item of this.filteredItems) { - DataService.collectUnique(item.lineOfResearch, this.linesOfResearch); - } - } else if (!this.projects.length) { - for (let item of this.filteredItems) { - DataService.collectUnique(item.project, this.projects); - } + private getUniqueProjects() { + for (let item of this.filteredItems) { + DataService.collectUnique(item.project, this.projects); + } + } + + private getUniqueLinesOfResearch() { + for (let project of this.allProjects) { + DataService.collectUnique(project.lineOfResearch, this.linesOfResearch); } } @@ -327,6 +370,9 @@ export class DataService { } } + countItems(): number { + return this.selectedTreeNode ? this.selectedTreeNode.accumulativeItemCount : this.totalItemsCount; + } // ------------------------- shopping cart ------------------------- diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index b88e273..617037e 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -74,21 +74,25 @@ export class ResourceService { .catch(this.handleError.bind(this)); } - getItems(conceptCodes?: string[], projects?: string[], jsonSearchQuery?: Object): Observable { + + getItems(firstResult: number, maxResults: number, order?: string,propertyName?: string, + conceptCodes?: string[], projects?: string[], jsonSearchQuery?: Object): Observable { + let headers = new Headers(); headers.append('Content-Type', 'application/json'); const options = new RequestOptions({headers: headers}); const url = this.endpoint.apiUrl + PATH_ITEMS; - let body = null; - if(projects || jsonSearchQuery) { - body = { + let body = { conceptCodes: conceptCodes, projects: projects, - searchQuery: jsonSearchQuery - } - } + searchQuery: jsonSearchQuery, + firstResult: firstResult, + maxResults: maxResults, + order: order, + propertyName: propertyName + }; // use POST because of url length limits in some of the browsers (limit of characters) return this.http.post(url, body, options) @@ -96,14 +100,24 @@ export class ResourceService { .catch(this.handleError.bind(this)); } - getProjects(): Observable { + getProjects(conceptCodes?: string[], jsonSearchQuery?: JSON): Observable { let headers = new Headers(); - let url = this.endpoint.apiUrl + PATH_PROJECTS; + headers.append('Content-Type', 'application/json'); - return this.http.get(url, { - headers: headers - }) - .map((response: Response) => response.json().projects as Project[]) + const options = new RequestOptions({headers: headers}); + const url = this.endpoint.apiUrl + PATH_PROJECTS; + let body = null; + + if(conceptCodes || jsonSearchQuery) { + body = { + conceptCodes: conceptCodes, + searchQuery: jsonSearchQuery + } + } + + // use POST because of url length limits in some of the browsers (limit of characters) + return this.http.post(url, body, options) + .map((res: Response) => res.json().projects as Project[]) .catch(this.handleError.bind(this)); } From 441d19dbc7ceb528fa037ec1baf000816106d828 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 16 Nov 2017 08:07:45 +0100 Subject: [PATCH 058/104] Change fetching projects, fix projects filter --- .../datashowcase/ProjectController.groovy | 40 +++++++------------ .../thehyve/datashowcase/UrlMappings.groovy | 2 +- .../datashowcase/ProjectService.groovy | 3 +- .../src/app/services/data.service.ts | 27 ++++++++----- .../src/app/services/resource.service.ts | 14 ++++++- 5 files changed, 47 insertions(+), 39 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy index 27156fd..468d0c5 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy @@ -6,6 +6,7 @@ package nl.thehyve.datashowcase +import nl.thehyve.datashowcase.representation.SearchQueryRepresentation import org.springframework.beans.factory.annotation.Autowired class ProjectController { @@ -15,43 +16,32 @@ class ProjectController { @Autowired ProjectService projectService + /** + * Fetches all projects + */ + def index() { + respond projects: projectService.projects + } + /** * Fetches all projects for items with filter criteria. * Supported criteria: conceptCodes, free text search query. * @return the list of projects as JSON. */ - def index() { - def args = getGetOrPostParams() + def search() { + def args = request.JSON as Map Set concepts = args.conceptCodes as Set - def searchQuery = args.searchQuery as Map + log.info "Query input: ${args.searchQuery}" + def searchQuery = new SearchQueryRepresentation() + bindData(searchQuery, args.searchQuery) try { - if (concepts || searchQuery) { - respond projects: projectService.getProjects(concepts, searchQuery) - } else { - respond projects: projectService.projects - } + respond projects: projectService.getProjects(concepts, searchQuery) } catch (Exception e) { response.status = 400 log.error 'An error occurred when fetching projects.', e respond error: "An error occurred when fetching projects. Error: $e.message" } } - /** - * Both GET and POST are supported for projects filtering - * Parameters can be either passed as request params or request body (JSON) - * @return a map of query parameters. - */ - protected Map getGetOrPostParams() { - if (request.method == "POST") { - return (Map)request.JSON - } - return params.collectEntries { String k, v -> - if (v instanceof Object[] || v instanceof List) { - [k, v.collect { URLDecoder.decode(it, 'UTF-8') }] - } else { - [k, URLDecoder.decode(v, 'UTF-8')] - } - } - } + } diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy index 20661eb..e38403f 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy @@ -20,7 +20,7 @@ class UrlMappings { action = [GET: 'index', POST: 'search'] } "/api/projects"(controller: 'project') { - action = [GET: 'index', POST: 'index'] + action = [GET: 'index', POST: 'search'] } "/api/keywords"(controller: 'keyword', includes: ['index']) "/api/concepts"(controller: 'concept', includes: ['index']) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy index a558021..b134bb5 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy @@ -9,6 +9,7 @@ package nl.thehyve.datashowcase import grails.gorm.transactions.Transactional import groovy.transform.CompileStatic import nl.thehyve.datashowcase.representation.ProjectRepresentation +import nl.thehyve.datashowcase.representation.SearchQueryRepresentation import nl.thehyve.datashowcase.search.SearchCriteriaBuilder import org.grails.core.util.StopWatch import org.hibernate.Criteria @@ -49,7 +50,7 @@ class ProjectService { }) } - List getProjects(Set concepts, Map searchQuery) { + List getProjects(Set concepts, SearchQueryRepresentation searchQuery) { Criterion searchQueryCriterion = searchQuery ? searchCriteriaBuilder.buildCriteria(searchQuery) : null def stopWatch = new StopWatch('Fetch projects filtered items') diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index dc2c87f..980df85 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -176,7 +176,7 @@ export class DataService { // ------------------------- filters and item table ------------------------- fetchAllProjectsAndItems() { - this.resourceService.getProjects() + this.resourceService.getAllProjects() .subscribe( (projects: Project[]) => { this.allProjects = projects; @@ -199,7 +199,6 @@ export class DataService { } fetchFilters() { - this.projects.length = 0; this.projects.length = 0; this.linesOfResearch.length = 0; @@ -256,8 +255,7 @@ export class DataService { ); } - - static orderFlagToOrderName(order: number){ + orderFlagToOrderName(order: number){ return order == 1 ? "asc" : "desc"; } @@ -347,19 +345,26 @@ export class DataService { setSearchQuery(query: Object) { this.searchQuery = query; this.fetchItems(); + this.fetchFilters(); } private getUniqueProjects() { - for (let item of this.filteredItems) { - DataService.collectUnique(item.project, this.projects); - } + this.allProjects.forEach( p => { + if(!this.selectedResearchLines.length){ + this.projects.push({label: p.name, value: p.name}); + } else { + if (this.selectedResearchLines.includes(p.lineOfResearch)) { + this.projects.push({label: p.name, value: p.name}); + } + } + }); } private getUniqueLinesOfResearch() { - for (let project of this.allProjects) { - DataService.collectUnique(project.lineOfResearch, this.linesOfResearch); - } - } + if(this.filteredItems.length) + for (let item of this.filteredItems) { + //DataService.collectUnique() + }} private static collectUnique(element, list: CheckboxOption[]) { let values = list.map(function (a) { diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index 617037e..a486ef9 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -100,7 +100,19 @@ export class ResourceService { .catch(this.handleError.bind(this)); } - getProjects(conceptCodes?: string[], jsonSearchQuery?: JSON): Observable { + getAllProjects(): Observable { + let headers = new Headers(); + let url = this.endpoint.apiUrl + PATH_PROJECTS; + + return this.http.get(url, { + headers: headers + }) + .map((response: Response) => response.json().projects as Project[]) + .catch(this.handleError.bind(this)); + } + + + getProjects(conceptCodes?: string[], jsonSearchQuery?: Object): Observable { let headers = new Headers(); headers.append('Content-Type', 'application/json'); From 7033737dbcadee7e7a863f3e1c2b6f6fb8cdeb10 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 16 Nov 2017 08:45:07 +0100 Subject: [PATCH 059/104] Fix projects and researchLines filters --- .../src/app/services/data.service.ts | 33 +++++++++++++------ 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 980df85..ad6fb67 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -349,22 +349,35 @@ export class DataService { } private getUniqueProjects() { - this.allProjects.forEach( p => { - if(!this.selectedResearchLines.length){ - this.projects.push({label: p.name, value: p.name}); - } else { - if (this.selectedResearchLines.includes(p.lineOfResearch)) { - this.projects.push({label: p.name, value: p.name}); + this.allProjects.forEach(ap => { + if (!this.projects.find(p => p.value == ap.name)){ + if (!this.selectedResearchLines.length) { + this.projects.push({label: ap.name, value: ap.name}); + } else { + if (this.selectedResearchLines.includes(ap.lineOfResearch)) { + this.projects.push({label: ap.name, value: ap.name}); + } } } }); } private getUniqueLinesOfResearch() { - if(this.filteredItems.length) - for (let item of this.filteredItems) { - //DataService.collectUnique() - }} + if (!this.selectedProjects.length) { + this.allProjects.forEach(p => { + if(!this.linesOfResearch.find(l=> l.value == p.lineOfResearch)) { + this.linesOfResearch.push({label: p.lineOfResearch, value: p.lineOfResearch}); + } + }); + } else { + this.selectedProjects.forEach(p => { + let researchLine = this.allProjects.find(ap => ap.name == p).lineOfResearch; + if(!this.linesOfResearch.find(l=> l.value == researchLine)) { + this.linesOfResearch.push({label: researchLine, value: researchLine}); + } + }); + } + } private static collectUnique(element, list: CheckboxOption[]) { let values = list.map(function (a) { From 80e5ae479c059fb7d785cc60427d30ba2dc3ecc0 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 16 Nov 2017 10:14:48 +0100 Subject: [PATCH 060/104] Change the response of /items call Return page, totalCount and items --- .../datashowcase/ItemController.groovy | 8 +++-- .../thehyve/datashowcase/ItemService.groovy | 35 ++++++++++++++++++- .../app/item-table/item-table.component.ts | 2 +- .../src/app/models/itemResponse.ts | 13 +++++++ .../src/app/services/data.service.ts | 10 +++--- .../src/app/services/resource.service.ts | 5 +-- 6 files changed, 60 insertions(+), 13 deletions(-) create mode 100644 data-showcase/src/main/user-interface/src/app/models/itemResponse.ts diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index b5e751a..530ee64 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -46,7 +46,7 @@ class ItemController { /** * Fetches all items with filter criteria. * Supported criteria: conceptCodes, projects, free text search query. - * @return the list of items as JSON. + * @return the list of items, total count and page number as JSON . */ def search() { try { @@ -66,9 +66,11 @@ class ItemController { response.contentType = 'application/json' response.characterEncoding = 'utf-8' - def data = [items: itemService.getItems( + def items = itemService.getItems( firstResult, maxResults, order, propertyName, concepts, projects, searchQuery) - ] + def count = itemService.getItemsCount(concepts, projects, searchQuery) + int page = (firstResult/maxResults + 1) + def data = [ totalCount: count, page: page, items: items] new ObjectMapper().writeValue(response.outputStream, data) } catch (Exception e) { diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index b9d3cf6..6cfbd57 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -148,7 +148,6 @@ class ItemService { if(searchQueryCriterion) { criteria.add(searchQueryCriterion) } - criteria.addOrder(Order.asc('i.name')) criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) .setFirstResult(firstResult) .setMaxResults(maxResults) @@ -158,6 +157,7 @@ class ItemService { criteria.addOrder(Order.asc(property)) } def items = criteria.list() as List + stopWatch.stop() stopWatch.start('Map to representations') @@ -169,6 +169,39 @@ class ItemService { result } + @Transactional(readOnly = true) + Long getItemsCount(Set concepts, Set projects, SearchQueryRepresentation searchQuery) { + + Criterion searchQueryCriterion = searchQuery ? searchCriteriaBuilder.buildCriteria(searchQuery) : null + def stopWatch = new StopWatch('Items count') + stopWatch.start('Retrieve from database') + def session = sessionFactory.openStatelessSession() + + Criteria criteria = session.createCriteria(Item, "i") + .createAlias("i.concept", "c") + .createAlias("i.project", "p") + .createAlias("c.keywords", "k") + if(concepts) { + criteria.add( Restrictions.in('c.conceptCode', concepts)) + } + if(projects) { + criteria.add( Restrictions.in('p.name', projects)) + } + if(!dataShowcaseEnvironment.internalInstance) { + criteria.add( Restrictions.eq('i.publicItem',true)) + } + if(searchQueryCriterion) { + criteria.add(searchQueryCriterion) + } + criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) + .setProjection(Projections.rowCount()) + Long totalItemsCount = (Long)criteria.uniqueResult() + + stopWatch.stop() + log.info "Total item count: ${totalItemsCount}" + totalItemsCount + } + @CacheEvict(value = 'items', allEntries = true) void clearItemsCache() { log.info "Clear items cache." diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts index 8fc6163..c9b555a 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts @@ -80,7 +80,7 @@ export class ItemTableComponent implements OnInit { } countItems(): number { - return this.dataService.countItems() + return this.dataService.totalItemsCount; } changeSort(event) { diff --git a/data-showcase/src/main/user-interface/src/app/models/itemResponse.ts b/data-showcase/src/main/user-interface/src/app/models/itemResponse.ts new file mode 100644 index 0000000..656bef0 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/models/itemResponse.ts @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +import {Item} from "./item"; + +export class ItemResponse { + items: Item[]; + totalCount: number; + page: number; +} diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index ad6fb67..74290f5 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -14,6 +14,7 @@ import {Subject} from "rxjs/Subject"; import {BehaviorSubject} from "rxjs/BehaviorSubject"; import {Environment} from "../models/environment"; import {CheckboxOption} from '../models/CheckboxOption'; +import {ItemResponse} from "../models/itemResponse"; type LoadingState = 'loading' | 'complete'; type Order = 'asc' | 'desc'; @@ -234,8 +235,9 @@ export class DataService { this.resourceService.getItems(this.itemsFirstResult, this.itemsMaxResults, order, this.itemsPropertyName, codes, projects, this.searchQuery).subscribe( - (items: Item[]) => { - for (let item of items) { + (response: ItemResponse) => { + this.totalItemsCount = response.totalCount; + for (let item of response.items) { if (this.allProjects && this.allProjects.length > 0) { item.lineOfResearch = this.projectToResearchLine(item.project); } @@ -388,10 +390,6 @@ export class DataService { } } - countItems(): number { - return this.selectedTreeNode ? this.selectedTreeNode.accumulativeItemCount : this.totalItemsCount; - } - // ------------------------- shopping cart ------------------------- addToShoppingCart(newItemSelection: Item[]) { diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index a486ef9..2fd6e00 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -21,6 +21,7 @@ import {Item} from "../models/item"; import {Project} from "../models/project"; import {Environment} from "../models/environment"; import { Concept } from '../models/concept'; +import {ItemResponse} from "../models/itemResponse"; @Injectable() @@ -76,7 +77,7 @@ export class ResourceService { getItems(firstResult: number, maxResults: number, order?: string,propertyName?: string, - conceptCodes?: string[], projects?: string[], jsonSearchQuery?: Object): Observable { + conceptCodes?: string[], projects?: string[], jsonSearchQuery?: Object): Observable { let headers = new Headers(); headers.append('Content-Type', 'application/json'); @@ -96,7 +97,7 @@ export class ResourceService { // use POST because of url length limits in some of the browsers (limit of characters) return this.http.post(url, body, options) - .map((res: Response) => res.json().items as Item[]) + .map((res: Response) => res.json() as ItemResponse) .catch(this.handleError.bind(this)); } From b14beccbb1be94aa9bf3ebba815292c9ea8ed97d Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 16 Nov 2017 10:20:03 +0100 Subject: [PATCH 061/104] Remove unnecessary counts --- .../src/main/user-interface/src/app/services/data.service.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 74290f5..5dfda23 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -159,7 +159,6 @@ export class DataService { .subscribe( (nodes: TreeNode[]) => { this.loadingTreeNodes = 'complete'; - nodes.forEach( node => this.totalItemsCount += node.accumulativeItemCount); let treeNodes = this.processTreeNodes(nodes); treeNodes.forEach((function (node) { this.treeNodes.push(node); // to ensure the treeNodes pointer remains unchanged From 19f06d6228ccf6c72d3de9811a258e57c0394add Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 16 Nov 2017 10:33:28 +0100 Subject: [PATCH 062/104] Fix tests --- .../src/app/app.component.spec.ts | 5 ++-- .../item-table/item-table.component.spec.ts | 5 ++-- .../datashowcase/PublicItemServiceSpec.groovy | 24 +++++++++++-------- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts index aad8c44..f1d1bac 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts @@ -9,7 +9,7 @@ import { TestBed, async } from '@angular/core/testing'; import { AppComponent } from './app.component'; import {TreeNodesComponent} from "./tree-nodes/tree-nodes.component"; import { - AutoCompleteModule, DataTableModule, DialogModule, FieldsetModule, PanelModule, + AutoCompleteModule, DataTableModule, DialogModule, FieldsetModule, PaginatorModule, PanelModule, TreeModule } from "primeng/primeng"; import {FormsModule} from "@angular/forms"; @@ -57,7 +57,8 @@ describe('AppComponent', () => { BrowserAnimationsModule, DialogModule, HttpModule, - InfoModule + InfoModule, + PaginatorModule ], providers: [ DataService, diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts index 2141f35..ee481c9 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts @@ -9,7 +9,7 @@ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import {ItemFilter, ItemTableComponent} from './item-table.component'; import {FormsModule} from "@angular/forms"; import { - AutoCompleteModule, DataListModule, DataTableModule, FieldsetModule, ListboxModule, + AutoCompleteModule, DataListModule, DataTableModule, FieldsetModule, ListboxModule, PaginatorModule, PanelModule } from "primeng/primeng"; import {BrowserAnimationsModule} from "@angular/platform-browser/animations"; @@ -35,7 +35,8 @@ describe('ItemTableComponent', () => { ListboxModule, BrowserAnimationsModule, DataTableModule, - HttpModule + HttpModule, + PaginatorModule ], providers: [ DataService, diff --git a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy index 6014486..309b1e7 100644 --- a/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy +++ b/data-showcase/src/test/groovy/nl/thehyve/datashowcase/PublicItemServiceSpec.groovy @@ -55,7 +55,7 @@ class PublicItemServiceSpec extends Specification { setupData() when: log.info "Running test ..." - def items = itemService.items + def items = itemService.getItems(0, 9999, 'asc', 'name') then: "2 items being returned" items.size() == 2 items*.name == ['ageA', 'heightB'] @@ -68,12 +68,16 @@ class PublicItemServiceSpec extends Specification { @Requires({ -> Environment.grailsEnvironmentIn(Constants.PUBLIC_ENVIRONMENTS) }) void "test free text filter"() { given: + int firstResult = 0 + int maxResults = 9999 + String order = 'asc' + String propertyName = 'name' setupData() SearchQueryRepresentation searchQuery when: "Filter on single word without field and operator specified" searchQuery = parseQuery(["type": "string", "value": "ageA"]) - def items = itemService.getItems([] as Set, [] as Set, searchQuery) + def items = itemService.getItems(firstResult, maxResults, order, propertyName, [] as Set, [] as Set, searchQuery) then: items.size() == 1 items*.name == ['ageA'] @@ -83,7 +87,7 @@ class PublicItemServiceSpec extends Specification { ["type": "string", "value": "ageA"], ["type": "string", "value": "heightB"] ]]) - items = itemService.getItems([] as Set, [] as Set, searchQuery) + items = itemService.getItems(firstResult, maxResults, order, propertyName, [] as Set, [] as Set, searchQuery) then: items.size() == 2 items*.name as Set == ['ageA', 'heightB'] as Set @@ -93,7 +97,7 @@ class PublicItemServiceSpec extends Specification { "values": [ ["type": "string", "value": "a_e%"] ]]) - items = itemService.getItems([] as Set, [] as Set, searchQuery) + items = itemService.getItems(firstResult, maxResults, order, propertyName, [] as Set, [] as Set, searchQuery) then: items.size() == 1 items*.name == ['ageA'] @@ -103,7 +107,7 @@ class PublicItemServiceSpec extends Specification { ["type": "string", "value": "Personal information"], ["type": "string", "value": "Family related"]] ]) - items = itemService.getItems([] as Set, [] as Set, searchQuery) + items = itemService.getItems(firstResult, maxResults, order, propertyName, [] as Set, [] as Set, searchQuery) then: items.size() == 2 items*.name as Set == ['ageA', 'heightB'] as Set @@ -117,7 +121,7 @@ class PublicItemServiceSpec extends Specification { ["type": "string", "value": "ageB"] ]] ]]) - items = itemService.getItems([] as Set, [] as Set, searchQuery) + items = itemService.getItems(firstResult, maxResults, order, propertyName, [] as Set, [] as Set, searchQuery) then: items.size() == 0 @@ -126,7 +130,7 @@ class PublicItemServiceSpec extends Specification { searchQuery = parseQuery(["type": "=", "value": invalidProperty, "values": [ ["type": "string", "value": "value"] ]]) - itemService.getItems([] as Set, [] as Set, searchQuery) + itemService.getItems(firstResult, maxResults, order, propertyName, [] as Set, [] as Set, searchQuery) then: "Exception is thrown" IllegalArgumentException ex = thrown() ex.message == "Unsupported property: ${invalidProperty}." @@ -136,7 +140,7 @@ class PublicItemServiceSpec extends Specification { searchQuery = parseQuery(["type": invalidOperator, "value": "label", "values": [ ["type": "string", "value": "value"] ]]) - itemService.getItems([] as Set, [] as Set, searchQuery) + itemService.getItems(firstResult, maxResults, order, propertyName, [] as Set, [] as Set, searchQuery) then: "Exception is thrown" IllegalArgumentException ex2 = thrown() ex2.message == "Unsupported type: ${invalidOperator}." @@ -171,8 +175,8 @@ class PublicItemServiceSpec extends Specification { ["type": "string", "value": "hoogte"] ]] ]]) - def itemsForQuery1 = itemService.getItems([] as Set, [] as Set, searchQuery1) - def itemsForQuery2 = itemService.getItems([] as Set, [] as Set, searchQuery2) + def itemsForQuery1 = itemService.getItems(firstResult, maxResults, order, propertyName, [] as Set, [] as Set, searchQuery1) + def itemsForQuery2 = itemService.getItems(firstResult, maxResults, order, propertyName, [] as Set, [] as Set, searchQuery2) then: "Results are different, depending on the distribution of brackets" itemsForQuery1 != itemsForQuery2 itemsForQuery1.size() == 2 From 500cf58239f59e6e85dd8c5a656db7a4257019de Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 16 Nov 2017 11:22:17 +0100 Subject: [PATCH 063/104] Remove unnecessarily defined transaction --- .../services/nl/thehyve/datashowcase/ProjectService.groovy | 5 ----- 1 file changed, 5 deletions(-) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy index b134bb5..5941aef 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy @@ -58,10 +58,7 @@ class ProjectService { // stateless session does not support collections of associated objects // http://forum.spring.io/forum/spring-projects/batch/37785-collections-cannot-be-fetched-by-a-stateless-session def session = sessionFactory.openSession() - Transaction tx = null try { - tx = session.beginTransaction(); - Criteria criteria = session.createCriteria(Project, "p") .createAlias("p.items", "i") .createAlias("i.concept", "c") @@ -90,8 +87,6 @@ class ProjectService { log.info "Projects fetched.\n${stopWatch.prettyPrint()}" result } catch (HibernateException e) { - if (tx != null) - tx.rollback() e.printStackTrace() } finally { session.close() From c8c1fff5609ff09458cea91060588c3d63010494 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Thu, 16 Nov 2017 12:12:53 +0100 Subject: [PATCH 064/104] Replace CheckboxOption with SelectItem from PrimeNG. --- .../checkbox-filter/checkbox-filter.component.ts | 7 +++---- .../user-interface/src/app/models/CheckboxOption.ts | 10 ---------- .../user-interface/src/app/services/data.service.ts | 11 +++++------ 3 files changed, 8 insertions(+), 20 deletions(-) delete mode 100644 data-showcase/src/main/user-interface/src/app/models/CheckboxOption.ts diff --git a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts index 7e3d1fd..e3bef62 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.ts @@ -6,8 +6,7 @@ import {Component, OnInit} from '@angular/core'; import {DataService} from "../../services/data.service"; -import {Item} from "../../models/item"; -import { CheckboxOption } from '../../models/CheckboxOption'; +import { SelectItem } from 'primeng/primeng'; @Component({ selector: 'app-checkbox-filter', @@ -19,8 +18,8 @@ export class CheckboxFilterComponent implements OnInit { rerender: boolean = false; spinner: boolean = false; - projects: CheckboxOption[] = []; - researchLines: CheckboxOption[] = []; + projects: SelectItem[] = []; + researchLines: SelectItem[] = []; selectedProjects: string[] = []; selectedResearchLines: string[] = []; diff --git a/data-showcase/src/main/user-interface/src/app/models/CheckboxOption.ts b/data-showcase/src/main/user-interface/src/app/models/CheckboxOption.ts deleted file mode 100644 index 430f4a7..0000000 --- a/data-showcase/src/main/user-interface/src/app/models/CheckboxOption.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright (c) 2017 The Hyve B.V. - * This file is distributed under the GNU Affero General Public License - * (see accompanying file LICENSE). - */ - -export class CheckboxOption { - label: string; - value: string; -} diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 5dfda23..ae86de5 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -5,7 +5,7 @@ */ import {Injectable} from '@angular/core'; -import {TreeNode as TreeNodeLib} from 'primeng/primeng'; +import { SelectItem, TreeNode as TreeNodeLib } from 'primeng/primeng'; import {ResourceService} from './resource.service'; import {TreeNode} from "../models/tree-node"; import {Item} from "../models/item"; @@ -13,7 +13,6 @@ import {Project} from "../models/project"; import {Subject} from "rxjs/Subject"; import {BehaviorSubject} from "rxjs/BehaviorSubject"; import {Environment} from "../models/environment"; -import {CheckboxOption} from '../models/CheckboxOption'; import {ItemResponse} from "../models/itemResponse"; type LoadingState = 'loading' | 'complete'; @@ -65,9 +64,9 @@ export class DataService { // selected checkboxes for research lines filter private selectedResearchLines: string[] = []; // list of project names available for current item list - public projects: CheckboxOption[] = []; + public projects: SelectItem[] = []; // list of research lines available for current item list - public linesOfResearch: CheckboxOption[] = []; + public linesOfResearch: SelectItem[] = []; // list of all projects private allProjects: Project[] = []; @@ -380,12 +379,12 @@ export class DataService { } } - private static collectUnique(element, list: CheckboxOption[]) { + private static collectUnique(element, list: SelectItem[]) { let values = list.map(function (a) { return a.value; }); if (element && !values.includes(element)) { - list.push({label: element, value: element} as CheckboxOption); + list.push({label: element, value: element} as SelectItem); } } From 6b9eaa0b5dee49971badfc09e40973d77285f6e4 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Thu, 16 Nov 2017 12:13:21 +0100 Subject: [PATCH 065/104] Fix query type is null errors. --- .../nl/thehyve/datashowcase/ItemController.groovy | 7 +++++-- .../nl/thehyve/datashowcase/ProjectController.groovy | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index 530ee64..20b0aa4 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -59,8 +59,11 @@ class ItemController { String order = args.order String propertyName = args.propertyName log.info "Query input: ${args.searchQuery}" - def searchQuery = new SearchQueryRepresentation() - bindData(searchQuery, args.searchQuery) + def searchQuery = null + if (args.searchQuery) { + searchQuery = new SearchQueryRepresentation() + bindData(searchQuery, args.searchQuery) + } response.status = 200 response.contentType = 'application/json' diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy index 468d0c5..658af22 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ProjectController.groovy @@ -31,9 +31,12 @@ class ProjectController { def search() { def args = request.JSON as Map Set concepts = args.conceptCodes as Set - log.info "Query input: ${args.searchQuery}" - def searchQuery = new SearchQueryRepresentation() - bindData(searchQuery, args.searchQuery) + def searchQuery = null + if (args.searchQuery) { + log.info "Query input: ${args.searchQuery}" + searchQuery = new SearchQueryRepresentation() + bindData(searchQuery, args.searchQuery) + } try { respond projects: projectService.getProjects(concepts, searchQuery) From 4ba1b654cc11c6e1221422b8001bf6c981d982d8 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Thu, 16 Nov 2017 12:13:44 +0100 Subject: [PATCH 066/104] Sort projects and lines of research. --- .../datashowcase/ProjectService.groovy | 64 +++++++++---------- .../src/app/services/data.service.ts | 17 +++++ 2 files changed, 46 insertions(+), 35 deletions(-) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy index 5941aef..28ab3cc 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy @@ -17,6 +17,7 @@ import org.hibernate.HibernateException import org.hibernate.SessionFactory import org.hibernate.Transaction import org.hibernate.criterion.Criterion +import org.hibernate.criterion.Order import org.hibernate.criterion.Projections import org.hibernate.criterion.Restrictions import org.modelmapper.ModelMapper @@ -45,7 +46,7 @@ class ProjectService { } List getProjects() { - Project.findAll().collect({ + Project.findAll([sort: 'name', order: 'asc']).collect({ modelMapper.map(it, ProjectRepresentation) }) } @@ -55,42 +56,35 @@ class ProjectService { Criterion searchQueryCriterion = searchQuery ? searchCriteriaBuilder.buildCriteria(searchQuery) : null def stopWatch = new StopWatch('Fetch projects filtered items') stopWatch.start('Retrieve from database') - // stateless session does not support collections of associated objects - // http://forum.spring.io/forum/spring-projects/batch/37785-collections-cannot-be-fetched-by-a-stateless-session - def session = sessionFactory.openSession() - try { - Criteria criteria = session.createCriteria(Project, "p") - .createAlias("p.items", "i") - .createAlias("i.concept", "c") - .createAlias("c.keywords", "k") - .setProjection(Projections.projectionList() - .add(Projections.distinct(Projections.property("p.name").as("name"))) - .add(Projections.property("p.lineOfResearch").as("lineOfResearch"))) - if (concepts) { - criteria.add(Restrictions.in('c.conceptCode', concepts)) - } - if (dataShowcaseEnvironment.internalInstance) { - criteria.add(Restrictions.eq('i.publicItem', true)) - } - if (searchQueryCriterion) { - criteria.add(searchQueryCriterion) - } - criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) - def projects = criteria.list() as List - stopWatch.stop() - stopWatch.start('Map to representations') - def result = projects.collect { Map projectData -> - map(projectData) - } - stopWatch.stop() - log.info "Projects fetched.\n${stopWatch.prettyPrint()}" - result - } catch (HibernateException e) { - e.printStackTrace() - } finally { - session.close() + Criteria criteria = sessionFactory.currentSession.createCriteria(Project, "p") + .createAlias("p.items", "i") + .createAlias("i.concept", "c") + .createAlias("c.keywords", "k") + .setProjection(Projections.projectionList() + .add(Projections.distinct(Projections.property("p.name").as("name"))) + .add(Projections.property("p.lineOfResearch").as("lineOfResearch"))) + if (concepts) { + criteria.add(Restrictions.in('c.conceptCode', concepts)) } + if (dataShowcaseEnvironment.internalInstance) { + criteria.add(Restrictions.eq('i.publicItem', true)) + } + if (searchQueryCriterion) { + criteria.add(searchQueryCriterion) + } + criteria.addOrder(Order.asc('p.name')) + criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) + def projects = criteria.list() as List + stopWatch.stop() + + stopWatch.start('Map to representations') + def result = projects.collect { Map projectData -> + map(projectData) + } + stopWatch.stop() + log.info "Projects fetched.\n${stopWatch.prettyPrint()}" + result } } diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index ae86de5..6d3018b 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -184,6 +184,7 @@ export class DataService { this.projects.push({label: project.name, value: project.name}); DataService.collectUnique(project.lineOfResearch, this.linesOfResearch); } + this.sortLinesOfResearch(); }, err => console.error(err) ); @@ -211,6 +212,7 @@ export class DataService { this.projects.push({label: project.name, value: project.name}); DataService.collectUnique(project.lineOfResearch, this.linesOfResearch); } + this.sortLinesOfResearch(); }, err => { console.error(err); @@ -362,6 +364,20 @@ export class DataService { }); } + private static compareSelectItems(a: SelectItem, b: SelectItem) { + if (a.value < b.value) { + return -1; + } else if (a.value > b.value) { + return 1; + } else { + return 0; + } + } + + private sortLinesOfResearch() { + this.linesOfResearch.sort(DataService.compareSelectItems); + } + private getUniqueLinesOfResearch() { if (!this.selectedProjects.length) { this.allProjects.forEach(p => { @@ -377,6 +393,7 @@ export class DataService { } }); } + this.sortLinesOfResearch(); } private static collectUnique(element, list: SelectItem[]) { From ab2814c3a005f4713116de6e6b7f3797778f40f1 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 17 Nov 2017 23:54:57 +0100 Subject: [PATCH 067/104] Add Date variable type. --- .../nl/thehyve/datashowcase/enumeration/VariableType.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/VariableType.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/VariableType.groovy index 1038975..e7edd32 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/VariableType.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/VariableType.groovy @@ -13,5 +13,6 @@ enum VariableType { Numerical, Categorical, Text, + Date, None } From 242fcdc752e8ea1875c4ba75408be314af1d7414 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 17 Nov 2017 23:55:09 +0100 Subject: [PATCH 068/104] Filter out empty research lines. --- .../services/nl/thehyve/datashowcase/DataImportService.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy index c275558..2a81721 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/DataImportService.groovy @@ -83,8 +83,8 @@ class DataImportService { stopWatch.stop() statelessSession.managedFlush() - //save research_lines - def linesOfResearch = json.projects?.lineOfResearch.unique().collect { + // save research_lines + def linesOfResearch = json.projects?.lineOfResearch.unique().findAll {it != null}.collect { if (it) new LineOfResearch(name: it) } as List validate(linesOfResearch) From c0342cd2ce0650148a865d56165869e43f2f4aad Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 17 Nov 2017 23:55:34 +0100 Subject: [PATCH 069/104] Make data stability field nullable. --- .../grails-app/domain/nl/thehyve/datashowcase/Summary.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy index 5099f82..aff4382 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy @@ -71,7 +71,7 @@ class Summary { static constraints = { observationCount nullable: false patientCount nullable: false - dataStability nullable: false + dataStability nullable: true minValue nullable: true maxValue nullable: true avgValue nullable: true From 8cd5181052f85809fabecb09530f70671e71e769 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 20 Nov 2017 18:05:13 +0100 Subject: [PATCH 070/104] Fix fetching of items without keywords; fix item count. --- .../services/nl/thehyve/datashowcase/ItemService.groovy | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index 6cfbd57..2cf7636 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -25,6 +25,7 @@ import org.hibernate.criterion.Criterion import org.hibernate.criterion.Order import org.hibernate.criterion.Projections import org.hibernate.criterion.Restrictions +import org.hibernate.sql.JoinType import org.hibernate.transform.Transformers import org.modelmapper.ModelMapper import org.springframework.beans.factory.annotation.Autowired @@ -125,7 +126,7 @@ class ItemService { Criteria criteria = session.createCriteria(Item, "i") .createAlias("i.concept", "c") .createAlias("i.project", "p") - .createAlias("c.keywords", "k") + .createAlias("c.keywords", "k", JoinType.LEFT_OUTER_JOIN) .setProjection(Projections.projectionList() .add(Projections.distinct(Projections.property("i.id").as("id"))) .add(Projections.property("i.name").as("name")) @@ -180,7 +181,7 @@ class ItemService { Criteria criteria = session.createCriteria(Item, "i") .createAlias("i.concept", "c") .createAlias("i.project", "p") - .createAlias("c.keywords", "k") + .createAlias("c.keywords", "k", JoinType.LEFT_OUTER_JOIN) if(concepts) { criteria.add( Restrictions.in('c.conceptCode', concepts)) } @@ -194,7 +195,7 @@ class ItemService { criteria.add(searchQueryCriterion) } criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) - .setProjection(Projections.rowCount()) + .setProjection(Projections.countDistinct('i.id')) Long totalItemsCount = (Long)criteria.uniqueResult() stopWatch.stop() From b54c9dcc640986b7e4e9a6bbafbddf8930d831b4 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 20 Nov 2017 22:46:54 +0100 Subject: [PATCH 071/104] Update primeng and node versions --- data-showcase/src/main/user-interface/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-showcase/src/main/user-interface/package.json b/data-showcase/src/main/user-interface/package.json index 8bb32a1..859aa84 100644 --- a/data-showcase/src/main/user-interface/package.json +++ b/data-showcase/src/main/user-interface/package.json @@ -29,7 +29,7 @@ "file-saver": "^1.3.3", "font-awesome": "^4.7.0", "logic-query-parser": "0.0.5", - "primeng": "4.2.2", + "primeng": "^4.3.0", "roboto-fontface": "^0.8.0", "rxjs": "^5.5.2", "zone.js": "^0.8.4" @@ -40,7 +40,7 @@ "@angular/language-service": "^4.0.0", "@types/jasmine": "~2.5.53", "@types/jasminewd2": "~2.0.2", - "@types/node": "~6.0.60", + "@types/node": "^6.0.92", "codelyzer": "~3.0.1", "cucumber": "^3.1.0", "es6-promise": "^4.1.1", From 97b8aa21027338b4d113b6a99e8797053b610b96 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 20 Nov 2017 22:53:03 +0100 Subject: [PATCH 072/104] Add select-all checkbox logic for a custom pagination --- .../app/item-table/item-table.component.html | 17 +++-- .../app/item-table/item-table.component.ts | 70 ++++++++++++++++--- .../src/app/item-table/item-table.module.ts | 8 ++- .../src/app/services/data.service.ts | 36 ++++++++-- 4 files changed, 109 insertions(+), 22 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html index 9f5fc58..aa5b5b1 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html @@ -9,12 +9,17 @@ [immutable]=false [value]="items | itemFilter: filterValue" [rows]="rowsPerPage" [paginator]="false" - [headerCheckboxToggleAllPages]="true" [resizableColumns]="true" - [(selection)]="itemsSelection" + [totalRecords] = "totalItemsCount()" + [(selection)]="itemsSelectionPerPage" [showHeaderCheckbox]=false + (onRowSelect)="handleRowSelect($event)" (onRowUnselect)="handleRowSelect($event)" (onSort)="changeSort($event)" [loading]="dataService.loadingItems == 'loading'" loadingIcon="fa fa-spin fa-refresh fa-fw"> + + + + @@ -43,15 +48,15 @@ - +
- Items selected: {{itemsSelection? itemsSelection.length : 0}}. - Total results in table: {{countItems()}}. - Number of pages: {{pageCount()}}. + Items selected: {{itemsSelection ? itemsSelection.length : 0}}. + Total results in table: {{totalItemsCount()}}. + Number of pages: {{pagesCount()}}.
diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts index c9b555a..6add9b1 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts @@ -16,7 +16,7 @@ import { }) export class ItemFilter implements PipeTransform { transform(value: Item[], filter: string): Item[] { - filter = filter ? filter.toLocaleLowerCase(): null; + filter = filter ? filter.toLocaleLowerCase() : null; return filter && value ? value.filter(item => (item.name.toLocaleLowerCase().indexOf(filter) !== -1) || @@ -47,16 +47,24 @@ export class ItemTableComponent implements OnInit { filterValue: string; items: Item[]; itemsSelection: Item[]; + itemsSelectionPerPage: Item[]; rowsPerPage: number; + allSelected: boolean; constructor(public dataService: DataService) { - this.rowsPerPage = 8; - this.items = this.dataService.filteredItems; this.dataService.itemsSelection$.subscribe( selection => { this.itemsSelection = selection; } ); + this.dataService.filteredItems$.subscribe( + items => { + this.items = items; + if(items.length > 0) { + this.updateCurrentPageItemsSelection(items); + } + } + ); this.dataService.textFilterInput$.subscribe( filter => { this.filterValue = filter; @@ -65,21 +73,25 @@ export class ItemTableComponent implements OnInit { } ngOnInit() { + this.rowsPerPage = 8; + this.allSelected = false; + this.itemsSelection = []; + this.itemsSelectionPerPage = []; } - addToCart(){ + addToCart() { this.dataService.addToShoppingCart(this.itemsSelection); } - showSummary(item: Item){ + showSummary(item: Item) { this.dataService.displayPopup(item); } - pageCount(): number { - return Math.ceil(this.countItems() / this.rowsPerPage) + pagesCount(): number { + return Math.ceil(this.totalItemsCount() / this.rowsPerPage); } - countItems(): number { + totalItemsCount(): number { return this.dataService.totalItemsCount; } @@ -91,9 +103,49 @@ export class ItemTableComponent implements OnInit { } paginate(event) { - console.log("PAGE: " + event.page); + console.log("On page: " + (event.page + 1)); this.dataService.itemsFirstResult = event.page * event.rows; this.dataService.itemsMaxResults = event.rows; this.dataService.fetchItems(); } + + handleHeaderCheckboxToggle(event) { + if (event) { + this.dataService.selectAllItems(true); + this.itemsSelectionPerPage = this.items; + this.allSelected = true; + console.log("All items selected"); + } else { + this.dataService.selectAllItems(false); + this.itemsSelectionPerPage = []; + this.allSelected = false; + console.log("All items deselected"); + } + } + + handleRowSelect($event){ + let item = this.itemsSelection ? this.itemsSelection.filter(items => (items.id === $event.data.id)) : []; + if($event.originalEvent.checked && item.length == 0){ + this.itemsSelection.push($event.data); + console.log("Item '" + $event.data.name +"' added to selection."); + if(this.itemsSelection.length == this.totalItemsCount()){ + this.allSelected = true; + } + } else if(!$event.originalEvent.checked && item.length > 0) { + this.itemsSelection.splice(this.itemsSelection.indexOf(item[0]),1); + console.log("\"Item '" + $event.data.name + "' removed from selection."); + if(this.allSelected) { + this.allSelected = false; + } + } + } + + updateCurrentPageItemsSelection(items: Item[]){ + if(this.allSelected) { + this.itemsSelectionPerPage = items; + } else { + this.itemsSelectionPerPage = items.filter(i => this.itemsSelection.some(is => is.id == i.id)); + } + } + } diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.module.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.module.ts index 2e444b6..d1df642 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.module.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.module.ts @@ -6,7 +6,10 @@ import {NgModule} from '@angular/core'; import {CommonModule} from '@angular/common'; -import {ButtonModule, DataTableModule, ListboxModule, PaginatorModule, PanelModule} from "primeng/primeng"; +import { + ButtonModule, CheckboxModule, DataTableModule, ListboxModule, PaginatorModule, + PanelModule +} from "primeng/primeng"; import {FormsModule} from "@angular/forms"; import {ItemFilter, ItemTableComponent} from "./item-table.component"; @@ -18,7 +21,8 @@ import {ItemFilter, ItemTableComponent} from "./item-table.component"; DataTableModule, ListboxModule, ButtonModule, - PaginatorModule + PaginatorModule, + CheckboxModule ], declarations: [ItemTableComponent, ItemFilter], exports: [ItemTableComponent, ItemFilter] diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 6d3018b..c201038 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -29,7 +29,8 @@ export class DataService { // the flag indicating if Items are still being loaded public loadingItems: LoadingState = 'complete'; // filtered list of items based on selected node and selected checkbox filters - public filteredItems: Item[] = []; + private filteredItemsSource = new Subject(); + public filteredItems$ = this.filteredItemsSource.asObservable(); // items selected in the itemTable private itemsSelectionSource = new Subject(); public itemsSelection$ = this.itemsSelectionSource.asObservable(); @@ -224,7 +225,7 @@ export class DataService { let t1 = new Date(); console.debug(`Fetching items ...`); this.loadingItems = 'loading'; - this.filteredItems.length = 0; + this.filteredItemsSource.next([]); this.clearErrorSearchMessage(); let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); @@ -241,11 +242,11 @@ export class DataService { if (this.allProjects && this.allProjects.length > 0) { item.lineOfResearch = this.projectToResearchLine(item.project); } - this.filteredItems.push(item); } + this.filteredItemsSource.next(response.items); this.loadingItems = "complete"; let t2 = new Date(); - console.info(`Found ${this.filteredItems.length} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); + console.info(`Found ${response.totalCount} items. (Took ${t2.getTime() - t1.getTime()} ms.)`); }, err => { if (err != String(undefined)) { @@ -265,6 +266,31 @@ export class DataService { this.searchErrorMessageSource.next(''); } + selectAllItems(selectAll: boolean){ + if(selectAll){ + let firstResult = 0; + let maxResult = 9999999; + let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); + let codes = Array.from(selectedConceptCodes); + let projects = this.getProjectsForSelectedResearchLines(); + + this.resourceService.getItems(firstResult, maxResult, null, null, + codes, projects, this.searchQuery).subscribe( + (response: ItemResponse) => { + this.itemsSelectionSource.next(response.items); + }, + err => { + if (err != String(undefined)) { + this.searchErrorMessageSource.next(err); + } + console.error(err); + } + ); + } else { + this.clearItemsSelection(); + } + } + static treeConceptCodes(treeNode: TreeNode): Set { if (treeNode == null) { return new Set(); @@ -337,7 +363,7 @@ export class DataService { } clearItemsSelection() { - this.itemsSelectionSource.next(null); + this.itemsSelectionSource.next([]); } setTextFilterInput(text: string) { From 1912b65b19102958e3679a2e26292cd93cc48527 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Tue, 21 Nov 2017 08:41:44 +0100 Subject: [PATCH 073/104] Fix fetching of projects for items without keywords --- .../services/nl/thehyve/datashowcase/ProjectService.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy index 28ab3cc..040f370 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ProjectService.groovy @@ -20,6 +20,7 @@ import org.hibernate.criterion.Criterion import org.hibernate.criterion.Order import org.hibernate.criterion.Projections import org.hibernate.criterion.Restrictions +import org.hibernate.sql.JoinType import org.modelmapper.ModelMapper import org.springframework.beans.factory.annotation.Autowired @@ -60,7 +61,7 @@ class ProjectService { Criteria criteria = sessionFactory.currentSession.createCriteria(Project, "p") .createAlias("p.items", "i") .createAlias("i.concept", "c") - .createAlias("c.keywords", "k") + .createAlias("c.keywords", "k", JoinType.LEFT_OUTER_JOIN) .setProjection(Projections.projectionList() .add(Projections.distinct(Projections.property("p.name").as("name"))) .add(Projections.property("p.lineOfResearch").as("lineOfResearch"))) From 2d972a082de9749f3da45d9a69fcc52dcb867707 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Tue, 21 Nov 2017 11:01:08 +0100 Subject: [PATCH 074/104] Make maxResults an optional parameter (defaults to all items). --- .../datashowcase/ItemController.groovy | 19 +++---- .../thehyve/datashowcase/ItemService.groovy | 50 +++++++++++++------ .../src/app/services/data.service.ts | 3 +- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy index 20b0aa4..28fed6d 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/ItemController.groovy @@ -23,18 +23,19 @@ class ItemController { */ def index() { try { - def args = request.JSON as Map - int firstResult = args.firstResult - int maxResults = args.maxResults + int firstResult = args.firstResult ?: 0 + def maxResults = args.maxResults as Integer String order = args.order String propertyName = args.propertyName response.status = 200 response.contentType = 'application/json' response.characterEncoding = 'utf-8' - def data = [items: itemService.getItems(firstResult, maxResults, order, propertyName)] - log.info "Writing ${data.items.size()} items ..." + def items = itemService.getItems(firstResult, maxResults, order, propertyName) + def count = itemService.getItemsCount() + int page = maxResults ? (firstResult/maxResults + 1) : 1 + def data = [totalCount: count, page: page, items: items] new ObjectMapper().writeValue(response.outputStream, data) } catch (Exception e) { response.status = 400 @@ -54,8 +55,8 @@ class ItemController { def args = request.JSON as Map Set concepts = args.conceptCodes as Set Set projects = args.projects as Set - int firstResult = args.firstResult - int maxResults = args.maxResults + int firstResult = args.firstResult ?: 0 + def maxResults = args.maxResults as Integer String order = args.order String propertyName = args.propertyName log.info "Query input: ${args.searchQuery}" @@ -72,8 +73,8 @@ class ItemController { def items = itemService.getItems( firstResult, maxResults, order, propertyName, concepts, projects, searchQuery) def count = itemService.getItemsCount(concepts, projects, searchQuery) - int page = (firstResult/maxResults + 1) - def data = [ totalCount: count, page: page, items: items] + int page = maxResults ? (firstResult/maxResults + 1) : 1 + def data = [totalCount: count, page: page, items: items] new ObjectMapper().writeValue(response.outputStream, data) } catch (Exception e) { diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy index 2cf7636..cab96e6 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/ItemService.groovy @@ -71,13 +71,13 @@ class ItemService { @Cacheable('items') @Transactional(readOnly = true) - List getItems(int firstResult, int maxResults, String order, String propertyName) { + List getItems(int firstResult, Integer maxResults, String order, String propertyName) { def property = propertyNameFromRepresentationName(propertyName) def stopWatch = new StopWatch('Fetch items') stopWatch.start('Retrieve from database') def session = sessionFactory.openStatelessSession() - def items = session.createQuery( + def query = session.createQuery( """ select i.id as id, @@ -99,10 +99,15 @@ class ItemService { 'desc' : 'asc' } """ - ).setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) - .setFirstResult(firstResult) - .setMaxResults(maxResults) - .list() as List + ) + query.setResultTransformer(Transformers.ALIAS_TO_ENTITY_MAP) + if (firstResult > 0) { + query.setFirstResult(firstResult) + } + if (maxResults) { + query.setMaxResults(maxResults) + } + def items = query.list() as List stopWatch.stop() stopWatch.start('Map to representations') def result = items.collect { Map itemData -> @@ -114,7 +119,15 @@ class ItemService { } @Transactional(readOnly = true) - List getItems(int firstResult, int maxResults, String order, String propertyName, + Long getItemsCount() { + def session = sessionFactory.currentSession + session.createCriteria(Item, "i") + .setProjection(Projections.rowCount()) + .uniqueResult() as Long + } + + @Transactional(readOnly = true) + List getItems(int firstResult, Integer maxResults, String order, String propertyName, Set concepts, Set projects, SearchQueryRepresentation searchQuery) { def property = propertyNameFromRepresentationName(propertyName) @@ -137,21 +150,26 @@ class ItemService { .add(Projections.property("c.labelLong").as("labelLong")) .add(Projections.property("c.variableType").as("variableType")) .add(Projections.property("p.name").as("projectName"))) - if(concepts) { - criteria.add( Restrictions.in('c.conceptCode', concepts)) + if (concepts) { + criteria.add(Restrictions.in('c.conceptCode', concepts)) } - if(projects) { - criteria.add( Restrictions.in('p.name', projects)) + if (projects) { + criteria.add(Restrictions.in('p.name', projects)) } - if(!dataShowcaseEnvironment.internalInstance) { - criteria.add( Restrictions.eq('i.publicItem',true)) + if (!dataShowcaseEnvironment.internalInstance) { + criteria.add(Restrictions.eq('i.publicItem', true)) } - if(searchQueryCriterion) { + if (searchQueryCriterion) { criteria.add(searchQueryCriterion) } criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) - .setFirstResult(firstResult) - .setMaxResults(maxResults) + if (firstResult > 0) { + criteria.setFirstResult(firstResult) + } + if (maxResults) { + criteria.setMaxResults(maxResults) + } + if (order == "desc") { criteria.addOrder(Order.desc(property)) } else { diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index c201038..c5ec44c 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -269,12 +269,11 @@ export class DataService { selectAllItems(selectAll: boolean){ if(selectAll){ let firstResult = 0; - let maxResult = 9999999; let selectedConceptCodes = DataService.treeConceptCodes(this.selectedTreeNode); let codes = Array.from(selectedConceptCodes); let projects = this.getProjectsForSelectedResearchLines(); - this.resourceService.getItems(firstResult, maxResult, null, null, + this.resourceService.getItems(firstResult, null, null, null, codes, projects, this.searchQuery).subscribe( (response: ItemResponse) => { this.itemsSelectionSource.next(response.items); From 012d87e649c1eaabb8e9e3cf9dfe1c38d49221bf Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Tue, 21 Nov 2017 11:41:47 +0100 Subject: [PATCH 075/104] Fix clearing items selection on a filter change --- .../app/item-table/item-table.component.html | 2 +- .../src/app/item-table/item-table.component.ts | 17 +++++++++-------- .../src/app/services/data.service.ts | 5 +++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html index aa5b5b1..366f0a9 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html @@ -17,7 +17,7 @@ [loading]="dataService.loadingItems == 'loading'" loadingIcon="fa fa-spin fa-refresh fa-fw"> - + diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts index 6add9b1..ee94fff 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts @@ -49,12 +49,14 @@ export class ItemTableComponent implements OnInit { itemsSelection: Item[]; itemsSelectionPerPage: Item[]; rowsPerPage: number; - allSelected: boolean; constructor(public dataService: DataService) { this.dataService.itemsSelection$.subscribe( selection => { this.itemsSelection = selection; + if(selection.length === 0){ + this.itemsSelectionPerPage = []; + } } ); this.dataService.filteredItems$.subscribe( @@ -74,7 +76,6 @@ export class ItemTableComponent implements OnInit { ngOnInit() { this.rowsPerPage = 8; - this.allSelected = false; this.itemsSelection = []; this.itemsSelectionPerPage = []; } @@ -113,12 +114,12 @@ export class ItemTableComponent implements OnInit { if (event) { this.dataService.selectAllItems(true); this.itemsSelectionPerPage = this.items; - this.allSelected = true; + this.dataService.allItemsSelected = true; console.log("All items selected"); } else { this.dataService.selectAllItems(false); this.itemsSelectionPerPage = []; - this.allSelected = false; + this.dataService.allItemsSelected = false; console.log("All items deselected"); } } @@ -129,19 +130,19 @@ export class ItemTableComponent implements OnInit { this.itemsSelection.push($event.data); console.log("Item '" + $event.data.name +"' added to selection."); if(this.itemsSelection.length == this.totalItemsCount()){ - this.allSelected = true; + this.dataService.allItemsSelected = true; } } else if(!$event.originalEvent.checked && item.length > 0) { this.itemsSelection.splice(this.itemsSelection.indexOf(item[0]),1); console.log("\"Item '" + $event.data.name + "' removed from selection."); - if(this.allSelected) { - this.allSelected = false; + if(this.dataService.allItemsSelected) { + this.dataService.allItemsSelected = false; } } } updateCurrentPageItemsSelection(items: Item[]){ - if(this.allSelected) { + if(this.dataService.allItemsSelected) { this.itemsSelectionPerPage = items; } else { this.itemsSelectionPerPage = items.filter(i => this.itemsSelection.some(is => is.id == i.id)); diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index c5ec44c..0a23168 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -37,6 +37,8 @@ export class DataService { // items added to the shopping cart public shoppingCartItems = new BehaviorSubject([]); public totalItemsCount: number = 0; + // if the select-all-items checkbox is selected + public allItemsSelected: boolean = false; // text filter input private textFilterInputSource = new Subject(); @@ -346,6 +348,7 @@ export class DataService { clearAllFilters() { this.clearCheckboxFilterSelection(); this.setTextFilterInput(''); + this.clearItemsSelection(); this.rerenderCheckboxFiltersSource.next(true); } @@ -362,6 +365,7 @@ export class DataService { } clearItemsSelection() { + this.allItemsSelected = false; this.itemsSelectionSource.next([]); } @@ -371,6 +375,7 @@ export class DataService { setSearchQuery(query: Object) { this.searchQuery = query; + this.clearItemsSelection(); this.fetchItems(); this.fetchFilters(); } From 3e9cbc350381b18b3971bdc227574f14dc00b5f2 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Tue, 21 Nov 2017 11:46:14 +0100 Subject: [PATCH 076/104] Enable uploading empty strings as values. --- data-showcase/grails-app/conf/application.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data-showcase/grails-app/conf/application.yml b/data-showcase/grails-app/conf/application.yml index be2a6f6..248b6f0 100644 --- a/data-showcase/grails-app/conf/application.yml +++ b/data-showcase/grails-app/conf/application.yml @@ -27,6 +27,8 @@ grails: mapping: version: false autoTimestamp: false + databinding: + convertEmptyStringsToNull: false info: app: From ec690ef309a866e7ad263ec218dcf9f411e3f68d Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 20 Nov 2017 16:34:07 +0100 Subject: [PATCH 077/104] Reimplemented the front end query parser with nearley. --- .../search/SearchCriteriaBuilder.groovy | 6 +- .../src/main/user-interface/package.json | 13 +- .../text-filter/text-filter.component.ts | 1 - .../src/app/models/binary-tree.ts | 32 --- .../src/app/models/search-query.ts | 28 --- .../src/app/search-text-parser/grammar.ne | 84 +++++++ .../src/app/search-text-parser/grammar.ts | 84 +++++++ .../src/app/search-text-parser/index.spec.ts | 230 ++++++++++++++++++ .../src/app/search-text-parser/index.ts | 87 +++++++ .../app/search-text-parser/search-query.ts | 46 ++++ .../src/app/search-text-parser/syntax.ts | 36 +++ .../src/app/services/search-parser.service.ts | 99 +------- .../src/main/user-interface/tsconfig.json | 5 +- 13 files changed, 586 insertions(+), 165 deletions(-) delete mode 100644 data-showcase/src/main/user-interface/src/app/models/binary-tree.ts delete mode 100644 data-showcase/src/main/user-interface/src/app/models/search-query.ts create mode 100644 data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne create mode 100644 data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts create mode 100644 data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts create mode 100644 data-showcase/src/main/user-interface/src/app/search-text-parser/index.ts create mode 100644 data-showcase/src/main/user-interface/src/app/search-text-parser/search-query.ts create mode 100644 data-showcase/src/main/user-interface/src/app/search-text-parser/syntax.ts diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy index 660a58e..ccee914 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy @@ -189,7 +189,7 @@ class SearchCriteriaBuilder { def propertyName = query.value def property = SearchField.NONE def args = query.values.collect { it.value } - if (propertyName) { + if (propertyName && propertyName != '*') { property = SearchField.forName(propertyName) if (property == SearchField.NONE) { throw new IllegalArgumentException("Unsupported property: ${propertyName}.") @@ -200,8 +200,8 @@ class SearchCriteriaBuilder { // applying an operator to a field with a list of values return buildSingleCriteria(operator, property, args) } else { - log.info "Applying unary ${operator} on args: ${args}" - // applying a unary operator to all fields with a list of values + log.info "Applying ${operator} on args: ${args}" + // applying an operator to all fields with a list of values return applyToAllSearchFields(operator, args) } } else { diff --git a/data-showcase/src/main/user-interface/package.json b/data-showcase/src/main/user-interface/package.json index 859aa84..2996173 100644 --- a/data-showcase/src/main/user-interface/package.json +++ b/data-showcase/src/main/user-interface/package.json @@ -3,11 +3,14 @@ "version": "0.0.1", "license": "MIT", "scripts": { + "grammar": "./node_modules/nearley/bin/nearleyc.js ./src/app/search-text-parser/grammar.ne -o ./src/app/search-text-parser/grammar.ts", + "pretest": "npm run grammar", + "prestart": "npm run grammar", "ng": "ng", "start": "ng serve", - "buildProd": "ng build --prod --watch=false", - "buildDev": "ng build --dev --watch=false", - "build": "ng build --watch=true", + "buildProd": "npm run grammar && ng build --prod --watch=false", + "buildDev": "npm run grammar && ng build --dev --watch=false", + "build": "npm run grammar && ng build --watch=true", "test": "ng test --single-run", "lint": "ng lint", "e2e": "ng e2e" @@ -24,12 +27,14 @@ "@angular/platform-browser-dynamic": "^4.0.0", "@angular/router": "^4.0.0", "@types/file-saver": "0.0.1", + "@types/nearley": "^2.11.0", "angular-tree-component": "^3.8.0", "core-js": "^2.4.1", "file-saver": "^1.3.3", "font-awesome": "^4.7.0", - "logic-query-parser": "0.0.5", "primeng": "^4.3.0", + "moo": "^0.4.3", + "nearley": "^2.11.0", "roboto-fontface": "^0.8.0", "rxjs": "^5.5.2", "zone.js": "^0.8.4" diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts index bafe736..5171245 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.ts @@ -25,7 +25,6 @@ export class TextFilterComponent implements OnInit { delay: number; constructor(public dataService: DataService, - public searchParserService: SearchParserService, private element: ElementRef) { this.dataService.searchErrorMessage$.subscribe( message => { diff --git a/data-showcase/src/main/user-interface/src/app/models/binary-tree.ts b/data-showcase/src/main/user-interface/src/app/models/binary-tree.ts deleted file mode 100644 index d1eda71..0000000 --- a/data-showcase/src/main/user-interface/src/app/models/binary-tree.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2017 The Hyve B.V. - * This file is distributed under the GNU Affero General Public License - * (see accompanying file LICENSE). - */ - -export type LexemeType = 'and' | 'or' | 'string' | 'not'; - -export class Lexeme { - type: LexemeType; - value: string; - - static forType(type: LexemeType): Lexeme { - let result = new Lexeme(); - result.type = type; - return result; - } -} - -export class BinaryTree { - lexeme: Lexeme; - left: BinaryTree; - right: BinaryTree; - - static forBranches(type: LexemeType, left: BinaryTree, right: BinaryTree) : BinaryTree { - let result = new BinaryTree(); - result.lexeme = Lexeme.forType(type); - result.left = left; - result.right = right; - return result; - } -} diff --git a/data-showcase/src/main/user-interface/src/app/models/search-query.ts b/data-showcase/src/main/user-interface/src/app/models/search-query.ts deleted file mode 100644 index 08dfe41..0000000 --- a/data-showcase/src/main/user-interface/src/app/models/search-query.ts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2017 The Hyve B.V. - * This file is distributed under the GNU Affero General Public License - * (see accompanying file LICENSE). - */ - -export type QueryType = 'and' | 'or' | 'string' | 'not' | '=' | '!=' | 'contains' | 'like' | 'in'; - -export class SearchQuery { - type: QueryType; - value: string; - values: SearchQuery[]; - - static forValues(type: QueryType, values: SearchQuery[]) : SearchQuery { - let result = new SearchQuery(); - result.type = type; - result.values = values; - return result; - } - - static forValue(value: string) : SearchQuery { - let result = new SearchQuery(); - result.type = 'string'; - result.value = value; - return result; - } - -} diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne new file mode 100644 index 0000000..d470ce7 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne @@ -0,0 +1,84 @@ +@preprocessor typescript +@{% +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +import { syntax } from './syntax'; + +const defaultOperator = 'and'; + +function stripQuotes(word) { + return word.value.replace(/^"|"$/g, ''); +} +function buildValue(value) { + return { type: 'string', value: value }; +} + +function buildSequence(d) { + if (d[2] == null) { + return d[0]; + } else { + return { type: defaultOperator, left: d[0], right: d[2] }; + } +} + +function buildToken(d) { + return d[0].value; +} + +function buildLowercaseToken(d) { + return d[0].value.toLowerCase(); +} + +%} + +# Pass your lexer object using the @lexer option: +@lexer syntax + +search -> _ expression _ {% function(d) { return d[1]; } %} + +word -> %word {% function(d) { return buildValue(stripQuotes(d[0])); } %} + +field -> + "name" {% buildToken %} + | "label" {% buildToken %} + | "labelLong" {% buildToken %} + | "labelNl" {% buildToken %} + | "labelNlLong" {% buildToken %} + | "keywords" {% buildToken %} + +comparator -> + "=" {% buildLowercaseToken %} + | "!=" {% buildLowercaseToken %} + | "contains" {% buildLowercaseToken %} + | "CONTAINS" {% buildLowercaseToken %} + | "like" {% buildLowercaseToken %} + | "LIKE" {% buildLowercaseToken %} + +comparison -> + field _ comparator _ word {% + function(d) { return { type: 'comparison', field: d[0], comparator: d[2], arg: d[4] }; } + %} + | comparator _ word {% + function(d) { return { type: 'comparison', field: '*', comparator: d[0], arg: d[2] }; } + %} + +sequence -> + null {% function(d) { return null; } %} + | word _ sequence {% buildSequence %} + | comparison _ sequence {% buildSequence %} + +inner -> + "(" _ expression _ ")" {% function(d) { return d[2]; } %} + | ("not"|"NOT") _ inner {% function(d) { return { type: 'not', arg: d[2] }; } %} + | sequence {% function(d) { return d[0]; } %} + +expression -> + inner _ ("and"|"AND") _ expression {% function(d) { return { type: 'and', left: d[0], right: d[4] }; } %} + | inner _ ("or"|"OR") _ expression {% function(d) { return { type: 'or', left: d[0], right: d[4] }; } %} + | inner {% function(d) { return d[0]; } %} + +_ -> null | %WS {% function(d) { return null; } %} diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts new file mode 100644 index 0000000..dec0ccb --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts @@ -0,0 +1,84 @@ +// Generated automatically by nearley +// http://github.com/Hardmath123/nearley +function id(d:any[]):any {return d[0];} +declare var word:any; +declare var WS:any; + +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +import { syntax } from './syntax'; + +const defaultOperator = 'and'; + +function stripQuotes(word) { + return word.value.replace(/^"|"$/g, ''); +} +function buildValue(value) { + return { type: 'string', value: value }; +} + +function buildSequence(d) { + if (d[2] == null) { + return d[0]; + } else { + return { type: defaultOperator, left: d[0], right: d[2] }; + } +} + +function buildToken(d) { + return d[0].value; +} + +function buildLowercaseToken(d) { + return d[0].value.toLowerCase(); +} + +export interface Token {value:any; [key: string]:any}; +export interface Lexer {reset:(chunk:string, info:any) => void; next:() => Token | undefined; save:() => any; formatError:(token:Token) => string; has:(tokenType:string) => boolean}; +export interface NearleyRule {name:string; symbols:NearleySymbol[]; postprocess?:(d:any[],loc?:number,reject?:{})=>any}; +export type NearleySymbol = string | {literal:any} | {test:(token:any) => boolean}; +export var Lexer:Lexer|undefined = syntax; +export var ParserRules:NearleyRule[] = [ + {"name": "search", "symbols": ["_", "expression", "_"], "postprocess": function(d) { return d[1]; }}, + {"name": "word", "symbols": [(syntax.has("word") ? {type: "word"} : word)], "postprocess": function(d) { return buildValue(stripQuotes(d[0])); }}, + {"name": "field", "symbols": [{"literal":"name"}], "postprocess": buildToken}, + {"name": "field", "symbols": [{"literal":"label"}], "postprocess": buildToken}, + {"name": "field", "symbols": [{"literal":"labelLong"}], "postprocess": buildToken}, + {"name": "field", "symbols": [{"literal":"labelNl"}], "postprocess": buildToken}, + {"name": "field", "symbols": [{"literal":"labelNlLong"}], "postprocess": buildToken}, + {"name": "field", "symbols": [{"literal":"keywords"}], "postprocess": buildToken}, + {"name": "comparator", "symbols": [{"literal":"="}], "postprocess": buildLowercaseToken}, + {"name": "comparator", "symbols": [{"literal":"!="}], "postprocess": buildLowercaseToken}, + {"name": "comparator", "symbols": [{"literal":"contains"}], "postprocess": buildLowercaseToken}, + {"name": "comparator", "symbols": [{"literal":"CONTAINS"}], "postprocess": buildLowercaseToken}, + {"name": "comparator", "symbols": [{"literal":"like"}], "postprocess": buildLowercaseToken}, + {"name": "comparator", "symbols": [{"literal":"LIKE"}], "postprocess": buildLowercaseToken}, + {"name": "comparison", "symbols": ["field", "_", "comparator", "_", "word"], "postprocess": + function(d) { return { type: 'comparison', field: d[0], comparator: d[2], arg: d[4] }; } + }, + {"name": "comparison", "symbols": ["comparator", "_", "word"], "postprocess": + function(d) { return { type: 'comparison', field: '*', comparator: d[0], arg: d[2] }; } + }, + {"name": "sequence", "symbols": [], "postprocess": function(d) { return null; }}, + {"name": "sequence", "symbols": ["word", "_", "sequence"], "postprocess": buildSequence}, + {"name": "sequence", "symbols": ["comparison", "_", "sequence"], "postprocess": buildSequence}, + {"name": "inner", "symbols": [{"literal":"("}, "_", "expression", "_", {"literal":")"}], "postprocess": function(d) { return d[2]; }}, + {"name": "inner$subexpression$1", "symbols": [{"literal":"not"}]}, + {"name": "inner$subexpression$1", "symbols": [{"literal":"NOT"}]}, + {"name": "inner", "symbols": ["inner$subexpression$1", "_", "inner"], "postprocess": function(d) { return { type: 'not', arg: d[2] }; }}, + {"name": "inner", "symbols": ["sequence"], "postprocess": function(d) { return d[0]; }}, + {"name": "expression$subexpression$1", "symbols": [{"literal":"and"}]}, + {"name": "expression$subexpression$1", "symbols": [{"literal":"AND"}]}, + {"name": "expression", "symbols": ["inner", "_", "expression$subexpression$1", "_", "expression"], "postprocess": function(d) { return { type: 'and', left: d[0], right: d[4] }; }}, + {"name": "expression$subexpression$2", "symbols": [{"literal":"or"}]}, + {"name": "expression$subexpression$2", "symbols": [{"literal":"OR"}]}, + {"name": "expression", "symbols": ["inner", "_", "expression$subexpression$2", "_", "expression"], "postprocess": function(d) { return { type: 'or', left: d[0], right: d[4] }; }}, + {"name": "expression", "symbols": ["inner"], "postprocess": function(d) { return d[0]; }}, + {"name": "_", "symbols": []}, + {"name": "_", "symbols": [(syntax.has("WS") ? {type: "WS"} : WS)], "postprocess": function(d) { return null; }} +]; +export var ParserStart:string = "search"; diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts b/data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts new file mode 100644 index 0000000..2fe8342 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +import { SearchTextParser } from '.' +import { SearchQuery } from './search-query'; + +describe('Search text parser', () => { + + it('nested with comparison', () => { + let line = '"cow chicken" and not ("moo" or (label = grass))'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + let result = parser.flatten(tree); + let expected = {type: 'and', values: [ + {type: 'string', value :'cow chicken'}, + {type: 'not', values: [ + {type: 'or', values: [ + {type: 'string', value: 'moo'}, + {type: '=', value: 'label', values: [ + {type: 'string', value: 'grass'} + ]} + ]} + ]} + ]} as SearchQuery; + + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + + it('nested', () => { + let line = '"moo" or (label = grass)'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + let result = parser.flatten(tree); + let expected = {type: 'or', values: [ + {type: 'string', value :'moo'}, + {type: '=', value: 'label', values: [ + {type: 'string', value: 'grass'} + ]} + ]} as SearchQuery; + + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + + it('sequences', () => { + let line = 'moo cow and (bla or this)'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + let result = parser.flatten(tree); + let expected = {type: 'and', values: [ + {type: 'string', value :'moo'}, + {type: 'string', value: 'cow'}, + {type: 'or', values: [ + {type: 'string', value: 'bla'}, + {type: 'string', value: 'this'} + ]} + ]} as SearchQuery; + + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + + it('infix comparison', () => { + let line = 'contains corn or = rice'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + console.log('Tree', JSON.stringify(tree, null, ' ')); + + let result = parser.flatten(tree); + let expected = {type: 'or', values: [ + {type: 'contains', value: '*', values: [ + {type: 'string', value: 'corn'} + ]}, + {type: '=', value: '*', values: [ + {type: 'string', value: 'rice'} + ]} + ]} as SearchQuery; + + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + + it('empty string', () => { + let line = ''; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).toBeNull(); + + let result = parser.flatten(tree); + + expect(result).toBeNull(); + }); + + it('loose use of comma\'s and operators', () => { + let line = 'and cow, goose, sheep'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + let result = parser.flatten(tree); + let expected = {type: 'and', values: [ + {type: 'string', value: 'cow'}, + {type: 'string', value: 'goose'}, + {type: 'string', value: 'sheep'} + ]} as SearchQuery; + + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + + it('freely mix words and comparisons', () => { + let line = 'cow label != goose'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + let result = parser.flatten(tree); + let expected = {type: 'and', values: [ + {type: 'string', value: 'cow'}, + {type: '!=', value: 'label', values: [ + {type: 'string', value: 'goose'} + ]} + ]} as SearchQuery; + + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + + it('comparison with like', () => { + let line = 'label like goose'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + let result = parser.flatten(tree); + let expected = {type: 'like', value: 'label', values: [ + {type: 'string', value: 'goose'} + ]} as SearchQuery; + + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + + it('comparison with uppercase like', () => { + let line = 'label LIKE goose'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + let result = parser.flatten(tree); + let expected = {type: 'like', value: 'label', values: [ + {type: 'string', value: 'goose'} + ]} as SearchQuery; + + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + + it('empty and or', () => { + let line = 'and or and'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + }); + + it('test precedence of not over or', () => { + let line = 'not cow or goose'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + let result = parser.flatten(tree); + let expected = {type: 'or', values: [ + {type: 'not', values: [ + {type: 'string', value: 'cow'} + ]}, + {type: 'string', value: 'goose'}, + ]} as SearchQuery; + + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + + it('test elimination of double negation', () => { + let line = 'not not goose'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + let result = parser.flatten(tree); + let expected = {type: 'string', value: 'goose'} as SearchQuery; + + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + + it('test upper case and lower case operators', () => { + let line = 'not goose AND NOT cow'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + let result = parser.flatten(tree); + let expected = {type: 'and', values: [ + {type: 'not', values: [ + {type: 'string', value: 'goose'} + ]}, + {type: 'not', values: [ + {type: 'string', value: 'cow'} + ]} + ]} as SearchQuery; + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + +}); diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/index.ts b/data-showcase/src/main/user-interface/src/app/search-text-parser/index.ts new file mode 100644 index 0000000..951d065 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/index.ts @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +import { CompiledRules, Grammar, Parser } from 'nearley'; +import { Lexer, ParserRules, ParserStart } from './grammar'; +import { ParseTree, QueryType, SearchQuery } from './search-query'; + +export class SearchTextParser { + + parser: Parser; + + constructor() { + let rules: CompiledRules = { + Lexer: Lexer, + ParserStart: ParserStart, + ParserRules: ParserRules + }; + this.parser = new Parser(Grammar.fromCompiled(rules)); + } + + parse(line: string): ParseTree { + this.parser.feed(line); + if (this.parser.results.length == 0) { + return null; + } else { + return this.parser.results[0]; + } + } + + private flattenBinaryNode(node: ParseTree): SearchQuery { + let values = []; + let left = this.flattenNode(node.left); + if (left == null) { + // skip + } else if (left.type == node.type) { + values = values.concat(left.values); + } else { + values.push(left); + } + let right = this.flattenNode(node.right); + if (right == null) { + // skip + } else if (right.type == node.type) { + values = values.concat(right.values); + } else { + values.push(right); + } + return SearchQuery.forValues(node.type, values); + } + + private flattenNode(node: ParseTree): SearchQuery { + if (node === null || node === undefined) { + return null; + } + switch(node.type) { + case 'not': + if (node.arg == null) { + return null; + } else if (node.arg.type == 'not') { + return this.flatten(node.arg.arg); + } else { + return SearchQuery.forValues('not',[this.flatten(node.arg)]); + } + case 'or': + case 'and': + return this.flattenBinaryNode(node); + case 'string': + return SearchQuery.forValue(node.value); + case 'comparison': + let query = new SearchQuery(); + query.type = node.comparator as QueryType; + query.value = node.field; + query.values = [this.flattenNode(node.arg)]; + return query; + default: + throw Error(`Unexpected node type ${node.type}`); + } + } + + flatten(tree: ParseTree): SearchQuery { + return this.flattenNode( tree); + } + +} \ No newline at end of file diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/search-query.ts b/data-showcase/src/main/user-interface/src/app/search-text-parser/search-query.ts new file mode 100644 index 0000000..b9e2d4c --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/search-query.ts @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +export type Field = 'name' | 'label' | 'labelLong' | 'labelNl' | 'labelNlLong'; +export type Comparator = '=' | '!=' | 'contains' | 'like'; +export type Operator = 'and' | 'or' | 'not'; +export type NodeType = 'string' | 'comparison' | Operator; +export type QueryType = NodeType | Comparator; + +export class ParseTree { + type: NodeType; + // for comparison: + field: Field; + comparator: Comparator; + // for comparison, string: + value: string; + // for negation: + arg: ParseTree; + // for and, or: + left: ParseTree; + right: ParseTree; +} + +export class SearchQuery { + type: QueryType; + value: string; + values: SearchQuery[]; + + static forValues(type: QueryType, values: SearchQuery[]) : SearchQuery { + let result = new SearchQuery(); + result.type = type; + result.values = values; + return result; + } + + static forValue(value: string) : SearchQuery { + let result = new SearchQuery(); + result.type = 'string'; + result.value = value; + return result; + } + +} diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/syntax.ts b/data-showcase/src/main/user-interface/src/app/search-text-parser/syntax.ts new file mode 100644 index 0000000..6f87bd6 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/syntax.ts @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2017 The Hyve B.V. + * This file is distributed under the GNU Affero General Public License + * (see accompanying file LICENSE). + */ + +import { compile } from 'moo'; + +export const syntax = compile({ + word: { match: /[^ \t\n(),;"]+|"(?:\\["\\]|[^\n"\\])*"/, + keywords: { + 'not': 'not', + 'NOT': 'not', + 'and': 'and', + 'AND': 'AND', + 'or': 'or', + 'OR': 'OR', + '=': '=', + '!=': '!=', + 'contains': 'contains', + 'CONTAINS': 'CONTAINS', + 'like': 'like', + 'LIKE': 'LIKE', + 'name': 'name', + 'label': 'label', + 'labelNl': 'labelNl', + 'labelLong': 'labelLong', + 'labelNlLong': 'labelNlLong', + 'keywords': 'keywords' + } + }, + '(': '(', + ')': ')', + WS: /[ \t,;]+/, + NL: { match: /\n/, lineBreaks: true } +}); diff --git a/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts index 8d648d9..fb2ce3c 100644 --- a/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/search-parser.service.ts @@ -1,11 +1,8 @@ import { Injectable } from '@angular/core'; -import parser from 'logic-query-parser'; -import { QueryType, SearchQuery } from '../models/search-query'; -import { BinaryTree } from '../models/binary-tree'; +import { SearchTextParser } from '../search-text-parser'; +import { SearchQuery } from "../search-text-parser/search-query"; -const booleanOperators: Set = new Set(['and', 'or', 'not']); const valueOperators: Set = new Set(['=', '!=', 'like', 'contains']); -const fields: Set = new Set(['name', 'keywords', 'label', 'labelLong', 'labelNl', 'labelNlLong']); @Injectable() export class SearchParserService { @@ -22,7 +19,7 @@ export class SearchParserService { let args = query.values.map(SearchParserService.toHtml).join(' '); return `not (${args})`; } if (query.type == 'and' || query.type == 'or') { - return query.values.map(SearchParserService.toHtml).join(` ${query.type} `); + return '(' + query.values.map(SearchParserService.toHtml).join(`) ${query.type} (`) + ')'; } else if (valueOperators.has(query.type)) { let property = query.value ? `${query.value} `: ''; let args = query.values.map(SearchParserService.toHtml).join(' '); @@ -32,89 +29,6 @@ export class SearchParserService { } } - static parseQueryExpression(junctionOperator: QueryType, parts: string[]): SearchQuery { - if (parts.length == 0) { - return null - } - if (parts.length == 1) { - // single value, apply CONTAINS operator - return SearchQuery.forValue(parts[0]); - } - console.debug(`Parse query expression. parts = ${parts}`); - let property: string; - let operator: QueryType; - if (parts.length > 1) { - if (valueOperators.has(parts[0])) { - // unary operator applies to a list of values - operator = parts.shift() as QueryType; - console.debug(`Apply unary operator ${operator} to parts: ${parts}`); - return SearchQuery.forValues(operator, parts.map(value => SearchQuery.forValue(value))); - } - } - if (parts.length > 2) { - if (fields.has(parts[0]) && valueOperators.has(parts[1])) { - // binary operator applies to a field and a list of values - property = parts.shift(); - operator = parts.shift() as QueryType; - console.debug(`Apply operator ${operator} to field ${property} with values: ${parts}`); - let result = new SearchQuery(); - result.type = operator; - result.value = property; - result.values = parts.map(value => SearchQuery.forValue(value)); - return result; - } - } - return SearchQuery.forValues(junctionOperator, parts.map(value => SearchQuery.forValue(value))); - } - - - static processQuery(query: SearchQuery): SearchQuery { - if (query == null) { - return null; - } - let type = query.type; - if (!booleanOperators.has(type)) { - return query; - } - let values = query.values; - let valueTypes: Set = new Set(values.map(obj => obj.type)); - if (valueTypes.size == 1 && valueTypes.has('string')) { - // Only string values. Will parse the string expression. - return SearchParserService.parseQueryExpression(type, values.map(obj => obj.value)); - } else { - let result = new SearchQuery(); - result.type = type; - result.values = values.map(SearchParserService.processQuery); - return result; - } - } - - static parseNegation(binaryTree: BinaryTree): BinaryTree { - if (binaryTree == null) { - return null; - } - console.debug(`Parsing binary tree of type ${binaryTree.lexeme.type}...`, binaryTree); - if (binaryTree.lexeme.type == 'string') { - return binaryTree; - } else if (binaryTree.lexeme.type == 'and' || binaryTree.lexeme.type == 'or') { - if (binaryTree.left != null) { - let leftLexeme = binaryTree.left.lexeme; - if (leftLexeme.type == 'string' && leftLexeme.value.toLowerCase() == 'not') { - // apply 'not' operator to the right - return BinaryTree.forBranches('not', null, SearchParserService.parseNegation(binaryTree.right)); - } else { - // recursion - return BinaryTree.forBranches(binaryTree.lexeme.type, SearchParserService.parseNegation(binaryTree.left), SearchParserService.parseNegation(binaryTree.right)); - } - } else { - // left is null - return BinaryTree.forBranches(binaryTree.lexeme.type, null, SearchParserService.parseNegation(binaryTree.right)); - } - } else { - throw `Unexpected type: ${binaryTree.lexeme.type}`; - } - } - /* Generate binary tree with a logic query string as input * and parse it to JSON object, * using logic-query-parser library*/ @@ -122,10 +36,9 @@ export class SearchParserService { if (text == null || text.length == 0) { return null; } - let binaryTree = parser.parse(text) as BinaryTree; - binaryTree = SearchParserService.parseNegation(binaryTree); - let query = parser.utils.binaryTreeToQueryJson(binaryTree) as SearchQuery; - query = SearchParserService.processQuery(query); + let parser = new SearchTextParser(); + let binaryTree = parser.parse(text); + let query = parser.flatten(binaryTree) as SearchQuery; console.debug(`Query`, query); return query; } diff --git a/data-showcase/src/main/user-interface/tsconfig.json b/data-showcase/src/main/user-interface/tsconfig.json index ccfa574..aaec839 100644 --- a/data-showcase/src/main/user-interface/tsconfig.json +++ b/data-showcase/src/main/user-interface/tsconfig.json @@ -9,11 +9,8 @@ "experimentalDecorators": true, "target": "es5", "module": "commonjs", - "types": [ - "node" - ], "typeRoots": [ - "./node_modules/@types" + "node_modules/@types" ], "lib": [ "es2016", From a410f2fc2c28dff94f1ffee0c54745c28bbee334 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 20 Nov 2017 18:06:47 +0100 Subject: [PATCH 078/104] Update info popup. --- .../src/app/info/info.component.css | 21 +++ .../src/app/info/info.component.html | 147 ++++++------------ 2 files changed, 67 insertions(+), 101 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/info/info.component.css b/data-showcase/src/main/user-interface/src/app/info/info.component.css index 06fe6a7..d5bc335 100644 --- a/data-showcase/src/main/user-interface/src/app/info/info.component.css +++ b/data-showcase/src/main/user-interface/src/app/info/info.component.css @@ -13,3 +13,24 @@ .info-bar { height: 200px; } + +dl.explanation { + display: table; +} +.explanation dt, .explanation dd { + float: left; + display: inline-block; +} + +.explanation dt { + clear: left; + padding-right: 1em; +} + +.short-title dt { + min-width: 5em; +} + +.long-title dt { + min-width: 8em; +} diff --git a/data-showcase/src/main/user-interface/src/app/info/info.component.html b/data-showcase/src/main/user-interface/src/app/info/info.component.html index 1290fa4..f6e790a 100644 --- a/data-showcase/src/main/user-interface/src/app/info/info.component.html +++ b/data-showcase/src/main/user-interface/src/app/info/info.component.html @@ -14,117 +14,62 @@

Searching:

General structure of the query is: -
< FIELD OPERATOR value(s) >, +
field operator value,

- where FIELD is a name of the property to which the condition is applied (see: Supported fields). + where field is a name of the property to which the condition is applied (see: Supported fields).
- If FIELD is not specified, condition will be applied to all the supported fields. + If field is not specified, condition will be applied to all the supported fields. If condition is satisfied for at least one of the fields of an item, this item will be added to returned item list.

- Combination of multiple query criterion can be composed using 'AND' and 'OR' operators + Combination of multiple query criterion can be composed using and, or, and not operators (see: Supported operators). - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
'name' Name of the item.
'keywords' - Key words associated with concept.
'label' - The short name of the variable in English associated with concept.
'labelLong' - A textual description of the variable in English associated with concept.
'labelNl' - The short name of the variable in Dutch associated with concept.
'labelNlLong' - A textual description of the variable in Dutch associated with concept.
-
If any other field is used, an exception will be returned. +
+
name
+
Name of the item.
+
keywords
+
Key words associated with concept.
+
label
+
The short name of the variable in English associated with concept.
+
labelLong
+
A textual description of the variable in English associated with concept.
+
labelNl
+
The short name of the variable in Dutch associated with concept.
+
labelNlLong
+
A textual description of the variable in Dutch associated with concept.
+
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
'=' - equals,
- e.g. name = "value1" -
'!=' - not equals,
- e.g. labelEn != "value1" -
'IN' - allows to specify multiple values. Values have to be placed between round brackets.
- e.g. name IN ("value1", "value2") -
'LIKE' - like, search for a specified pattern. There are two wildcards used:
-
    -
  • % - The percent sign represents zero, one, or multiple characters;
    • -
  • _ - The underscore represents a single character;/n - e.g. labelNl LIKE "va_ue%"
-
'AND' - logical conjunction,
- e.g. (name = "value1") AND (labelLong != "value2") -
'OR' - logical disjunction,
- e.g. (name = "value1") OR (name = "value2") -
+
+
=
+
equals, e.g., name = "value1" +
+
!=
+
not equals, e.g., labelEn != "value1" +
+
like
+
like, search for a specified pattern. There are two wildcards used: +
    +
  • % The percent sign represents zero, one, or multiple characters;
    • +
  • _ The underscore represents a single character; + e.g., labelNl like "va_ue%"
+
+
AND
+
logical conjunction, + e.g., (name = "value1") AND (labelLong != "value2") +
+
OR
+
logical disjunction, + e.g., (name = "value1") OR (name = "value2") +
+
NOT
+
logical negation, + e.g., NOT (name = "value1") +
+

All the combinations of junction operators are possible, when using round brackets. -
e.g. ((name = "value1") OR (name = "value2")) AND (labelLong = "value3") +
e.g., ((name = "value1") OR (name = "value2")) AND (NOT labelLong = "value3")
From 365f553ff16b44090b81ada56162ed08088c19c3 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Tue, 21 Nov 2017 12:19:40 +0100 Subject: [PATCH 079/104] Add selected page reset on filter change reset the table to the first page after new filters are selected (new item list is returned) --- .../src/app/item-table/item-table.component.html | 3 ++- .../main/user-interface/src/app/services/data.service.ts | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html index 366f0a9..68d2168 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html @@ -48,7 +48,8 @@ - +
diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 0a23168..fbfee2b 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -311,7 +311,6 @@ export class DataService { } updateItemTable() { - this.clearItemsSelection(); this.clearAllFilters(); } @@ -319,6 +318,7 @@ export class DataService { this.projects.length = 0; this.selectedResearchLines = selectedResearchLines; this.clearItemsSelection(); + this.resetTableToTheFirstPage(); this.fetchItems(); this.getUniqueProjects(); } @@ -327,6 +327,7 @@ export class DataService { this.linesOfResearch.length = 0; this.selectedProjects = selectedProjects; this.clearItemsSelection(); + this.resetTableToTheFirstPage(); this.fetchItems(); this.getUniqueLinesOfResearch(); } @@ -347,6 +348,7 @@ export class DataService { clearAllFilters() { this.clearCheckboxFilterSelection(); + this.resetTableToTheFirstPage() this.setTextFilterInput(''); this.clearItemsSelection(); this.rerenderCheckboxFiltersSource.next(true); @@ -369,6 +371,10 @@ export class DataService { this.itemsSelectionSource.next([]); } + resetTableToTheFirstPage() { + this.itemsFirstResult = 0; + } + setTextFilterInput(text: string) { this.textFilterInputSource.next(text); } @@ -376,6 +382,7 @@ export class DataService { setSearchQuery(query: Object) { this.searchQuery = query; this.clearItemsSelection(); + this.resetTableToTheFirstPage(); this.fetchItems(); this.fetchFilters(); } From a2667603226bf4668179c873c011ff275fc3e3e0 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Tue, 21 Nov 2017 12:54:50 +0100 Subject: [PATCH 080/104] Support wildcard and 'keyword' in search query. --- .../enumeration/SearchField.groovy | 4 ++++ .../search/SearchCriteriaBuilder.groovy | 1 + .../src/app/info/info.component.html | 4 +++- .../src/app/search-text-parser/grammar.ne | 2 ++ .../src/app/search-text-parser/grammar.ts | 2 ++ .../src/app/search-text-parser/index.spec.ts | 22 +++++++++++++++++-- .../src/app/search-text-parser/syntax.ts | 4 +++- 7 files changed, 35 insertions(+), 4 deletions(-) diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/SearchField.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/SearchField.groovy index 541096c..97dd7f0 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/SearchField.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/enumeration/SearchField.groovy @@ -17,6 +17,10 @@ enum SearchField { * Key words associated with concept. */ KEYWORDS('keywords'), + /** + * Key word associated with concept. + */ + KEYWORD('keyword'), /** * The short name of the variable in English associated with concept. */ diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy index ccee914..8fa1794 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy @@ -147,6 +147,7 @@ class SearchCriteriaBuilder { switch (field) { case SearchField.NAME: return ITEM_ALIAS + "." + SearchField.NAME.value + case SearchField.KEYWORD: case SearchField.KEYWORDS: return KEYWORDS_ALIAS + "." + "keyword" case SearchField.LABEL: diff --git a/data-showcase/src/main/user-interface/src/app/info/info.component.html b/data-showcase/src/main/user-interface/src/app/info/info.component.html index f6e790a..d8f6fb5 100644 --- a/data-showcase/src/main/user-interface/src/app/info/info.component.html +++ b/data-showcase/src/main/user-interface/src/app/info/info.component.html @@ -14,7 +14,7 @@

Searching:

General structure of the query is: -
field operator value, +
⟨field⟩ ⟨operator⟩ ⟨value⟩,

where field is a name of the property to which the condition is applied (see: Supported fields).
@@ -38,6 +38,8 @@

Searching:

The short name of the variable in Dutch associated with concept.
labelNlLong
A textual description of the variable in Dutch associated with concept.
+
*
+
Any of these fields.
 diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne index d470ce7..0adb4fa 100644 --- a/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne @@ -49,6 +49,8 @@ field -> | "labelNl" {% buildToken %} | "labelNlLong" {% buildToken %} | "keywords" {% buildToken %} + | "keyword" {% buildToken %} + | "*" {% buildToken %} comparator -> "=" {% buildLowercaseToken %} diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts index dec0ccb..205f510 100644 --- a/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts @@ -51,6 +51,8 @@ export var ParserRules:NearleyRule[] = [ {"name": "field", "symbols": [{"literal":"labelNl"}], "postprocess": buildToken}, {"name": "field", "symbols": [{"literal":"labelNlLong"}], "postprocess": buildToken}, {"name": "field", "symbols": [{"literal":"keywords"}], "postprocess": buildToken}, + {"name": "field", "symbols": [{"literal":"keyword"}], "postprocess": buildToken}, + {"name": "field", "symbols": [{"literal":"*"}], "postprocess": buildToken}, {"name": "comparator", "symbols": [{"literal":"="}], "postprocess": buildLowercaseToken}, {"name": "comparator", "symbols": [{"literal":"!="}], "postprocess": buildLowercaseToken}, {"name": "comparator", "symbols": [{"literal":"contains"}], "postprocess": buildLowercaseToken}, diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts b/data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts index 2fe8342..cc6f497 100644 --- a/data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts @@ -77,8 +77,6 @@ describe('Search text parser', () => { expect(tree).not.toBeNull(); - console.log('Tree', JSON.stringify(tree, null, ' ')); - let result = parser.flatten(tree); let expected = {type: 'or', values: [ {type: 'contains', value: '*', values: [ @@ -92,6 +90,26 @@ describe('Search text parser', () => { expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); }); + it('wildcard comparison', () => { + let line = '* contains corn or keyword = rice'; + let parser = new SearchTextParser(); + let tree = parser.parse(line); + + expect(tree).not.toBeNull(); + + let result = parser.flatten(tree); + let expected = {type: 'or', values: [ + {type: 'contains', value: '*', values: [ + {type: 'string', value: 'corn'} + ]}, + {type: '=', value: 'keyword', values: [ + {type: 'string', value: 'rice'} + ]} + ]} as SearchQuery; + + expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); + }); + it('empty string', () => { let line = ''; let parser = new SearchTextParser(); diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/syntax.ts b/data-showcase/src/main/user-interface/src/app/search-text-parser/syntax.ts index 6f87bd6..15d2184 100644 --- a/data-showcase/src/main/user-interface/src/app/search-text-parser/syntax.ts +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/syntax.ts @@ -26,7 +26,9 @@ export const syntax = compile({ 'labelNl': 'labelNl', 'labelLong': 'labelLong', 'labelNlLong': 'labelNlLong', - 'keywords': 'keywords' + 'keywords': 'keywords', + 'keyword': 'keyword', + '*': '*' } }, '(': '(', From cddeb36773d06b7e296bb93c10dafd140b969724 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Tue, 21 Nov 2017 15:34:32 +0100 Subject: [PATCH 081/104] Add keywords to summary-popup --- .../datashowcase/KeywordController.groovy | 4 +++ .../thehyve/datashowcase/UrlMappings.groovy | 3 +- .../datashowcase/KeywordService.groovy | 28 +++++++++++++++++++ .../src/app/constants/endpoints.constants.ts | 1 + .../item-summary/item-summary.component.html | 10 +++++-- .../item-summary/item-summary.component.ts | 5 +++- .../src/app/services/data.service.ts | 14 ++++++++++ .../src/app/services/resource.service.ts | 18 +++++++++--- 8 files changed, 74 insertions(+), 9 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/KeywordController.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/KeywordController.groovy index 98853a7..7422f5b 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/KeywordController.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/KeywordController.groovy @@ -19,4 +19,8 @@ class KeywordController { respond keywords: keywordService.keywords } + def show(String conceptCode) { + respond keywords: keywordService.getKeywordsForConcept(conceptCode) + } + } diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy index e38403f..9e3ab99 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy @@ -22,7 +22,8 @@ class UrlMappings { "/api/projects"(controller: 'project') { action = [GET: 'index', POST: 'search'] } - "/api/keywords"(controller: 'keyword', includes: ['index']) + "/api/keywords/$conceptCode"(controller: 'keyword', action: 'show') + "/api/keywords"(controller: 'keyword', action: 'index') "/api/concepts"(controller: 'concept', includes: ['index']) "/api/lines_of_research"(controller: 'researchLine', includes: ['index']) "/api/tree_nodes"(controller: 'tree', includes: ['index']) diff --git a/data-showcase/grails-app/services/nl/thehyve/datashowcase/KeywordService.groovy b/data-showcase/grails-app/services/nl/thehyve/datashowcase/KeywordService.groovy index ae60d1a..4b01b0a 100644 --- a/data-showcase/grails-app/services/nl/thehyve/datashowcase/KeywordService.groovy +++ b/data-showcase/grails-app/services/nl/thehyve/datashowcase/KeywordService.groovy @@ -7,6 +7,13 @@ package nl.thehyve.datashowcase import grails.gorm.transactions.Transactional +import org.grails.core.util.StopWatch +import org.hibernate.Criteria +import org.hibernate.SessionFactory +import org.hibernate.criterion.Order +import org.hibernate.criterion.Projections +import org.hibernate.criterion.Restrictions +import org.hibernate.sql.JoinType import org.modelmapper.ModelMapper import org.springframework.beans.factory.annotation.Autowired @@ -16,10 +23,31 @@ class KeywordService { @Autowired ModelMapper modelMapper + SessionFactory sessionFactory + List getKeywords() { Keyword.findAll().collect({ it.keyword }) } + List getKeywordsForConcept(String conceptCode) { + def stopWatch = new StopWatch("Fetch keywords for concept: $conceptCode.") + stopWatch.start('Retrieve from database') + def session = sessionFactory.openSession() + + Criteria criteria = session.createCriteria(Concept, "c") + .createAlias("c.keywords", "k", JoinType.LEFT_OUTER_JOIN) + .setProjection(Projections.property("k.keyword").as("keyword")) + .add( Restrictions.eq('c.conceptCode', conceptCode)) + criteria.setResultTransformer(Criteria.ALIAS_TO_ENTITY_MAP) + criteria.addOrder(Order.asc("keyword")) + def keywords = criteria.list() as List + + stopWatch.stop() + + log.info "${keywords.size()} keywords fetched for concept: $conceptCode.\n${stopWatch.prettyPrint()}" + keywords.collect{it.keyword} + } + } diff --git a/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts b/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts index 726060b..5c4545a 100644 --- a/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts +++ b/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts @@ -8,5 +8,6 @@ export const PATH_TREE_NODES = "/api/tree_nodes"; export const PATH_ITEMS = "/api/items"; export const PATH_PROJECTS = "/api/projects"; export const PATH_CONCEPTS = "/api/concepts"; +export const PATH_KEYWORDS = "/api/keywords"; export const PATH_LOGOS = "/api/file/logo"; export const PATH_ENVIRONMENT = "/api/environment"; diff --git a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html index 5111cfc..536a05f 100644 --- a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html @@ -4,9 +4,9 @@ ~ (see accompanying file LICENSE). --> - - + +
@@ -39,6 +39,10 @@ + + + +
Name: {{item?.name}}Type of variable: {{item?.type}}
Keywords:{{dataService.keywordsForConcept.length > 0 ? dataService.keywordsForConcept.join(', ') : '-'}}

Summary statistics: diff --git a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts index d735ab0..6568747 100644 --- a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts @@ -18,11 +18,14 @@ export class ItemSummaryComponent implements OnInit { display: boolean = false; item: Item = null; - constructor(private dataService: DataService) { + constructor(public dataService: DataService) { dataService.itemSummaryVisible$.subscribe( visibleItem => { this.display = true; this.item = visibleItem; + if(visibleItem.concept) { + dataService.fetchKeywords(visibleItem.concept); + } }); } diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 6d3018b..68813e7 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -70,6 +70,9 @@ export class DataService { // list of all projects private allProjects: Project[] = []; + // keywords for a conceptCode + public keywordsForConcept: string[] = []; + // NTR logo private ntrLogoUrlSummary = new Subject(); public ntrLogoUrl$ = this.ntrLogoUrlSummary.asObservable(); @@ -198,6 +201,17 @@ export class DataService { } } + fetchKeywords(conceptCode: string) { + this.keywordsForConcept = []; + this.resourceService.getKeywords(conceptCode) + .subscribe( + (keywords: string[]) => { + this.keywordsForConcept = keywords; + }, + err => console.error(err) + ); + } + fetchFilters() { this.projects.length = 0; this.linesOfResearch.length = 0; diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index 2fd6e00..d24b87f 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -13,9 +13,9 @@ import {Http, Response, Headers, ResponseContentType, RequestOptions} from '@ang import {Endpoint} from "../models/endpoint"; import {AppConfig} from "../config/app.config"; import { - PATH_CONCEPTS, - PATH_ENVIRONMENT, PATH_ITEMS, PATH_LOGOS, PATH_PROJECTS, - PATH_TREE_NODES + PATH_CONCEPTS, + PATH_ENVIRONMENT, PATH_ITEMS, PATH_KEYWORDS, PATH_LOGOS, PATH_PROJECTS, + PATH_TREE_NODES } from "../constants/endpoints.constants"; import {Item} from "../models/item"; import {Project} from "../models/project"; @@ -112,7 +112,6 @@ export class ResourceService { .catch(this.handleError.bind(this)); } - getProjects(conceptCodes?: string[], jsonSearchQuery?: Object): Observable { let headers = new Headers(); headers.append('Content-Type', 'application/json'); @@ -134,6 +133,17 @@ export class ResourceService { .catch(this.handleError.bind(this)); } + getKeywords(conceptCode: string): Observable { + let headers = new Headers(); + let url = this.endpoint.apiUrl + PATH_KEYWORDS + "/" + conceptCode; + + return this.http.get(url, { + headers: headers + }) + .map((response: Response) => response.json().keywords as string[]) + .catch(this.handleError.bind(this)); + } + getConcepts(): Observable { let headers = new Headers(); let url = this.endpoint.apiUrl + PATH_CONCEPTS; From 2bcb45a51cfc4241de97e4c883c9a08699850575 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 22 Nov 2017 13:38:49 +0100 Subject: [PATCH 082/104] Change the url for fetching keywords for a concept --- .../controllers/nl/thehyve/datashowcase/UrlMappings.groovy | 2 +- .../user-interface/src/app/constants/endpoints.constants.ts | 2 +- .../user-interface/src/app/services/resource.service.ts | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy index 9e3ab99..dc2582f 100644 --- a/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy +++ b/data-showcase/grails-app/controllers/nl/thehyve/datashowcase/UrlMappings.groovy @@ -22,7 +22,7 @@ class UrlMappings { "/api/projects"(controller: 'project') { action = [GET: 'index', POST: 'search'] } - "/api/keywords/$conceptCode"(controller: 'keyword', action: 'show') + "/api/keywords/by_concept/$conceptCode"(controller: 'keyword', action: 'show') "/api/keywords"(controller: 'keyword', action: 'index') "/api/concepts"(controller: 'concept', includes: ['index']) "/api/lines_of_research"(controller: 'researchLine', includes: ['index']) diff --git a/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts b/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts index 5c4545a..be03cc3 100644 --- a/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts +++ b/data-showcase/src/main/user-interface/src/app/constants/endpoints.constants.ts @@ -8,6 +8,6 @@ export const PATH_TREE_NODES = "/api/tree_nodes"; export const PATH_ITEMS = "/api/items"; export const PATH_PROJECTS = "/api/projects"; export const PATH_CONCEPTS = "/api/concepts"; -export const PATH_KEYWORDS = "/api/keywords"; +export const PATH_KEYWORDS_BY_CONCEPT = "/api/keywords/by_concept"; export const PATH_LOGOS = "/api/file/logo"; export const PATH_ENVIRONMENT = "/api/environment"; diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index d24b87f..4713be6 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -14,8 +14,8 @@ import {Endpoint} from "../models/endpoint"; import {AppConfig} from "../config/app.config"; import { PATH_CONCEPTS, - PATH_ENVIRONMENT, PATH_ITEMS, PATH_KEYWORDS, PATH_LOGOS, PATH_PROJECTS, - PATH_TREE_NODES + PATH_ENVIRONMENT, PATH_ITEMS, PATH_LOGOS, PATH_PROJECTS, + PATH_TREE_NODES, PATH_KEYWORDS_BY_CONCEPT } from "../constants/endpoints.constants"; import {Item} from "../models/item"; import {Project} from "../models/project"; @@ -135,7 +135,7 @@ export class ResourceService { getKeywords(conceptCode: string): Observable { let headers = new Headers(); - let url = this.endpoint.apiUrl + PATH_KEYWORDS + "/" + conceptCode; + let url = this.endpoint.apiUrl + PATH_KEYWORDS_BY_CONCEPT + "/" + conceptCode; return this.http.get(url, { headers: headers From bdbe0f7f57f7211151f456f3f9102474c0f34631 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 22 Nov 2017 14:16:18 +0100 Subject: [PATCH 083/104] Move the keyword fetching function to ItemSummaryComponent --- .../item-summary/item-summary.component.html | 2 +- .../item-summary/item-summary.component.ts | 19 +++++++++++++++++-- .../src/app/services/data.service.ts | 14 -------------- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html index 536a05f..b542fa7 100644 --- a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html @@ -41,7 +41,7 @@ Keywords: - {{dataService.keywordsForConcept.length > 0 ? dataService.keywordsForConcept.join(', ') : '-'}} + {{keywordsForConcept.length > 0 ? keywordsForConcept.join(', ') : '-'}}
diff --git a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts index 6568747..2c29683 100644 --- a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts @@ -7,6 +7,7 @@ import { Component, OnInit } from '@angular/core'; import {DataService} from "../services/data.service"; import {Item} from "../models/item"; +import {ResourceService} from "../services/resource.service"; @Component({ selector: 'app-item-summary', @@ -17,18 +18,32 @@ export class ItemSummaryComponent implements OnInit { display: boolean = false; item: Item = null; + keywordsForConcept: string[] = []; - constructor(public dataService: DataService) { + constructor(private dataService: DataService, + private resourceService: ResourceService) { dataService.itemSummaryVisible$.subscribe( visibleItem => { this.display = true; this.item = visibleItem; if(visibleItem.concept) { - dataService.fetchKeywords(visibleItem.concept); + this.fetchKeywords(visibleItem.concept); } }); } ngOnInit() { } + + + fetchKeywords(conceptCode: string) { + this.keywordsForConcept = []; + this.resourceService.getKeywords(conceptCode) + .subscribe( + (keywords: string[]) => { + this.keywordsForConcept = keywords; + }, + err => console.error(err) + ); + } } diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 68813e7..6d3018b 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -70,9 +70,6 @@ export class DataService { // list of all projects private allProjects: Project[] = []; - // keywords for a conceptCode - public keywordsForConcept: string[] = []; - // NTR logo private ntrLogoUrlSummary = new Subject(); public ntrLogoUrl$ = this.ntrLogoUrlSummary.asObservable(); @@ -201,17 +198,6 @@ export class DataService { } } - fetchKeywords(conceptCode: string) { - this.keywordsForConcept = []; - this.resourceService.getKeywords(conceptCode) - .subscribe( - (keywords: string[]) => { - this.keywordsForConcept = keywords; - }, - err => console.error(err) - ); - } - fetchFilters() { this.projects.length = 0; this.linesOfResearch.length = 0; From 1733f83e84b31f7a434779fe645d7532febd76b0 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 22 Nov 2017 14:28:07 +0100 Subject: [PATCH 084/104] Rename "patient counts" to "subject counts" --- .../grails-app/domain/nl/thehyve/datashowcase/Summary.groovy | 2 +- .../datashowcase/representation/SummaryRepresentation.groovy | 2 +- .../src/app/item-summary/item-summary.component.html | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy index aff4382..d3e94fc 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy @@ -18,7 +18,7 @@ class Summary { Long observationCount /** - * The number of patients for whom there are observations associated with the variable. + * The number of patients (subjects) for whom there are observations associated with the variable. */ Long patientCount diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/SummaryRepresentation.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/SummaryRepresentation.groovy index fe5eb73..faa56d0 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/SummaryRepresentation.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/representation/SummaryRepresentation.groovy @@ -17,7 +17,7 @@ abstract class SummaryRepresentation { Long observationCount /** - * The number of patients for whom there are observations associated with the variable. + * The number of patients (subjects) for whom there are observations associated with the variable. */ Long patientCount diff --git a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html index 5111cfc..4a8fb32 100644 --- a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html @@ -44,7 +44,7 @@ Summary statistics: - + From fc7cd95b5c8d1ef995a6b89b737c3189905595c7 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 22 Nov 2017 13:33:22 +0100 Subject: [PATCH 085/104] Extend e2e tests for data loading --- data-showcase-api-e2e/build.gradle | 1 + .../src/test/groovy/base/RestHelper.groovy | 5 + .../src/test/groovy/config/Config.groovy | 2 + .../groovy/tests/rest/DataImportSpec.groovy | 99 ++++++++- .../src/test/resources/test.json | 190 ++++++++++++++++++ .../src/test/resources/test_empty.json | 0 .../src/test/resources/test_invalid.json | 189 +++++++++++++++++ 7 files changed, 478 insertions(+), 8 deletions(-) create mode 100644 data-showcase-api-e2e/src/test/resources/test.json create mode 100644 data-showcase-api-e2e/src/test/resources/test_empty.json create mode 100644 data-showcase-api-e2e/src/test/resources/test_invalid.json diff --git a/data-showcase-api-e2e/build.gradle b/data-showcase-api-e2e/build.gradle index 0d46b53..7653b0b 100644 --- a/data-showcase-api-e2e/build.gradle +++ b/data-showcase-api-e2e/build.gradle @@ -23,6 +23,7 @@ dependencies { testCompile 'org.spockframework:spock-core:1.0-groovy-2.4' testCompile 'io.github.http-builder-ng:http-builder-ng-apache:0.17.0' + testCompile 'io.github.http-builder-ng:http-builder-ng-okhttp:0.17.0' } tasks.withType(Test) { systemProperties = System.getProperties() } diff --git a/data-showcase-api-e2e/src/test/groovy/base/RestHelper.groovy b/data-showcase-api-e2e/src/test/groovy/base/RestHelper.groovy index 8995064..524b1ec 100644 --- a/data-showcase-api-e2e/src/test/groovy/base/RestHelper.groovy +++ b/data-showcase-api-e2e/src/test/groovy/base/RestHelper.groovy @@ -8,6 +8,7 @@ package base import groovyx.net.http.FromServer import groovyx.net.http.HttpBuilder +import groovyx.net.http.OkHttpEncoders import static config.Config.AUTH_NEEDED import static config.Config.DEFAULT_USER @@ -81,6 +82,10 @@ class RestHelper { request.contentType = requestMap.contentType ?: ContentTypeFor.JSON request.body = requestMap.body + if(requestMap.contentType == 'multipart/form-data') { + request.encoder 'multipart/form-data', OkHttpEncoders.&multipart + } + if (!requestMap.skipOauth && AUTH_NEEDED) { testContext.getAuthAdapter().authenticate(getRequest(), (requestMap.user ?: DEFAULT_USER)) } diff --git a/data-showcase-api-e2e/src/test/groovy/config/Config.groovy b/data-showcase-api-e2e/src/test/groovy/config/Config.groovy index a979c15..45e69e5 100644 --- a/data-showcase-api-e2e/src/test/groovy/config/Config.groovy +++ b/data-showcase-api-e2e/src/test/groovy/config/Config.groovy @@ -39,4 +39,6 @@ class Config { //settings public static final boolean AUTH_NEEDED = false public static final String DEFAULT_USER = 'default' + public static final String TOKEN = 'zh6WWMJcnIbRqU4KnK0sFPRapldWK6PS7hC5Gnm/GYU=' + } diff --git a/data-showcase-api-e2e/src/test/groovy/tests/rest/DataImportSpec.groovy b/data-showcase-api-e2e/src/test/groovy/tests/rest/DataImportSpec.groovy index 55207bd..a6a30ab 100644 --- a/data-showcase-api-e2e/src/test/groovy/tests/rest/DataImportSpec.groovy +++ b/data-showcase-api-e2e/src/test/groovy/tests/rest/DataImportSpec.groovy @@ -7,9 +7,36 @@ package tests.rest import base.RESTSpec +import static config.Config.TOKEN +import static groovyx.net.http.MultipartContent.multipart class DataImportSpec extends RESTSpec { + def "upload valid data"() { + given: + def env = get([path: '/api/environment']) + assert env['grailsEnvironment'] == 'test' + assert env['environment'] == 'Public' + get([path: '/api/test/clearDatabase']) + + when: + def file = new File(getClass().getResource("/test.json").toURI()) + def requestToken = TOKEN + def request = [ + path : '/api/data_import/upload', + contentType: 'multipart/form-data', + body : multipart { + field 'requestToken', requestToken + part 'file', 'test.json', 'text/plain', file + }, + statusCode : 200 + ] + def response = post(request) + + then: + assert response.message == "Data successfully uploaded" + } + def "upload data with invalid requestToken"() { given: def env = get([path: '/api/environment']) @@ -19,19 +46,75 @@ class DataImportSpec extends RESTSpec { get([path: '/api/test/clearDatabase']) when: - def file = null - //file = new GrailsMockMultipartFile('testFile', 'test file contents'.bytes) + def requestToken = null + def file = new File(getClass().getResource("/test.json").toURI()) def request = [ - path: '/api/data_import/upload', - body: toJSON([ - file : file, - requestToken: null - ]), - statusCode: 401 + path : '/api/data_import/upload', + contentType: 'multipart/form-data', + body : multipart { + field 'requestToken', requestToken + part 'file', 'test.json', 'text/plain', file + }, + statusCode : 401 ] def response = post(request) then: assert response.error == 'requestToken is required to upload the data' } + + def "upload invalid data"() { + given: + def env = get([path: '/api/environment']) + assert env['grailsEnvironment'] == 'test' + assert env['environment'] == 'Public' + get([path: '/api/test/clearDatabase']) + + when: + // File with invalid data: + // project names not unique, concept code of the first item is missing + def file = new File(getClass().getResource("/test_invalid.json").toURI()) + def requestToken = TOKEN + def request = [ + path : '/api/data_import/upload', + contentType: 'multipart/form-data', + body : multipart { + field 'requestToken', requestToken + part 'file', 'test.json', 'text/plain', file + }, + statusCode : 400 + ] + def response = post(request) + + then: + assert response.status == 400 + assert response.error == "Bad request" + } + + def "upload empty file"() { + given: + def env = get([path: '/api/environment']) + assert env['grailsEnvironment'] == 'test' + assert env['environment'] == 'Public' + get([path: '/api/test/clearDatabase']) + + when: + def file = new File(getClass().getResource("/test_invalid.json").toURI()) + def requestToken = TOKEN + def request = [ + path : '/api/data_import/upload', + contentType: 'multipart/form-data', + body : multipart { + field 'requestToken', requestToken + part 'file', 'test.json', 'text/plain', file + }, + statusCode : 400 + ] + def response = post(request) + + then: + assert response.status == 400 + assert response.error == "Bad request" + } + } diff --git a/data-showcase-api-e2e/src/test/resources/test.json b/data-showcase-api-e2e/src/test/resources/test.json new file mode 100644 index 0000000..80e11ba --- /dev/null +++ b/data-showcase-api-e2e/src/test/resources/test.json @@ -0,0 +1,190 @@ +{ + "items": [ + { + "conceptCode": "age", + "publicItem": true, + "summary": { + "observationCount": 102, + "dataStability": "Committed", + "maxValue": null, + "patientCount": 100, + "minValue": null, + "avgValue": null, + "stdDevValue": null, + "values": [ + { + "value": "<= 65", + "frequency": 35, + "label": "Young" + }, + { + "value": "> 65", + "frequency": 65, + "label": "Old" + } + ] + }, + "itemPath": "/Personal information/Basic information/Age", + "name": "ageA", + "projectName": "Project A" + }, + { + "conceptCode": "height", + "publicItem": true, + "summary": { + "observationCount": 402, + "dataStability": "Committed", + "maxValue": null, + "patientCount": 200, + "minValue": null, + "avgValue": null, + "stdDevValue": null, + "values": [ + { + "value": "<= 175", + "frequency": 36, + "label": "Short" + }, + { + "value": "> 175", + "frequency": 64, + "label": "Tall" + } + ] + }, + "itemPath": "/Personal information/Extended information/Height", + "name": "heightB", + "projectName": "Project B" + } + ], + "concepts": [ + { + "labelNlLong": "Leeftijd van het onderwerp", + "labelNl": "Leeftijd", + "conceptCode": "age", + "variableType": "Categorical", + "labelLong": "Age at time of survey", + "label": "Age", + "keywords": [ + "Personal information" + ] + }, + { + "labelNlLong": "Hoogte van het onderwerp", + "labelNl": "Hoogte", + "conceptCode": "height", + "variableType": "Categorical", + "labelLong": "Height at time of survey", + "label": "Height", + "keywords": [ + "Personal information", + "Family related", + "Body characteristics" + ] + }, + { + "labelNlLong": "Gewicht (kg)", + "labelNl": "Gewicht", + "conceptCode": "weight", + "variableType": "Numerical", + "labelLong": "Weight (kg)", + "label": "Weight", + "keywords": [ + "Personal information", + "Family related", + "Body characteristics" + ] + } + ], + "projects": [ + { + "name": "Project A", + "description": "First test project", + "lineOfResearch": "Research line 1" + }, + { + "name": "Project B", + "description": "Second test project", + "lineOfResearch": "Research line 2" + } + ], + "tree_nodes": [ + { + "conceptCode": null, + "path": "/Personal information", + "nodeType": "Domain", + "label": "Personal information", + "children": [ + { + "conceptCode": null, + "path": "/Personal information/Basic information", + "nodeType": "Domain", + "label": "Basic information", + "children": [ + { + "conceptCode": "age", + "path": "/Personal information/Basic information/Age", + "nodeType": "Concept", + "label": "Age", + "children": [ + + ] + }, + { + "conceptCode": "weight", + "path": "/Personal information/Basic information/Weight", + "nodeType": "Concept", + "label": "Weight", + "children": [ + + ] + } + ] + }, + { + "conceptCode": null, + "path": "/Personal information/Extended information", + "nodeType": "Domain", + "label": "Extended information", + "children": [ + { + "conceptCode": "height", + "path": "/Personal information/Extended information/Height", + "nodeType": "Concept", + "label": "Height", + "children": [ + + ] + }, + { + "conceptCode": null, + "path": "/Personal information/Extended information/Some details", + "nodeType": "Domain", + "label": "Some details", + "children": [ + + ] + } + ] + } + ] + }, + { + "conceptCode": null, + "path": "/Other information", + "nodeType": "Domain", + "label": "Other information", + "children": [ + { + "conceptCode": null, + "path": "/Other information/Some information", + "nodeType": "Domain", + "label": "Some information", + "children": [ + + ] + } + ] + } + ] +} diff --git a/data-showcase-api-e2e/src/test/resources/test_empty.json b/data-showcase-api-e2e/src/test/resources/test_empty.json new file mode 100644 index 0000000..e69de29 diff --git a/data-showcase-api-e2e/src/test/resources/test_invalid.json b/data-showcase-api-e2e/src/test/resources/test_invalid.json new file mode 100644 index 0000000..e35b702 --- /dev/null +++ b/data-showcase-api-e2e/src/test/resources/test_invalid.json @@ -0,0 +1,189 @@ +{ + "items": [ + { + "publicItem": true, + "summary": { + "observationCount": 102, + "dataStability": "Committed", + "maxValue": null, + "patientCount": 100, + "minValue": null, + "avgValue": null, + "stdDevValue": null, + "values": [ + { + "value": "<= 65", + "frequency": 35, + "label": "Young" + }, + { + "value": "> 65", + "frequency": 65, + "label": "Old" + } + ] + }, + "itemPath": "/Personal information/Basic information/Age", + "name": "ageA", + "projectName": "Project A" + }, + { + "conceptCode": "height", + "publicItem": true, + "summary": { + "observationCount": 402, + "dataStability": "Committed", + "maxValue": null, + "patientCount": 200, + "minValue": null, + "avgValue": null, + "stdDevValue": null, + "values": [ + { + "value": "<= 175", + "frequency": 36, + "label": "Short" + }, + { + "value": "> 175", + "frequency": 64, + "label": "Tall" + } + ] + }, + "itemPath": "/Personal information/Extended information/Height", + "name": "heightB", + "projectName": "Project B" + } + ], + "concepts": [ + { + "labelNlLong": "Leeftijd van het onderwerp", + "labelNl": "Leeftijd", + "conceptCode": "age", + "variableType": "Categorical", + "labelLong": "Age at time of survey", + "label": "Age", + "keywords": [ + "Personal information" + ] + }, + { + "labelNlLong": "Hoogte van het onderwerp", + "labelNl": "Hoogte", + "conceptCode": "height", + "variableType": "Categorical", + "labelLong": "Height at time of survey", + "label": "Height", + "keywords": [ + "Personal information", + "Family related", + "Body characteristics" + ] + }, + { + "labelNlLong": "Gewicht (kg)", + "labelNl": "Gewicht", + "conceptCode": "weight", + "variableType": "Numerical", + "labelLong": "Weight (kg)", + "label": "Weight", + "keywords": [ + "Personal information", + "Family related", + "Body characteristics" + ] + } + ], + "projects": [ + { + "name": "Project A", + "description": "First test project", + "lineOfResearch": "Research line 1" + }, + { + "name": "Project A", + "description": "Second test project", + "lineOfResearch": "Research line 2" + } + ], + "tree_nodes": [ + { + "conceptCode": null, + "path": "/Personal information", + "nodeType": "Domain", + "label": "Personal information", + "children": [ + { + "conceptCode": null, + "path": "/Personal information/Basic information", + "nodeType": "Domain", + "label": "Basic information", + "children": [ + { + "conceptCode": "age", + "path": "/Personal information/Basic information/Age", + "nodeType": "Concept", + "label": "Age", + "children": [ + + ] + }, + { + "conceptCode": "weight", + "path": "/Personal information/Basic information/Weight", + "nodeType": "Concept", + "label": "Weight", + "children": [ + + ] + } + ] + }, + { + "conceptCode": null, + "path": "/Personal information/Extended information", + "nodeType": "Domain", + "label": "Extended information", + "children": [ + { + "conceptCode": "height", + "path": "/Personal information/Extended information/Height", + "nodeType": "Concept", + "label": "Height", + "children": [ + + ] + }, + { + "conceptCode": null, + "path": "/Personal information/Extended information/Some details", + "nodeType": "Domain", + "label": "Some details", + "children": [ + + ] + } + ] + } + ] + }, + { + "conceptCode": null, + "path": "/Other information", + "nodeType": "Domain", + "label": "Other information", + "children": [ + { + "conceptCode": null, + "path": "/Other information/Some information", + "nodeType": "Domain", + "label": "Some information", + "children": [ + + ] + } + ] + } + ] +} From bddd3d9b9c7d4d0dc69262df6fa1978f4e10fba5 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 23 Nov 2017 09:07:41 +0100 Subject: [PATCH 086/104] Change .gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1c13b20..4785fef 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,8 @@ gradle/wrapper out # IDEs and editors -/.idea +.idea +*/.idea *.iml /lib .project From 5c3e35427e114251d64804d870b8084ce9b929b8 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 23 Nov 2017 09:28:08 +0100 Subject: [PATCH 087/104] Change the token for e2e tests --- data-showcase-api-e2e/src/test/groovy/config/Config.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-showcase-api-e2e/src/test/groovy/config/Config.groovy b/data-showcase-api-e2e/src/test/groovy/config/Config.groovy index 45e69e5..bef7892 100644 --- a/data-showcase-api-e2e/src/test/groovy/config/Config.groovy +++ b/data-showcase-api-e2e/src/test/groovy/config/Config.groovy @@ -39,6 +39,6 @@ class Config { //settings public static final boolean AUTH_NEEDED = false public static final String DEFAULT_USER = 'default' - public static final String TOKEN = 'zh6WWMJcnIbRqU4KnK0sFPRapldWK6PS7hC5Gnm/GYU=' + public static final String TOKEN = 'TestToken123!' } From b5cdc392f764a12e82a242284199c82d9480e62a Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 23 Nov 2017 10:09:57 +0100 Subject: [PATCH 088/104] Change default operator in the search text parser --- .../main/user-interface/src/app/search-text-parser/grammar.ne | 2 +- .../main/user-interface/src/app/search-text-parser/grammar.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne index 0adb4fa..973937a 100644 --- a/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ne @@ -8,7 +8,7 @@ import { syntax } from './syntax'; -const defaultOperator = 'and'; +const defaultOperator = 'or'; function stripQuotes(word) { return word.value.replace(/^"|"$/g, ''); diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts index 205f510..0ffaf1f 100644 --- a/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/grammar.ts @@ -12,7 +12,7 @@ declare var WS:any; import { syntax } from './syntax'; -const defaultOperator = 'and'; +const defaultOperator = 'or'; function stripQuotes(word) { return word.value.replace(/^"|"$/g, ''); From 399fa1ca196ff796b6c497c6e83bdacb1bc801b9 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 23 Nov 2017 10:29:16 +0100 Subject: [PATCH 089/104] Fix search text parsing tests --- .../src/app/app.component.spec.ts | 5 +++-- .../app/item-table/item-table.component.spec.ts | 5 +++-- .../src/app/search-text-parser/index.spec.ts | 17 ++++++++++------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts index f1d1bac..8eeddcd 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts @@ -9,7 +9,7 @@ import { TestBed, async } from '@angular/core/testing'; import { AppComponent } from './app.component'; import {TreeNodesComponent} from "./tree-nodes/tree-nodes.component"; import { - AutoCompleteModule, DataTableModule, DialogModule, FieldsetModule, PaginatorModule, PanelModule, + AutoCompleteModule, CheckboxModule, DataTableModule, DialogModule, FieldsetModule, PaginatorModule, PanelModule, TreeModule } from "primeng/primeng"; import {FormsModule} from "@angular/forms"; @@ -58,7 +58,8 @@ describe('AppComponent', () => { DialogModule, HttpModule, InfoModule, - PaginatorModule + PaginatorModule, + CheckboxModule ], providers: [ DataService, diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts index ee481c9..fc39de0 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts @@ -9,7 +9,7 @@ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import {ItemFilter, ItemTableComponent} from './item-table.component'; import {FormsModule} from "@angular/forms"; import { - AutoCompleteModule, DataListModule, DataTableModule, FieldsetModule, ListboxModule, PaginatorModule, + AutoCompleteModule, CheckboxModule, DataListModule, DataTableModule, FieldsetModule, ListboxModule, PaginatorModule, PanelModule } from "primeng/primeng"; import {BrowserAnimationsModule} from "@angular/platform-browser/animations"; @@ -36,7 +36,8 @@ describe('ItemTableComponent', () => { BrowserAnimationsModule, DataTableModule, HttpModule, - PaginatorModule + PaginatorModule, + CheckboxModule ], providers: [ DataService, diff --git a/data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts b/data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts index cc6f497..d1333ca 100644 --- a/data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/search-text-parser/index.spec.ts @@ -59,8 +59,10 @@ describe('Search text parser', () => { let result = parser.flatten(tree); let expected = {type: 'and', values: [ - {type: 'string', value :'moo'}, - {type: 'string', value: 'cow'}, + {type: 'or', values: [ + {type: 'string', value: 'moo'}, + {type: 'string', value: 'cow'} + ]}, {type: 'or', values: [ {type: 'string', value: 'bla'}, {type: 'string', value: 'this'} @@ -131,10 +133,11 @@ describe('Search text parser', () => { let result = parser.flatten(tree); let expected = {type: 'and', values: [ - {type: 'string', value: 'cow'}, - {type: 'string', value: 'goose'}, - {type: 'string', value: 'sheep'} - ]} as SearchQuery; + {type: 'or', values :[ + {type: 'string', value: 'cow'}, + {type: 'string', value: 'goose'}, + {type: 'string', value: 'sheep'} + ]}]} as SearchQuery; expect(JSON.stringify(result)).toEqual(JSON.stringify(expected)); }); @@ -147,7 +150,7 @@ describe('Search text parser', () => { expect(tree).not.toBeNull(); let result = parser.flatten(tree); - let expected = {type: 'and', values: [ + let expected = {type: 'or', values: [ {type: 'string', value: 'cow'}, {type: '!=', value: 'label', values: [ {type: 'string', value: 'goose'} From c0c4c7d978df7d4c71620e05564b5198f9bbd780 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 23 Nov 2017 10:45:07 +0100 Subject: [PATCH 090/104] Fix e2e tests --- .../groovy/tests/rest/DataImportSpec.groovy | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/data-showcase-api-e2e/src/test/groovy/tests/rest/DataImportSpec.groovy b/data-showcase-api-e2e/src/test/groovy/tests/rest/DataImportSpec.groovy index a6a30ab..ecfd8ed 100644 --- a/data-showcase-api-e2e/src/test/groovy/tests/rest/DataImportSpec.groovy +++ b/data-showcase-api-e2e/src/test/groovy/tests/rest/DataImportSpec.groovy @@ -46,7 +46,7 @@ class DataImportSpec extends RESTSpec { get([path: '/api/test/clearDatabase']) when: - def requestToken = null + def requestToken = "" def file = new File(getClass().getResource("/test.json").toURI()) def request = [ path : '/api/data_import/upload', @@ -61,6 +61,22 @@ class DataImportSpec extends RESTSpec { then: assert response.error == 'requestToken is required to upload the data' + + when: + requestToken = "invalid" + request = [ + path : '/api/data_import/upload', + contentType: 'multipart/form-data', + body : multipart { + field 'requestToken', requestToken + part 'file', 'test.json', 'text/plain', file + }, + statusCode : 401 + ] + response = post(request) + + then: + assert response.error == "requestToken: $requestToken is invalid" } def "upload invalid data"() { @@ -88,7 +104,7 @@ class DataImportSpec extends RESTSpec { then: assert response.status == 400 - assert response.error == "Bad request" + assert response.error == "Bad Request" } def "upload empty file"() { @@ -114,7 +130,7 @@ class DataImportSpec extends RESTSpec { then: assert response.status == 400 - assert response.error == "Bad request" + assert response.error == "Bad Request" } } From 4fc210f44df2aa4d05a8533e2563bf2081159121 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Tue, 28 Nov 2017 10:42:18 +0100 Subject: [PATCH 091/104] Add "Add to cart" button to a metadata pop-up - add shopping cart update messages - fix tests --- .../src/app/app.component.spec.ts | 13 +++++--- .../main/user-interface/src/app/app.module.ts | 6 +++- .../checkbox-filter.component.spec.ts | 4 +++ .../src/app/filters/filters.component.spec.ts | 4 +++ .../text-filter/text-filter.component.spec.ts | 4 +++ .../item-summary/item-summary.component.css | 4 +++ .../item-summary/item-summary.component.html | 7 ++++- .../item-summary.component.spec.ts | 4 +++ .../item-summary/item-summary.component.ts | 8 +++++ .../app/item-table/item-table.component.html | 3 +- .../item-table/item-table.component.spec.ts | 4 +++ .../app/item-table/item-table.component.ts | 2 +- .../src/app/logos/logos.component.spec.ts | 4 +++ .../page-ribbon/page-ribbon.component.spec.ts | 4 +++ .../src/app/services/data.service.spec.ts | 4 +++ .../src/app/services/data.service.ts | 24 +++++++++++--- .../app/services/ds-message.service.spec.ts | 31 +++++++++++++++++++ .../src/app/services/ds-message.service.ts | 28 +++++++++++++++++ .../shopping-cart.component.spec.ts | 6 +++- .../tree-nodes/tree-nodes.component.spec.ts | 4 +++ 20 files changed, 154 insertions(+), 14 deletions(-) create mode 100644 data-showcase/src/main/user-interface/src/app/services/ds-message.service.spec.ts create mode 100644 data-showcase/src/main/user-interface/src/app/services/ds-message.service.ts diff --git a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts index 8eeddcd..8693c99 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/app.component.spec.ts @@ -9,11 +9,11 @@ import { TestBed, async } from '@angular/core/testing'; import { AppComponent } from './app.component'; import {TreeNodesComponent} from "./tree-nodes/tree-nodes.component"; import { - AutoCompleteModule, CheckboxModule, DataTableModule, DialogModule, FieldsetModule, PaginatorModule, PanelModule, + AutoCompleteModule, CheckboxModule, DataTableModule, DialogModule, FieldsetModule, GrowlModule, PaginatorModule, + PanelModule, TreeModule } from "primeng/primeng"; import {FormsModule} from "@angular/forms"; -import {CheckboxFilterComponent} from "./filters/checkbox-filter/checkbox-filter.component"; import {ListboxModule} from "primeng/components/listbox/listbox"; import {ItemFilter, ItemTableComponent} from "./item-table/item-table.component"; import {DataService} from "./services/data.service"; @@ -30,6 +30,8 @@ import {PageRibbonComponent} from "./page-ribbon/page-ribbon.component"; import {FiltersModule} from "./filters/filters.module"; import {InfoModule} from "./info/info.module"; import {SearchParserService} from "./services/search-parser.service"; +import {DSMessageService} from "./services/ds-message.service"; +import {MessageService} from "primeng/components/common/messageservice"; describe('AppComponent', () => { beforeEach(async(() => { @@ -59,7 +61,8 @@ describe('AppComponent', () => { HttpModule, InfoModule, PaginatorModule, - CheckboxModule + CheckboxModule, + GrowlModule ], providers: [ DataService, @@ -68,7 +71,9 @@ describe('AppComponent', () => { useClass: AppConfigMock }, ResourceService, - SearchParserService + SearchParserService, + DSMessageService, + MessageService ] }).compileComponents(); })); diff --git a/data-showcase/src/main/user-interface/src/app/app.module.ts b/data-showcase/src/main/user-interface/src/app/app.module.ts index f7f9b76..71e0faf 100644 --- a/data-showcase/src/main/user-interface/src/app/app.module.ts +++ b/data-showcase/src/main/user-interface/src/app/app.module.ts @@ -23,6 +23,8 @@ import {PageRibbonComponent} from './page-ribbon/page-ribbon.component'; import {FiltersModule} from "./filters/filters.module"; import {InfoModule} from "./info/info.module"; import {SearchParserService} from "./services/search-parser.service"; +import {GrowlModule} from "primeng/primeng"; +import {DSMessageService} from "./services/ds-message.service"; export function initConfig(config: AppConfig) { return () => config.load() @@ -44,12 +46,14 @@ export function initConfig(config: AppConfig) { ItemTableModule, ShoppingCartModule, ItemSummaryModule, - InfoModule + InfoModule, + GrowlModule ], providers: [ ResourceService, DataService, SearchParserService, + DSMessageService, AppConfig, { provide: APP_INITIALIZER, diff --git a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.spec.ts b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.spec.ts index 166aedb..8fda325 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.spec.ts @@ -15,6 +15,8 @@ import {BrowserAnimationsModule} from "@angular/platform-browser/animations"; import {HttpModule} from "@angular/http"; import {AppConfig} from "../../config/app.config"; import {AppConfigMock} from "../../config/app.config.mock"; +import {DSMessageService} from "../../services/ds-message.service"; +import {MessageService} from "primeng/components/common/messageservice"; describe('CheckboxFilterComponent', () => { let component: CheckboxFilterComponent; @@ -36,6 +38,8 @@ describe('CheckboxFilterComponent', () => { providers: [ DataService, ResourceService, + DSMessageService, + MessageService, { provide: AppConfig, useClass: AppConfigMock diff --git a/data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts b/data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts index 8b718ef..50a821c 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/filters.component.spec.ts @@ -12,6 +12,8 @@ import {ResourceService} from "../services/resource.service"; import {AppConfig} from "../config/app.config"; import {AppConfigMock} from "../config/app.config.mock"; import {SearchParserService} from "../services/search-parser.service"; +import {DSMessageService} from "../services/ds-message.service"; +import {MessageService} from "primeng/components/common/messageservice"; describe('FiltersComponent', () => { let component: FiltersComponent; @@ -33,6 +35,8 @@ describe('FiltersComponent', () => { providers: [ DataService, ResourceService, + DSMessageService, + MessageService, { provide: AppConfig, useClass: AppConfigMock diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.spec.ts b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.spec.ts index 3086711..1aec4e9 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.spec.ts @@ -15,6 +15,8 @@ import {HttpModule} from "@angular/http"; import {AppConfig} from "../../config/app.config"; import {AppConfigMock} from "../../config/app.config.mock"; import {SearchParserService} from "../../services/search-parser.service"; +import {DSMessageService} from "../../services/ds-message.service"; +import {MessageService} from "primeng/components/common/messageservice"; describe('TextFilterComponent', () => { let component: TextFilterComponent; @@ -31,6 +33,8 @@ describe('TextFilterComponent', () => { providers: [ DataService, ResourceService, + DSMessageService, + MessageService, { provide: AppConfig, useClass: AppConfigMock diff --git a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.css b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.css index 7697733..ce35e3c 100644 --- a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.css +++ b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.css @@ -28,3 +28,7 @@ th { font-weight: bold; background-color: #e7f3f1; } + +.export-button { + width: auto; +} diff --git a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html index 4489d09..cb22dc0 100644 --- a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html @@ -92,6 +92,11 @@
Patient counts:Subject counts: {{item?.summary.patientCount}}
-
+ + + + diff --git a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.spec.ts b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.spec.ts index b585140..f3c5a84 100644 --- a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.spec.ts @@ -15,6 +15,8 @@ import {ResourceService} from "../services/resource.service"; import {AppConfig} from "../config/app.config"; import {AppConfigMock} from "../config/app.config.mock"; import {BrowserAnimationsModule} from "@angular/platform-browser/animations"; +import {DSMessageService} from "../services/ds-message.service"; +import {MessageService} from "primeng/components/common/messageservice"; describe('ItemSummaryComponent', () => { let component: ItemSummaryComponent; @@ -33,6 +35,8 @@ describe('ItemSummaryComponent', () => { providers: [ DataService, ResourceService, + DSMessageService, + MessageService, { provide: AppConfig, useClass: AppConfigMock diff --git a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts index 2c29683..dc528db 100644 --- a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts @@ -46,4 +46,12 @@ export class ItemSummaryComponent implements OnInit { err => console.error(err) ); } + + addToCart(){ + this.dataService.addToShoppingCart([this.item]); + } + + close() { + this.display = false; + } } diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html index 68d2168..32a9669 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.html @@ -53,7 +53,8 @@
- +
Items selected: {{itemsSelection ? itemsSelection.length : 0}}. Total results in table: {{totalItemsCount()}}. diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts index fc39de0..fff1eec 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.spec.ts @@ -18,6 +18,8 @@ import {ResourceService} from "../services/resource.service"; import {HttpModule} from "@angular/http"; import {AppConfig} from "../config/app.config"; import {AppConfigMock} from "../config/app.config.mock"; +import {DSMessageService} from "../services/ds-message.service"; +import {MessageService} from "primeng/components/common/messageservice"; describe('ItemTableComponent', () => { let component: ItemTableComponent; @@ -42,6 +44,8 @@ describe('ItemTableComponent', () => { providers: [ DataService, ResourceService, + DSMessageService, + MessageService, { provide: AppConfig, useClass: AppConfigMock diff --git a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts index ee94fff..da8775e 100644 --- a/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-table/item-table.component.ts @@ -144,7 +144,7 @@ export class ItemTableComponent implements OnInit { updateCurrentPageItemsSelection(items: Item[]){ if(this.dataService.allItemsSelected) { this.itemsSelectionPerPage = items; - } else { + } else if (this.itemsSelection) { this.itemsSelectionPerPage = items.filter(i => this.itemsSelection.some(is => is.id == i.id)); } } diff --git a/data-showcase/src/main/user-interface/src/app/logos/logos.component.spec.ts b/data-showcase/src/main/user-interface/src/app/logos/logos.component.spec.ts index b59ca36..5756926 100644 --- a/data-showcase/src/main/user-interface/src/app/logos/logos.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/logos/logos.component.spec.ts @@ -12,6 +12,8 @@ import {ResourceService} from "../services/resource.service"; import {AppConfig} from "../config/app.config"; import {AppConfigMock} from "../config/app.config.mock"; import {HttpModule} from "@angular/http"; +import {DSMessageService} from "../services/ds-message.service"; +import {MessageService} from "primeng/components/common/messageservice"; describe('LogosComponent', () => { let component: LogosComponent; @@ -24,6 +26,8 @@ describe('LogosComponent', () => { providers: [ DataService, ResourceService, + DSMessageService, + MessageService, { provide: AppConfig, useClass: AppConfigMock diff --git a/data-showcase/src/main/user-interface/src/app/page-ribbon/page-ribbon.component.spec.ts b/data-showcase/src/main/user-interface/src/app/page-ribbon/page-ribbon.component.spec.ts index b7b0d10..1c34505 100644 --- a/data-showcase/src/main/user-interface/src/app/page-ribbon/page-ribbon.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/page-ribbon/page-ribbon.component.spec.ts @@ -12,6 +12,8 @@ import {AppConfig} from "../config/app.config"; import {AppConfigMock} from "../config/app.config.mock"; import {DataService} from "../services/data.service"; import {HttpModule} from "@angular/http"; +import {MessageService} from "primeng/components/common/messageservice"; +import {DSMessageService} from "../services/ds-message.service"; describe('PageRibbonComponent', () => { let component: PageRibbonComponent; @@ -24,6 +26,8 @@ describe('PageRibbonComponent', () => { providers: [ ResourceService, DataService, + DSMessageService, + MessageService, { provide: AppConfig, useClass: AppConfigMock diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.spec.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.spec.ts index 64aeb58..bcb7510 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.spec.ts @@ -11,6 +11,8 @@ import {ResourceService} from "./resource.service"; import {HttpModule} from "@angular/http"; import {AppConfig} from "../config/app.config"; import {AppConfigMock} from "../config/app.config.mock"; +import {DSMessageService} from "./ds-message.service"; +import {MessageService} from "primeng/components/common/messageservice"; describe('DataService', () => { beforeEach(() => { @@ -19,6 +21,8 @@ describe('DataService', () => { providers: [ DataService, ResourceService, + DSMessageService, + MessageService, { provide: AppConfig, useClass: AppConfigMock diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index fbfee2b..7a5dd25 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -5,7 +5,7 @@ */ import {Injectable} from '@angular/core'; -import { SelectItem, TreeNode as TreeNodeLib } from 'primeng/primeng'; +import {SelectItem, TreeNode as TreeNodeLib} from 'primeng/primeng'; import {ResourceService} from './resource.service'; import {TreeNode} from "../models/tree-node"; import {Item} from "../models/item"; @@ -14,6 +14,7 @@ import {Subject} from "rxjs/Subject"; import {BehaviorSubject} from "rxjs/BehaviorSubject"; import {Environment} from "../models/environment"; import {ItemResponse} from "../models/itemResponse"; +import {DSMessageService} from "./ds-message.service"; type LoadingState = 'loading' | 'complete'; type Order = 'asc' | 'desc'; @@ -34,11 +35,12 @@ export class DataService { // items selected in the itemTable private itemsSelectionSource = new Subject(); public itemsSelection$ = this.itemsSelectionSource.asObservable(); - // items added to the shopping cart - public shoppingCartItems = new BehaviorSubject([]); + //a number of items in the table public totalItemsCount: number = 0; // if the select-all-items checkbox is selected public allItemsSelected: boolean = false; + // items added to the shopping cart + public shoppingCartItems = new BehaviorSubject([]); // text filter input private textFilterInputSource = new Subject(); @@ -93,7 +95,8 @@ export class DataService { private environmentSource = new Subject(); public environment$ = this.environmentSource.asObservable(); - constructor(private resourceService: ResourceService) { + constructor(private resourceService: ResourceService, + private dsMessageService: DSMessageService) { this.fetchAllProjectsAndItems(); this.fetchAllTreeNodes(); this.setEnvironment(); @@ -348,7 +351,7 @@ export class DataService { clearAllFilters() { this.clearCheckboxFilterSelection(); - this.resetTableToTheFirstPage() + this.resetTableToTheFirstPage(); this.setTextFilterInput(''); this.clearItemsSelection(); this.rerenderCheckboxFiltersSource.next(true); @@ -444,14 +447,24 @@ export class DataService { // ------------------------- shopping cart ------------------------- + addToShoppingCart(newItemSelection: Item[]) { + let count: number = 0; let newItems: Item[] = this.shoppingCartItems.getValue(); let itemNames = newItems.map((item) => item.name); for (let item of newItemSelection) { if (!itemNames.includes(item.name)) { newItems.push(item); + count++; } } + if(count > 0) { + this.dsMessageService.addInfoMessage("success", "Shopping cart updated!", count + " item(s) added to the cart.") + } else if(newItemSelection.length > 0) { + this.dsMessageService.addInfoMessage("info", "No item added", "Item(s) already in the shopping cart.") + } else { + this.dsMessageService.addInfoMessage("info", "No item selected", "Select item(s) you want to add to the cart.") + } this.shoppingCartItems.next(newItems); } @@ -459,6 +472,7 @@ export class DataService { this.shoppingCartItems.next(items); } + // ------------------------- item summary ------------------------- displayPopup(item: Item) { diff --git a/data-showcase/src/main/user-interface/src/app/services/ds-message.service.spec.ts b/data-showcase/src/main/user-interface/src/app/services/ds-message.service.spec.ts new file mode 100644 index 0000000..3284ce9 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/services/ds-message.service.spec.ts @@ -0,0 +1,31 @@ +import {TestBed, inject} from '@angular/core/testing'; + +import {DSMessageService} from './ds-message.service'; +import {HttpModule} from "@angular/http"; +import {DataService} from "./data.service"; +import {ResourceService} from "./resource.service"; +import {AppConfig} from "../config/app.config"; +import {AppConfigMock} from "../config/app.config.mock"; +import {MessageService} from "primeng/components/common/messageservice"; + +describe('DSMessageService', () => { + beforeEach(() => { + TestBed.configureTestingModule({ + imports: [HttpModule], + providers: [ + DataService, + DSMessageService, + MessageService, + ResourceService, + { + provide: AppConfig, + useClass: AppConfigMock + } + ] + }); + }); + + it('should be created', inject([DSMessageService], (service: DSMessageService) => { + expect(service).toBeTruthy(); + })); +}); diff --git a/data-showcase/src/main/user-interface/src/app/services/ds-message.service.ts b/data-showcase/src/main/user-interface/src/app/services/ds-message.service.ts new file mode 100644 index 0000000..343fb14 --- /dev/null +++ b/data-showcase/src/main/user-interface/src/app/services/ds-message.service.ts @@ -0,0 +1,28 @@ +import {Component, Injectable} from '@angular/core'; +import {Message} from 'primeng/components/common/api'; +import {MessageService} from 'primeng/components/common/messageservice'; + +@Component({ + template: '' +}) +@Injectable() +export class DSMessageService { + + constructor(private messageService: MessageService) {} + + addInfoMessage(type: string, infoMsg: string, detail: string) { + this.messageService.add({severity: type, summary:infoMsg, detail:detail}); + } + addSingle() { + this.messageService.add({severity:'success', summary:'Service Message', detail:'Via MessageService'}); + } + + addMultiple() { + this.messageService.addAll([{severity:'success', summary:'Service Message', detail:'Via MessageService'}, + {severity:'info', summary:'Info Message', detail:'Via MessageService'}]); + } + + clear() { + this.messageService.clear(); + } +} diff --git a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.spec.ts b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.spec.ts index 7aa761a..380470b 100644 --- a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.spec.ts @@ -8,7 +8,7 @@ import { async, ComponentFixture, TestBed } from '@angular/core/testing'; import { ShoppingCartComponent } from './shopping-cart.component'; import {FormsModule} from "@angular/forms"; -import {DataTableModule, DialogModule, PanelModule} from "primeng/primeng"; +import {DataTableModule, DialogModule, GrowlModule, PanelModule} from "primeng/primeng"; import {DataService} from "../services/data.service"; import {ResourceService} from "../services/resource.service"; import {AppConfig} from "../config/app.config"; @@ -17,6 +17,8 @@ import {initConfig} from "../app.module"; import {HttpModule} from "@angular/http"; import {AppConfigMock} from "../config/app.config.mock"; import {BrowserAnimationsModule} from "@angular/platform-browser/animations"; +import {DSMessageService} from "../services/ds-message.service"; +import {MessageService} from "primeng/components/common/messageservice"; describe('ShoppingCartComponent', () => { let component: ShoppingCartComponent; @@ -36,6 +38,8 @@ describe('ShoppingCartComponent', () => { providers: [ DataService, ResourceService, + DSMessageService, + MessageService, { provide: AppConfig, useClass: AppConfigMock diff --git a/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.spec.ts b/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.spec.ts index 7daa770..ae68de6 100644 --- a/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/tree-nodes/tree-nodes.component.spec.ts @@ -16,6 +16,8 @@ import {BrowserAnimationsModule} from "@angular/platform-browser/animations"; import {HttpModule} from "@angular/http"; import {AppConfig} from "../config/app.config"; import {AppConfigMock} from "../config/app.config.mock"; +import {DSMessageService} from "../services/ds-message.service"; +import {MessageService} from "primeng/components/common/messageservice"; describe('TreeNodesComponent', () => { let component: TreeNodesComponent; @@ -36,6 +38,8 @@ describe('TreeNodesComponent', () => { providers: [ DataService, ResourceService, + DSMessageService, + MessageService, { provide: AppConfig, useClass: AppConfigMock From 34dca50d7a898ad76416ba0c0a01968ab438f8cd Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Tue, 28 Nov 2017 14:15:26 +0100 Subject: [PATCH 092/104] Add missing MessageService provider --- data-showcase/src/main/user-interface/src/app/app.module.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/data-showcase/src/main/user-interface/src/app/app.module.ts b/data-showcase/src/main/user-interface/src/app/app.module.ts index 71e0faf..cfab087 100644 --- a/data-showcase/src/main/user-interface/src/app/app.module.ts +++ b/data-showcase/src/main/user-interface/src/app/app.module.ts @@ -25,6 +25,7 @@ import {InfoModule} from "./info/info.module"; import {SearchParserService} from "./services/search-parser.service"; import {GrowlModule} from "primeng/primeng"; import {DSMessageService} from "./services/ds-message.service"; +import {MessageService} from "primeng/components/common/messageservice"; export function initConfig(config: AppConfig) { return () => config.load() @@ -54,6 +55,7 @@ export function initConfig(config: AppConfig) { DataService, SearchParserService, DSMessageService, + MessageService, AppConfig, { provide: APP_INITIALIZER, From 4df1bb99b27717657c29fdcfb46e75985aac780a Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Tue, 28 Nov 2017 16:36:10 +0100 Subject: [PATCH 093/104] Add filter labels --- .../checkbox-filter/checkbox-filter.component.css | 2 +- .../checkbox-filter/checkbox-filter.component.html | 4 ++-- .../filters/text-filter/text-filter.component.css | 12 ++++++++++-- .../filters/text-filter/text-filter.component.html | 3 +++ .../src/app/tree-nodes/tree-nodes.component.css | 5 +++++ .../src/app/tree-nodes/tree-nodes.component.html | 2 +- data-showcase/src/main/user-interface/src/styles.css | 3 +-- 7 files changed, 23 insertions(+), 8 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.css b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.css index d9dabcb..f513fd4 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.css +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.css @@ -8,6 +8,6 @@ background-color: transparent; border: none; /* padding: top right bottom left */ - padding: 2px 6px 2px 25px; + padding: 0 6px 0 25px; } diff --git a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html index 5b58497..8caeda6 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html +++ b/data-showcase/src/main/user-interface/src/app/filters/checkbox-filter/checkbox-filter.component.html @@ -9,14 +9,14 @@
- +
- + diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css index 2481ad8..3151011 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.css @@ -6,12 +6,20 @@ .text-filter-container { text-align: center; - height: 50px; - line-height: 50px; + height: 65px; + line-height: 40px; white-space: nowrap; + display: block; + padding-bottom: 5px; + border-bottom: 1px solid lightgray; } .text-filter-container:focus {outline:none;} +.search-label{ + line-height: 20px; + color: #00889C; +} + .search-error-message { color: red; text-align: center; diff --git a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html index 311a974..59d649a 100644 --- a/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html +++ b/data-showcase/src/main/user-interface/src/app/filters/text-filter/text-filter.component.html @@ -5,6 +5,9 @@ -->
+
+ Search by keywords, item name, item labels: +
- +

Select domain

diff --git a/data-showcase/src/main/user-interface/src/styles.css b/data-showcase/src/main/user-interface/src/styles.css index bca6d3e..c0f8a59 100644 --- a/data-showcase/src/main/user-interface/src/styles.css +++ b/data-showcase/src/main/user-interface/src/styles.css @@ -45,8 +45,7 @@ body .ds-tree-container { */ .ds-tree-filter-input { border-bottom: 1px solid darkgray; - padding-top: 10px; - height: 50px; + height: 40px; font-size: small !important; } .ds-tree-filter-input > input{ From 2a3f28145cc1318e5c8864cfc509d687bb78197d Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Tue, 28 Nov 2017 17:24:50 +0100 Subject: [PATCH 094/104] Fix displaying of update messages --- data-showcase/src/main/user-interface/src/app/app.component.html | 1 + 1 file changed, 1 insertion(+) diff --git a/data-showcase/src/main/user-interface/src/app/app.component.html b/data-showcase/src/main/user-interface/src/app/app.component.html index 187181d..868238f 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.html +++ b/data-showcase/src/main/user-interface/src/app/app.component.html @@ -22,6 +22,7 @@
+ From eee597d2fa7a02ff3e7063d028eac8f68cf4096f Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 1 Dec 2017 11:41:55 +0100 Subject: [PATCH 095/104] Fix UI tests when back end is unavailable. --- .../main/user-interface/src/app/services/resource.service.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts index 4713be6..b0e3bce 100644 --- a/data-showcase/src/main/user-interface/src/app/services/resource.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/resource.service.ts @@ -46,6 +46,10 @@ export class ResourceService { } else { errMsg = `Error: ${error.statusText}`; } + if (error.status in [0, 404]) { + console.error('Server not available.'); + return Observable.never(); + } } else { errMsg = error.message ? error.message : error.toString(); } From 6a92697812a562d996654b83e76f42f6333b0080 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Fri, 1 Dec 2017 11:49:19 +0100 Subject: [PATCH 096/104] Move notification logic to App component. When building the application with ng build --prod, two issues appeared: the @Component annotation on the DSMessageService, and the missing msgs field in the App component. This commit fixes both issues. --- .../user-interface/src/app/app.component.ts | 14 ++++++++-- .../app/services/ds-message.service.spec.ts | 6 ---- .../src/app/services/ds-message.service.ts | 28 ++++++------------- 3 files changed, 21 insertions(+), 27 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/app.component.ts b/data-showcase/src/main/user-interface/src/app/app.component.ts index 5da6cdb..e4f99d1 100644 --- a/data-showcase/src/main/user-interface/src/app/app.component.ts +++ b/data-showcase/src/main/user-interface/src/app/app.component.ts @@ -5,11 +5,15 @@ */ import {Component, OnInit, ViewChild} from '@angular/core'; +import { Message } from 'primeng/primeng'; +import { MessageService } from 'primeng/components/common/messageservice'; +import { DSMessageService } from './services/ds-message.service'; @Component({ selector: 'app-root', templateUrl: './app.component.html', - styleUrls: ['./app.component.css'] + styleUrls: ['./app.component.css'], + providers: [MessageService] }) export class AppComponent implements OnInit { @@ -22,7 +26,9 @@ export class AppComponent implements OnInit { private x_pos: number; // Stores x coordinate of the mouse pointer private x_gap: number; // Stores x gap (edge) between mouse and gutter - constructor() { + msgs: Message[] = []; + + constructor(private messageService: MessageService, private dsMessageService: DSMessageService) { } ngOnInit() { @@ -64,5 +70,9 @@ export class AppComponent implements OnInit { gutterElm.addEventListener('mousedown', onMouseDown.bind(this)); parentContainerElm.addEventListener('mousemove', onMouseMove.bind(this)); parentContainerElm.addEventListener('mouseup', onMouseUp.bind(this)); + + this.dsMessageService.messages.subscribe(message => { + this.messageService.add(message); + }); } } diff --git a/data-showcase/src/main/user-interface/src/app/services/ds-message.service.spec.ts b/data-showcase/src/main/user-interface/src/app/services/ds-message.service.spec.ts index 3284ce9..2b200f1 100644 --- a/data-showcase/src/main/user-interface/src/app/services/ds-message.service.spec.ts +++ b/data-showcase/src/main/user-interface/src/app/services/ds-message.service.spec.ts @@ -2,21 +2,15 @@ import {TestBed, inject} from '@angular/core/testing'; import {DSMessageService} from './ds-message.service'; import {HttpModule} from "@angular/http"; -import {DataService} from "./data.service"; -import {ResourceService} from "./resource.service"; import {AppConfig} from "../config/app.config"; import {AppConfigMock} from "../config/app.config.mock"; -import {MessageService} from "primeng/components/common/messageservice"; describe('DSMessageService', () => { beforeEach(() => { TestBed.configureTestingModule({ imports: [HttpModule], providers: [ - DataService, DSMessageService, - MessageService, - ResourceService, { provide: AppConfig, useClass: AppConfigMock diff --git a/data-showcase/src/main/user-interface/src/app/services/ds-message.service.ts b/data-showcase/src/main/user-interface/src/app/services/ds-message.service.ts index 343fb14..1c4567e 100644 --- a/data-showcase/src/main/user-interface/src/app/services/ds-message.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/ds-message.service.ts @@ -1,28 +1,18 @@ -import {Component, Injectable} from '@angular/core'; -import {Message} from 'primeng/components/common/api'; -import {MessageService} from 'primeng/components/common/messageservice'; +import {Injectable} from '@angular/core'; +import { Message } from 'primeng/primeng'; +import { Subject } from 'rxjs/Subject'; + +export type MessageType = 'success' | 'info' | 'warn' | 'error'; -@Component({ - template: '' -}) @Injectable() export class DSMessageService { - constructor(private messageService: MessageService) {} + messages = new Subject(); - addInfoMessage(type: string, infoMsg: string, detail: string) { - this.messageService.add({severity: type, summary:infoMsg, detail:detail}); - } - addSingle() { - this.messageService.add({severity:'success', summary:'Service Message', detail:'Via MessageService'}); - } + constructor() {} - addMultiple() { - this.messageService.addAll([{severity:'success', summary:'Service Message', detail:'Via MessageService'}, - {severity:'info', summary:'Info Message', detail:'Via MessageService'}]); + addInfoMessage(type: MessageType, infoMsg: string, detail: string) { + this.messages.next({severity: type, summary: infoMsg, detail:detail}); } - clear() { - this.messageService.clear(); - } } From 4d6c29b1a644e06c9a941c27a0b1e68192386f3e Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 29 Nov 2017 10:04:36 +0100 Subject: [PATCH 097/104] Fix fetching all filters and items Fix for NTRDEV-234: Search options are shown event if there is no item loaded --- .../src/app/services/data.service.ts | 21 +++---------------- 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 7a5dd25..8a4ae8d 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -97,8 +97,9 @@ export class DataService { constructor(private resourceService: ResourceService, private dsMessageService: DSMessageService) { - this.fetchAllProjectsAndItems(); - this.fetchAllTreeNodes(); + this.fetchFilters(); + this.fetchItems(); + this.fetchAllTreeNodes() this.setEnvironment(); } @@ -180,22 +181,6 @@ export class DataService { // ------------------------- filters and item table ------------------------- - fetchAllProjectsAndItems() { - this.resourceService.getAllProjects() - .subscribe( - (projects: Project[]) => { - this.allProjects = projects; - this.fetchItems(); - for (let project of projects) { - this.projects.push({label: project.name, value: project.name}); - DataService.collectUnique(project.lineOfResearch, this.linesOfResearch); - } - this.sortLinesOfResearch(); - }, - err => console.error(err) - ); - } - projectToResearchLine(projectName: string): string { if (this.allProjects) { return this.allProjects.find(p => p.name == projectName).lineOfResearch; From 83326aee0455f84ae2392ef1ebcae92c81d40b65 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Wed, 29 Nov 2017 10:25:37 +0100 Subject: [PATCH 098/104] Fix adding to cart when selecting all items at once Fix NTRDEV-233: "NTRUAT-48: Shopping cart: Research line is not shown when selecting items all at once" --- .../src/main/user-interface/src/app/services/data.service.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/data-showcase/src/main/user-interface/src/app/services/data.service.ts b/data-showcase/src/main/user-interface/src/app/services/data.service.ts index 8a4ae8d..af46694 100644 --- a/data-showcase/src/main/user-interface/src/app/services/data.service.ts +++ b/data-showcase/src/main/user-interface/src/app/services/data.service.ts @@ -266,6 +266,11 @@ export class DataService { this.resourceService.getItems(firstResult, null, null, null, codes, projects, this.searchQuery).subscribe( (response: ItemResponse) => { + for (let item of response.items) { + if (this.allProjects && this.allProjects.length > 0) { + item.lineOfResearch = this.projectToResearchLine(item.project); + } + } this.itemsSelectionSource.next(response.items); }, err => { From 75240530ab93f9a65cab7add3547d0d4b5f8f95e Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Thu, 30 Nov 2017 09:47:38 +0100 Subject: [PATCH 099/104] Prevent page change when sorting in the shoppingCart Fix NTRDEV-236 --- .../src/app/shopping-cart/shopping-cart.component.html | 4 +++- .../src/app/shopping-cart/shopping-cart.component.ts | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html index b42df6f..e62ec7c 100644 --- a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html +++ b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.html @@ -14,7 +14,9 @@ [immutable]=false [value]="items" [style]="table-content" - [rows]="10" [paginator]="true"> + [rows]="rowsPerPage" [paginator]="true" + (onSort)="changeSort($event)" + [(first)]="firstOnPage"> diff --git a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.ts b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.ts index 3f123a1..4252da8 100644 --- a/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.ts +++ b/data-showcase/src/main/user-interface/src/app/shopping-cart/shopping-cart.component.ts @@ -4,10 +4,11 @@ * (see accompanying file LICENSE). */ -import { Component, OnInit } from '@angular/core'; +import {Component, OnInit, ViewChild} from '@angular/core'; import {DataService} from "../services/data.service"; import {Item} from "../models/item"; import {saveAs} from "file-saver"; +import {DataTable} from "primeng/primeng"; @Component({ selector: 'app-shopping-cart', @@ -16,11 +17,14 @@ import {saveAs} from "file-saver"; }) export class ShoppingCartComponent implements OnInit { + @ViewChild('itemTable') dataTable: DataTable; display: boolean = false; items: Item[] = []; pathSelection: string[] = []; fileName: string; disabled: boolean = true; + firstOnPage: number = 0; + rowsPerPage: number = 10; constructor(private dataService: DataService) { dataService.shoppingCartItems.subscribe( @@ -40,6 +44,10 @@ export class ShoppingCartComponent implements OnInit { this.display = true; } + changeSort(event) { + this.dataTable['first'] = this.firstOnPage ; + } + deleteItem(itemToDelete: Item) { this.items = this.items.filter(item => item !== itemToDelete); this.dataService.setShoppingCartItems(this.items); From e8580550a8a09aca0e0b69939471c82939ab16fd Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 4 Dec 2017 10:13:59 +0100 Subject: [PATCH 100/104] Add link to source code in the help dialog. --- .../src/app/info/info.component.css | 4 ++++ .../src/app/info/info.component.html | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/info/info.component.css b/data-showcase/src/main/user-interface/src/app/info/info.component.css index d5bc335..58c26cf 100644 --- a/data-showcase/src/main/user-interface/src/app/info/info.component.css +++ b/data-showcase/src/main/user-interface/src/app/info/info.component.css @@ -7,6 +7,10 @@ .info-header { font-weight:normal; + margin-top: .7em; +} + +.info-title { color: #00889C; } diff --git a/data-showcase/src/main/user-interface/src/app/info/info.component.html b/data-showcase/src/main/user-interface/src/app/info/info.component.html index d8f6fb5..5493b5a 100644 --- a/data-showcase/src/main/user-interface/src/app/info/info.component.html +++ b/data-showcase/src/main/user-interface/src/app/info/info.component.html @@ -6,11 +6,13 @@ -

Help information

-

General:

+

Help information

+

General

+

General information about the application. -

-

Searching:

+

+ +

Search

General structure of the query is: @@ -74,7 +76,13 @@

Searching:


e.g., ((name = "value1") OR (name = "value2")) AND (NOT labelLong = "value3") -
+ +

About

+

+ This application is open source software. + The source code is available + under the terms of the GNU Affero General Public License. +

From 368567bebb424833e74b37777e356d29d8301cc7 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 8 Jan 2018 11:26:29 +0100 Subject: [PATCH 101/104] Fix item sumary fields visibility for Public environment NTRDEV-261 --- .../item-summary/item-summary.component.html | 74 ++++++++++--------- .../item-summary/item-summary.component.ts | 9 +++ 2 files changed, 47 insertions(+), 36 deletions(-) diff --git a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html index cb22dc0..199c79f 100644 --- a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html +++ b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.html @@ -57,42 +57,44 @@
-
- - - - - - - - - - - - - - - - - - -
Average:{{item?.summary.avgValue}}
Min:{{item?.summary.minValue}}
Max:{{item?.summary.maxValue}}
Standard deviation:{{item?.summary.stdDevValue}}
- - - - - - - - - - - - - -
Value:Frequency:Label:
{{value.value}}{{value.frequency}}{{value.label}}
- -
+
+
+ + + + + + + + + + + + + + + + + + +
Average:{{item?.summary.avgValue}}
Min:{{item?.summary.minValue}}
Max:{{item?.summary.maxValue}}
Standard deviation:{{item?.summary.stdDevValue}}
+ + + + + + + + + + + + + +
Value:Frequency:Label:
{{value.value}}{{value.frequency}}{{value.label}}
+ +
+
diff --git a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts index dc528db..31c120c 100644 --- a/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts +++ b/data-showcase/src/main/user-interface/src/app/item-summary/item-summary.component.ts @@ -8,6 +8,7 @@ import { Component, OnInit } from '@angular/core'; import {DataService} from "../services/data.service"; import {Item} from "../models/item"; import {ResourceService} from "../services/resource.service"; +import {Environment} from "../models/environment"; @Component({ selector: 'app-item-summary', @@ -19,9 +20,14 @@ export class ItemSummaryComponent implements OnInit { display: boolean = false; item: Item = null; keywordsForConcept: string[] = []; + environment: Environment; constructor(private dataService: DataService, private resourceService: ResourceService) { + dataService.environment$.subscribe( + environment => { + this.environment = environment; + }); dataService.itemSummaryVisible$.subscribe( visibleItem => { this.display = true; @@ -35,6 +41,9 @@ export class ItemSummaryComponent implements OnInit { ngOnInit() { } + isInternal(): boolean { + return this.environment && this.environment.environment == "Internal"; + } fetchKeywords(conceptCode: string) { this.keywordsForConcept = []; From fd97ef11195357702ba24dc5d3aa8c35234b2ca0 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 8 Jan 2018 14:57:18 +0100 Subject: [PATCH 102/104] Sort summary values by frequency NTRDEV-259 --- .../grails-app/domain/nl/thehyve/datashowcase/Summary.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy index d3e94fc..f567f53 100644 --- a/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy +++ b/data-showcase/grails-app/domain/nl/thehyve/datashowcase/Summary.groovy @@ -65,7 +65,7 @@ class Summary { static mapping = { version false - values batchSize: 1000 + values batchSize: 1000, sort: 'frequency', order:'desc' } static constraints = { From 48ca44d37557a62761b5737140c42ad466ea5850 Mon Sep 17 00:00:00 2001 From: Ewelina Grudzien Date: Mon, 8 Jan 2018 15:32:16 +0100 Subject: [PATCH 103/104] Fix a search field issue: operators 'like' and '=' seem to do the same NTRDEV-266 'ilike' hibernate restriction with match mode = "Exact" is accepting wildcards. Replaced with 'eq' restriction with ignoreCase() method. --- .../thehyve/datashowcase/search/SearchCriteriaBuilder.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy index 8fa1794..9d59be5 100644 --- a/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy +++ b/data-showcase/src/main/groovy/nl/thehyve/datashowcase/search/SearchCriteriaBuilder.groovy @@ -56,9 +56,9 @@ class SearchCriteriaBuilder { case Operator.CONTAINS: return Restrictions.ilike(propertyName, value as String, MatchMode.ANYWHERE) case Operator.EQUALS: - return Restrictions.ilike(propertyName, value as String, MatchMode.EXACT) + return Restrictions.eq(propertyName, value as String).ignoreCase() case Operator.NOT_EQUALS: - return Restrictions.not(Restrictions.ilike(propertyName, value as String, MatchMode.EXACT)) + return Restrictions.not(Restrictions.eq(propertyName, value as String).ignoreCase()) case Operator.LIKE: return Restrictions.ilike(propertyName, value as String) default: From 1bf0c523c10460097fac7396329d2e0a8312ac95 Mon Sep 17 00:00:00 2001 From: Gijs Kant Date: Mon, 8 Jan 2018 15:40:36 +0100 Subject: [PATCH 104/104] Prepare release 1.0.0. --- README.md | 8 ++++---- data-showcase/build.gradle | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 730877d..8a4583a 100644 --- a/README.md +++ b/README.md @@ -34,11 +34,11 @@ gradle wrapper cd src/main/user-interface npm start -# Package, creates build/lib/data-showcase-0.0.1-SNAPSHOT.war +# Package, creates build/lib/data-showcase-1.0.0.war ./gradlew assemble # Run the war -java -jar build/libs/data-showcase-0.0.1-SNAPSHOT.war +java -jar build/libs/data-showcase-1.0.0.war ``` ## Publish @@ -53,7 +53,7 @@ Publish to Nexus: # Deploy to the Nexus repository https://repo.thehyve.nl ./gradlew publish ``` -This publishes the artifact `nl.thehyve:data-showcase:0.0.1-SNAPSHOT:war`. +This publishes the artifact `nl.thehyve:data-showcase:1.0.0:war`. ## Deploy For deployment, fetch the application war file from the Nexus repository, @@ -62,7 +62,7 @@ and run the application with an external config file: MEMORY_OPTIONS="-Xms2g -Xmx2g -XX:MaxPermSize=512m" JAVA_OPTIONS="-server -Djava.awt.headless=true -Djava.security.egd=file:/dev/./urandom" APP_OPTIONS="-Dgrails.env=prod -Dspring.config.location=/home/user/data-showcase-internal.yml" -java -jar ${JAVA_OPTIONS} ${MEMORY_OPTIONS} ${APP_OPTIONS} data-showcase-0.0.1-SNAPHOT.war +java -jar ${JAVA_OPTIONS} ${MEMORY_OPTIONS} ${APP_OPTIONS} data-showcase-1.0.0.war ``` Example configuration file `data-showcase-internal.yml`: diff --git a/data-showcase/build.gradle b/data-showcase/build.gradle index 4c90c19..b18ebb0 100644 --- a/data-showcase/build.gradle +++ b/data-showcase/build.gradle @@ -13,7 +13,7 @@ buildscript { } ext { - dataShowcaseVersion = '0.0.1-SNAPSHOT' + dataShowcaseVersion = '1.0.0' nodeVersion = '7.10.1' modelMapperVersion = '1.1.0' }