Skip to content

Commit

Permalink
Delete private projects from search index #979
Browse files Browse the repository at this point in the history
  • Loading branch information
@chrisala
chrisala committed Sep 11, 2024
1 parent 2051935 commit 6e63a09
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 33 deletions.
6 changes: 5 additions & 1 deletion grails-app/domain/au/org/ala/ecodata/Project.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,10 @@ class Project {

List<OutputTarget> outputTargets

static embedded = ['associatedOrgs', 'fundings', 'mapLayersConfig', 'risks', 'geographicInfo', 'externalIds', 'outputTargets']
/** Container to allow program config overrides for an individual Project */
Map config

static embedded = ['associatedOrgs', 'fundings', 'mapLayersConfig', 'config', 'risks', 'geographicInfo', 'externalIds', 'outputTargets']

static transients = ['activities', 'plannedDurationInWeeks', 'actualDurationInWeeks', 'tempArgs', 'monitoringProtocolCategories']

Expand Down Expand Up @@ -234,6 +237,7 @@ class Project {
isBushfire nullable: true
bushfireCategories nullable: true
mapLayersConfig nullable: true
config nullable: true
managementUnitId nullable: true
mapDisplays nullable: true
terminationReason nullable: true
Expand Down
54 changes: 29 additions & 25 deletions grails-app/services/au/org/ala/ecodata/ElasticSearchService.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,17 +180,18 @@ class ElasticSearchService {
* @return IndexResponse
*/
def indexDoc(doc, String index, BulkProcessor bulkProcessor = null) {
String docId = getEntityId(doc)
if (!canIndex(doc)) {
deleteIfRequired(docId, index)
return
}
String docId = getEntityId(doc)

// The purpose of the as JSON call below is to convert Date objects into the format we use
// throughout the app - otherwise the elasticsearch XContentBuilder will transform them into
// ISO dates with milliseconds which causes BioCollect problems as it uses the _source field of the
// search result directly.
Map docMap = doc


index = index ?: DEFAULT_INDEX

// Delete index if it exists and doc.status == 'deleted'
Expand Down Expand Up @@ -257,14 +258,24 @@ class ElasticSearchService {
* @return isDeleted (Boolean)
*/
def checkForDelete(doc, docId, String index = DEFAULT_INDEX) {
def isDeleted = false
if (doc.status?.toLowerCase() == DELETED) {
isDeleted = deleteIfRequired(docId, index)
}

return isDeleted
}

/** Deletes the document with the supplied id from elasticsearch if it is indexed */
boolean deleteIfRequired(String docId, String index) {
def isDeleted = false
GetResponse resp

try {
GetRequest request = new GetRequest(index, docId)
resp = client.get(request, RequestOptions.DEFAULT)

if (resp.exists && doc.status?.toLowerCase() == DELETED) {
if (resp.exists) {
try {
deleteDocById(docId, index)
isDeleted = true
Expand Down Expand Up @@ -549,15 +560,9 @@ class ElasticSearchService {

switch (docType) {
case Project.class.name:
def doc = Project.findByProjectId(docId)
def projectMap = projectService.toMap(doc, "flat")
projectMap["className"] = docType
indexHomePage(doc, docType)
if(projectMap.siteId){
indexDocType(projectMap.siteId, Site.class.name)
}

break;
Project project = Project.findByProjectId(docId)
indexHomePage(project, docType)
break
case Site.class.name:
def doc = Site.findBySiteId(docId)
def siteMap = siteService.toMap(doc, SiteService.FLAT)
Expand All @@ -568,11 +573,10 @@ class ElasticSearchService {
}

doc?.projects?.each { projectId ->
def proj = Project.findByProjectId(projectId)
Map projectMap = prepareProjectForHomePageIndex(proj)
indexDoc(projectMap, HOMEPAGE_INDEX)
Project proj = Project.findByProjectId(projectId)
indexHomePage(proj, Project.class.name)
}
break;
break

case Record.class.name:
Record record = Record.findByOccurrenceID(docId)
Expand Down Expand Up @@ -605,7 +609,7 @@ class ElasticSearchService {
// update linked project -- index for homepage
def pDoc = Project.findByProjectId(doc.projectId)
if (pDoc) {
indexHomePage(pDoc, "au.org.ala.ecodata.Project")
indexHomePage(pDoc, Project.class.name)
}

if(activity.siteId){
Expand All @@ -631,8 +635,6 @@ class ElasticSearchService {
String projectId = UserPermission.findByIdAndEntityType(docId, Project.class.name)?.getEntityId()
if (projectId) {
Project doc = Project.findByProjectId(projectId)
Map projectMap = projectService.toMap(doc, "flat")
projectMap["className"] = Project.class.name
indexHomePage(doc, Project.class.name)
}
break
Expand Down Expand Up @@ -700,11 +702,10 @@ class ElasticSearchService {
try {
def docId = getEntityId(doc)

// Delete index if it exists and doc.status == 'deleted'
checkForDelete(doc, docId, HOMEPAGE_INDEX)

// Prevent deleted document from been indexed regardless of whether it has a previous index entry
if(doc.status?.toLowerCase() == DELETED) {
// Delete index if it exists and doc.status == 'deleted'
checkForDelete(doc, docId, HOMEPAGE_INDEX)
return null;
}

Expand Down Expand Up @@ -1054,7 +1055,7 @@ class ElasticSearchService {
*/
private Map prepareProjectForHomePageIndex(Project project) {
def projectMap = projectService.toMap(project, ProjectService.FLAT)
projectMap["className"] = new Project().getClass().name
projectMap["className"] = Project.class.name
// MERIT project needs private sites to be indexed for faceting purposes but Biocollect does not require private sites.
// Some Biocollect project have huge numbers of private sites. This will significantly hurt performance.
// Hence the if condition.
Expand Down Expand Up @@ -1125,7 +1126,10 @@ class ElasticSearchService {
if(projectMap.managementUnitId)
projectMap.managementUnitName = managementUnitService.get(projectMap.managementUnitId)?.name

// Populate program facets from the project program, if available
// Populate program facets from the project program, if available, do visibility check
if (project.config?.visibility) {
projectMap.visibility = project.config.visibility
}
if (project.programId) {
Program program = programService.get(project.programId)
if (program) {
Expand All @@ -1137,7 +1141,7 @@ class ElasticSearchService {
}
// This allows all projects associated with a particular program to be excluded from indexing.
// This is required to allow MERIT projects to be loaded before they have been announced.
if (program.inheritedConfig?.visibility) {
if (!projectMap.visibility && program.inheritedConfig?.visibility) {
projectMap.visibility = program.inheritedConfig.visibility
}
}
Expand Down
26 changes: 19 additions & 7 deletions src/test/groovy/au/org/ala/ecodata/ElasticSearchIndexServiceSpec.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,6 @@ class ElasticSearchIndexServiceSpec extends MongoSpec implements ServiceUnitTest
service.indexHomePage(project, Project.class.name)

then:
2 * client.get({GetRequest get -> get.index() == ElasticIndex.HOMEPAGE_INDEX && get.id() == projectProps.projectId}, RequestOptions.DEFAULT) >> Mock(GetResponse)
1 * client.index({ IndexRequest index -> index.index() == ElasticIndex.HOMEPAGE_INDEX && index.id() == projectProps.projectId}, RequestOptions.DEFAULT) >>
{ index, options -> result = new JsonSlurper().parseText(index.source().utf8ToString()); Mock(IndexResponse) }
1 * projectService.toMap(project, ProjectService.FLAT) >> projectProps
Expand Down Expand Up @@ -394,7 +393,6 @@ class ElasticSearchIndexServiceSpec extends MongoSpec implements ServiceUnitTest
service.indexHomePage(meritProject, Project.class.name)

then:
2 * client.get({GetRequest get -> get.index() == ElasticIndex.HOMEPAGE_INDEX && get.id() == meritProjectProps.projectId}, RequestOptions.DEFAULT) >> Mock(GetResponse)
1 * client.index({ IndexRequest index -> index.index() == ElasticIndex.HOMEPAGE_INDEX && index.id() == meritProjectProps.projectId}, RequestOptions.DEFAULT) >>
{ index, options -> meritResult = new JsonSlurper().parseText(index.source().utf8ToString()); Mock(IndexResponse) }
1 * projectService.toMap(meritProject, ProjectService.FLAT) >> meritProjectProps
Expand All @@ -411,7 +409,6 @@ class ElasticSearchIndexServiceSpec extends MongoSpec implements ServiceUnitTest
service.indexHomePage(biocollectProject, Project.class.name)

then:
2 * client.get({GetRequest get -> get.index() == ElasticIndex.HOMEPAGE_INDEX && get.id() == biocollectProjectProps.projectId}, RequestOptions.DEFAULT) >> Mock(GetResponse)
1 * client.index({ IndexRequest index -> index.index() == ElasticIndex.HOMEPAGE_INDEX && index.id() == biocollectProjectProps.projectId}, RequestOptions.DEFAULT) >>
{ index, options -> biocollectResult = new JsonSlurper().parseText(index.source().utf8ToString()); Mock(IndexResponse) }
1 * projectService.toMap(biocollectProject, ProjectService.FLAT) >> biocollectProjectProps
Expand Down Expand Up @@ -449,7 +446,6 @@ class ElasticSearchIndexServiceSpec extends MongoSpec implements ServiceUnitTest

then: "It will be indexed"
1 * siteService.toMap(_, SiteService.FLAT) >> [siteId:"site1", projects:[biocollectProject.projectId]]
3 * client.get(_, _) >> Mock(GetResponse)
1 * client.index({IndexRequest index ->
index.index() == ElasticIndex.DEFAULT_INDEX && index.id() == 'site1'}, RequestOptions.DEFAULT) >> Mock(IndexResponse)
1 * projectService.toMap(biocollectProject, ProjectService.FLAT) >> biocollectProjectProps
Expand All @@ -460,8 +456,8 @@ class ElasticSearchIndexServiceSpec extends MongoSpec implements ServiceUnitTest

then: "It will be indexed"
1 * siteService.toMap(_, SiteService.FLAT) >> [siteId:"site1", projects:[biocollectProject.projectId, meritProject.projectId]]
2 * client.get(_, _) >> Mock(GetResponse)
1 * client.index(_,_) >> Mock(IndexResponse)
1 * projectService.toMap(biocollectProject, ProjectService.FLAT) >> biocollectProjectProps
2 * client.index(_,_) >> Mock(IndexResponse)

}

Expand All @@ -481,7 +477,23 @@ class ElasticSearchIndexServiceSpec extends MongoSpec implements ServiceUnitTest
1 * activityService.toMap(_, ActivityService.FLAT) >> [activityId:"act1", projectId:worksProject.projectId, status: 'active']
1 * projectService.toMap (_, ProjectService.FLAT) >> [projectId:'p1', name:"project 1", isMERIT:false, isWorks: true]

3 * client.get(_, _) >> Mock(GetResponse)
2 * client.index(_,_) >> Mock(IndexResponse)
}

void "Projects marked as visibility:private will be deleted from the search index"() {
setup:
Map meritProjectProps = [projectId:'p1', name:"project 1", isMERIT:true, config:[visibility: 'private']]
Project project = new Project(meritProjectProps)
GetResponse getResponse = Mock(GetResponse)

when:
service.indexHomePage(project, Project.class.name)

then:
1 * projectService.toMap(project, ProjectService.FLAT) >> meritProjectProps

1 * client.get(_, _) >> getResponse
1 * getResponse.exists >> true
1 * client.delete(_, _)
}
}

0 comments on commit 6e63a09

Please sign in to comment.