Merge pull request #938 from AtlasOfLivingAustralia/feature/issue929

site coverage
AtlasOfLivingAustralia · Jun 5, 2024 · 1a610cc · 1a610cc
2 parents 9af965b + e96522c
commit 1a610cc
Show file tree

Hide file tree

Showing 14 changed files with 424 additions and 36 deletions.
diff --git a/grails-app/conf/application.groovy b/grails-app/conf/application.groovy
@@ -427,6 +427,8 @@ if(!spatial.geoJsonEnvelopeConversionThreshold){
     spatial.geoJsonEnvelopeConversionThreshold = 1_000_000
 }
 
+spatial.intersectionThreshold = 0.05
+
 homepageIdx {
     elasticsearch   {
         fieldsAndBoosts {
@@ -495,6 +497,13 @@ app {
     }
 }
 
+site.check.boundary.layers = [
+        'cl927',
+        'cl10946',
+        'cl10921',
+        'cl2112'
+]
+
 /******************************************************************************\
  *  EXTERNAL SERVERS
  \******************************************************************************/
@@ -551,29 +560,12 @@ if (!grails.cache.ehcache) {
             enabled = true
             ehcache {
                 cacheManagerName = appName + '-ehcache'
-                reloadable = true
-                diskStore = '/data/${appName}/ehcache'
+                diskStore = "/data/${appName}/ehcache"
+                ehcacheXmlLocation = 'classpath:ecodata-ehcache.xml'
             }
         }
     }
 }
-grails.cache.config = {
-
-    provider {
-        name "${appName}-ehcache"
-    }
-    diskStore {
-        path "/data/${appName}/ehcache"
-    }
-    cache {
-        name 'userDetailsCache'
-        timeToLiveSeconds 60 * 60 * 24
-        maxElementsInMemory 2000
-        maxElementsOnDisk 2000
-        overflowToDisk true
-        diskPersistent true
-    }
-}
 
 
 security {
@@ -636,7 +628,7 @@ environments {
     }
     test {
         // Override disk store so the travis build doesn't fail.
-        grails.cache.config = {
+        grails.cache.ehcache = {
             diskStore {
                 path '/tmp'
             }
@@ -665,7 +657,7 @@ environments {
         security.cas.loginUrl="${security.cas.casServerUrlPrefix}/login"
     }
     meritfunctionaltest {
-        grails.cache.config = {
+        grails.cache.ehcache = {
             diskStore {
                 path '/tmp'
             }

diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml
@@ -72,6 +72,7 @@ server:
     servlet:
         session:
             timeout: 30m
+    shutdown: "graceful"
 management:
     endpoints:
         enabled-by-default: false
@@ -202,6 +203,8 @@ quartz:
 
 environments:
     development:
+        server:
+            shutdown: "graceful"
         grails.config.locations:
             - ~/data/ecodata/config/ecodata-config.properties
     test:

diff --git a/grails-app/conf/ecodata-ehcache.xml b/grails-app/conf/ecodata-ehcache.xml
@@ -0,0 +1,61 @@
+<config xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+        xmlns='http://www.ehcache.org/v3'
+        xmlns:jsr107='http://www.ehcache.org/v3/jsr107'>
+
+    <persistence directory="/data/ecodata/ehcache"/>
+
+    <cache alias="spatialGeoJsonPid" uses-template="defaultSetting">
+        <expiry>
+            <none/>
+        </expiry>
+        <resources>
+            <disk unit="GB" persistent="true">10</disk>
+        </resources>
+    </cache>
+    <cache alias="spatialSearchObject" uses-template="defaultSetting">
+        <expiry>
+            <none/>
+        </expiry>
+
+        <resources>
+            <disk unit="MB" persistent="true">500</disk>
+        </resources>
+    </cache>
+
+    <cache alias="spatialSearchObjectMap" uses-template="defaultSetting">
+        <expiry>
+            <tti unit="minutes">2</tti>
+        </expiry>
+        <resources>
+            <heap unit="entries">200</heap>
+        </resources>
+    </cache>
+    <cache alias="spatialGeoJsonPidObject" uses-template="defaultSetting">
+        <expiry>
+            <tti unit="seconds">45</tti>
+        </expiry>
+        <resources>
+            <heap unit="entries">100</heap>
+        </resources>
+    </cache>
+    <cache alias="grailsTemplatesCache">
+        <key-type>java.io.Serializable</key-type>
+        <value-type>java.io.Serializable</value-type>
+        <expiry>
+            <ttl unit="hours">24</ttl>
+        </expiry>
+        <heap>10</heap>
+    </cache>
+    <cache alias="grailsBlocksCache">
+        <key-type>java.io.Serializable</key-type>
+        <value-type>java.io.Serializable</value-type>
+        <expiry>
+            <ttl unit="hours">24</ttl>
+        </expiry>
+        <heap>10</heap>
+    </cache>
+    <cache-template name="defaultSetting">
+        <key-type>java.io.Serializable</key-type>
+        <value-type>java.util.HashMap</value-type>
+    </cache-template>
+</config>
diff --git a/grails-app/conf/logback.xml b/grails-app/conf/logback.xml
@@ -11,11 +11,11 @@
     </encoder>
   </appender>
 
-  <logger name="au.org.ala.ecodata" level="INFO">
-    <appender-ref ref="STDOUT" />
-  </logger>
+  <logger name="au.org.ala.ecodata" level="DEBUG" />
+
+  <logger name="org.ehcache" level="DEBUG" />
 
-  <root level="error">
+  <root level="INFO">
     <appender-ref ref="STDOUT" />
   </root>
 </configuration>
diff --git a/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy b/grails-app/controllers/au/org/ala/ecodata/AdminController.groovy
@@ -11,6 +11,7 @@ import org.apache.http.HttpStatus
 import org.elasticsearch.action.search.SearchResponse
 import org.elasticsearch.search.SearchHit
 import org.grails.datastore.mapping.query.api.BuildableCriteria
+import org.grails.plugin.cache.GrailsCacheManager
 import org.joda.time.DateTime
 import org.joda.time.format.DateTimeFormatter
 import org.joda.time.format.ISODateTimeFormat
@@ -38,6 +39,7 @@ class AdminController {
     RecordService recordService
     ProjectActivityService projectActivityService
     ParatooService paratooService
+    GrailsCacheManager grailsCacheManager
 
     @AlaSecured(["ROLE_ADMIN"])
     def index() {}
@@ -250,6 +252,89 @@ class AdminController {
 
             offset += batchSize
         }
+
+        def result = [code:code]
+        render result as JSON
+    }
+
+    @AlaSecured(["ROLE_ADMIN"])
+    def updateSiteLocationMetadata() {
+        def code = 'success'
+        def total = 0
+        def offset = 0
+        def batchSize = 100
+        def isMERIT = params.getBoolean('isMERIT', true)
+        def startTime = System.currentTimeMillis(), finishTime, startInterimTime, endInterimTime, batchStartTime, batchEndTime
+        List<String> fids = grailsApplication.config.getProperty('site.check.boundary.layers', List<String>)
+        def totalSites
+        List projectIds = []
+        if (isMERIT) {
+            projectIds = projectService.getAllMERITProjectIds()
+            totalSites = Site.countByStatusAndProjectsInList('active', projectIds)
+        }
+        else {
+            totalSites = Site.countByStatus('active')
+        }
+
+        def count = batchSize // For first loop iteration
+        while (count == batchSize) {
+            batchStartTime = startInterimTime = System.currentTimeMillis()
+            def sites
+            if (isMERIT) {
+                sites = Site.findAllByProjectsInListAndStatus(projectIds, 'active', [offset: offset, max: batchSize, sort: "siteId", order: "asc"]).collect {
+                    siteService.toMap(it, 'flat')
+                }
+            }
+            else {
+                sites = Site.findAllByStatus('active', [offset: offset, max: batchSize, sort: "siteId", order: "asc"]).collect {
+                    siteService.toMap(it, 'flat')
+                }
+            }
+            count = sites.size()
+            endInterimTime = System.currentTimeMillis()
+            log.debug("Time taken to fetch ${batchSize} records: ${endInterimTime - startInterimTime} ms")
+            startInterimTime = endInterimTime
+            Site.withSession { session -> session.clear() }
+            Site.withNewSession {
+                sites.eachWithIndex { site, index ->
+                    try {
+                        total++
+                        if(total > 0 && (total % batchSize) == 0) {
+                            log.info("${total+1} or ${(total+1)*100/totalSites} % sites updated in db..")
+                        }
+
+                        if (!site.projects || !site.extent) {
+                            log.debug("Ignoring site ${site.siteId} due to no associated projects or no extent")
+                            return
+                        }
+                        siteService.populateLocationMetadataForSite(site, fids)
+                        endInterimTime = System.currentTimeMillis()
+                        log.debug("Time taken to update metadata ${site.siteId}: ${endInterimTime - startInterimTime} ms")
+                        startInterimTime = endInterimTime
+
+                        if (site?.extent) {
+                            siteService.update([extent: site.extent], site.siteId, false)
+                            endInterimTime = System.currentTimeMillis()
+                            log.debug("Time taken to update site ${site.siteId}: ${endInterimTime - startInterimTime} ms")
+                            startInterimTime = endInterimTime
+                        }
+                    }
+                    catch (Exception e) {
+                        log.error("Unable to complete the operation ", e)
+                        code = "error"
+                    }
+                }
+            }
+
+            offset += batchSize
+
+            batchEndTime = System.currentTimeMillis()
+            log.debug("Time taken to process ${batchSize} records: ${batchEndTime - batchStartTime} ms")
+        }
+
+        finishTime = System.currentTimeMillis()
+        log.debug("site update compled in ${finishTime - startTime} ms")
+
         def result = [code:code]
         render result as JSON
     }
@@ -782,4 +867,15 @@ class AdminController {
         render text: template as JSON, status: HttpStatus.SC_OK, contentType: 'application/json'
     }
 
+    @AlaSecured(["ROLE_ADMIN"])
+    def clearCache() {
+        def caches = grailsCacheManager.getCacheNames()
+        if (caches.contains(params.cache)) {
+            grailsCacheManager.getCache(params.cache).clear()
+            render text: [message: "Success"] as JSON, status: HttpStatus.SC_OK
+        }
+        else {
+            render text: [message: "Cache name not found"] as JSON, status: HttpStatus.SC_NOT_FOUND
+        }
+    }
 }
diff --git a/grails-app/domain/au/org/ala/ecodata/Site.groovy b/grails-app/domain/au/org/ala/ecodata/Site.groovy
@@ -32,6 +32,7 @@ class Site {
         name index: true
         siteId index: true
         projects index: true
+        status index: true
         version false
     }
 

diff --git a/grails-app/init/au/org/ala/ecodata/BootStrap.groovy b/grails-app/init/au/org/ala/ecodata/BootStrap.groovy
@@ -17,6 +17,7 @@ import org.bson.Transformer
 import org.bson.types.ObjectId
 import grails.core.ApplicationAttributes
 import org.grails.datastore.mapping.mongo.MongoDatastore
+import org.grails.plugin.cache.GrailsCacheManager
 import org.grails.web.json.JSONObject
 import org.grails.datastore.mapping.core.Datastore
 import org.springframework.beans.factory.annotation.Autowired
@@ -31,6 +32,7 @@ class BootStrap {
     def auditService
     def hubService
     def messageSource
+    def grailsCacheManager
     @Autowired
     MongoDatastore mongoDatastore
 
@@ -129,7 +131,10 @@ class BootStrap {
     }
 
     def destroy = {
+        // Close ehcache so that disk cached items are persisted. Otherwise, it will clear it the next time it starts up.
+        grailsCacheManager.destroy()
         // shutdown ES server
         elasticSearchService.destroy()
+        log.info "Shutting down - completed destroy method"
     }
 }
diff --git a/grails-app/services/au/org/ala/ecodata/ProjectService.groovy b/grails-app/services/au/org/ala/ecodata/ProjectService.groovy
@@ -651,6 +651,15 @@ class ProjectService {
         }
     }
 
+    List<String> getAllMERITProjectIds() {
+        Project.withCriteria {
+            eq("isMERIT", true)
+            projections {
+                property("projectId")
+            }
+        }
+    }
+
     /**
      * Performs a case-insensitive search by project name
      * @param name The project name to search for

diff --git a/grails-app/services/au/org/ala/ecodata/SiteService.groovy b/grails-app/services/au/org/ala/ecodata/SiteService.groovy
@@ -621,7 +621,7 @@ class SiteService {
         webService.getJson(url)
     }
 
-    def populateLocationMetadataForSite(Map site) {
+    def populateLocationMetadataForSite(Map site, List<String> fids = null) {
 
         Map siteGeom
         if (site.type == Site.TYPE_COMPOUND) {
@@ -655,7 +655,7 @@ class SiteService {
                 log.error("No geometry for site: ${site.siteId}")
             }
 
-            site.extent.geometry += lookupGeographicFacetsForSite(site)
+            site.extent.geometry += lookupGeographicFacetsForSite(site, fids)
         }
     }
 
@@ -852,7 +852,7 @@ class SiteService {
         }
     }
     Map<String, List<String>> lookupGeographicFacetsForSite(Map site, List<String> fidsToLookup = null) {
-
+        log.debug("Looking up geographic facets for site: "+site.siteId)
         Map<String, List<String>> geographicFacets = null
         switch (site.extent.source) {
             case 'pid':