feat: 目录统计相关job支持使用redis TencentBlueKing#2313

* feat: 目录统计支持使用redis TencentBlueKing#2313 * feat: 空目录清理支持使用redis TencentBlueKing#2313 * feat: 空目录清理调整 TencentBlueKing#2313 * feat: 目录统计调整 TencentBlueKing#2313 * feat: redis key调整调整 TencentBlueKing#2313 * feat: 代码调整 TencentBlueKing#2313
lannoy0523 · Jul 23, 2024 · 99bac8f · 99bac8f
1 parent 02a2968
commit 99bac8f
Show file tree

Hide file tree

Showing 9 changed files with 543 additions and 45 deletions.
diff --git a/.../main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt b/.../main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt
@@ -27,9 +27,11 @@
 
 package com.tencent.bkrepo.job.batch.task.stat
 
+import com.tencent.bkrepo.common.api.constant.StringPool
 import com.tencent.bkrepo.job.batch.base.ActiveProjectService
 import com.tencent.bkrepo.job.batch.base.JobContext
 import com.tencent.bkrepo.job.batch.context.EmptyFolderCleanupJobContext
+import com.tencent.bkrepo.job.batch.utils.FolderUtils
 import com.tencent.bkrepo.job.config.properties.ActiveProjectEmptyFolderCleanupJobProperties
 import org.slf4j.LoggerFactory
 import org.springframework.boot.context.properties.EnableConfigurationProperties
@@ -45,7 +47,7 @@ import java.time.Duration
 @Component
 @EnableConfigurationProperties(ActiveProjectEmptyFolderCleanupJobProperties::class)
 class ActiveProjectEmptyFolderCleanupJob(
-    private val properties: ActiveProjectEmptyFolderCleanupJobProperties,
+    val properties: ActiveProjectEmptyFolderCleanupJobProperties,
     executor: ThreadPoolTaskExecutor,
     private val activeProjectService: ActiveProjectService,
     private val mongoTemplate: MongoTemplate,
@@ -59,6 +61,15 @@ class ActiveProjectEmptyFolderCleanupJob(
         logger.info("empty folder cleanup job for active projects finished")
     }
 
+
+    override fun beforeRunProject(projectId: String) {
+        if (properties.userMemory) return
+        // 每次任务启动前要将redis上对应的key清理， 避免干扰
+        val key = KEY_PREFIX + StringPool.COLON +
+            FolderUtils.buildCacheKey(projectId = projectId, repoName = StringPool.EMPTY)
+        emptyFolderCleanup.removeRedisKey(key)
+    }
+
     override fun runRow(row: StatNode, context: JobContext) {
         require(context is EmptyFolderCleanupJobContext)
         try {
@@ -71,7 +82,12 @@ class ActiveProjectEmptyFolderCleanupJob(
                 folder = row.folder,
                 size = row.size
             )
-            emptyFolderCleanup.collectEmptyFolder(node, context)
+            emptyFolderCleanup.collectEmptyFolderWithMemory(
+                row = node,
+                context = context,
+                keyPrefix = KEY_PREFIX,
+                useMemory = properties.userMemory
+            )
         } catch (e: Exception) {
             logger.error("run empty folder clean for Node $row failed, ${e.message}")
         }
@@ -94,16 +110,38 @@ class ActiveProjectEmptyFolderCleanupJob(
     override fun onRunProjectFinished(collection: String, projectId: String, context: JobContext) {
         require(context is EmptyFolderCleanupJobContext)
         logger.info("will filter empty folder in project $projectId")
-        emptyFolderCleanup.emptyFolderHandler(
-            collection = collection,
-            context = context,
-            deletedEmptyFolder = properties.deletedEmptyFolder,
-            projectId = projectId,
-            deleteFolderRepos = properties.deleteFolderRepos
-        )
+
+        if (!properties.userMemory) {
+            emptyFolderCleanup.collectEmptyFolderWithRedis(
+                context = context,
+                force = true,
+                keyPrefix = KEY_PREFIX,
+                collectionName = null,
+                projectId = projectId
+            )
+        }
+        if (properties.userMemory) {
+            emptyFolderCleanup.emptyFolderHandlerWithMemory(
+                collection = collection,
+                context = context,
+                deletedEmptyFolder = properties.deletedEmptyFolder,
+                projectId = projectId,
+                deleteFolderRepos = properties.deleteFolderRepos
+            )
+        } else {
+            emptyFolderCleanup.emptyFolderHandlerWithRedis(
+                collection = collection,
+                keyPrefix = KEY_PREFIX,
+                context = context,
+                deletedEmptyFolder = properties.deletedEmptyFolder,
+                projectId = projectId,
+                deleteFolderRepos = properties.deleteFolderRepos
+            )
+        }
     }
 
     companion object {
         private val logger = LoggerFactory.getLogger(ActiveProjectEmptyFolderCleanupJob::class.java)
+        private const val KEY_PREFIX = "activeEmptyFolder"
     }
 }
diff --git a/.../src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt b/.../src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt
@@ -27,10 +27,12 @@
 
 package com.tencent.bkrepo.job.batch.task.stat
 
+import com.tencent.bkrepo.common.api.constant.StringPool
 import com.tencent.bkrepo.job.FOLDER
 import com.tencent.bkrepo.job.batch.base.ActiveProjectService
 import com.tencent.bkrepo.job.batch.base.JobContext
 import com.tencent.bkrepo.job.batch.context.NodeFolderJobContext
+import com.tencent.bkrepo.job.batch.utils.FolderUtils
 import com.tencent.bkrepo.job.config.properties.ActiveProjectNodeFolderStatJobProperties
 import org.slf4j.LoggerFactory
 import org.springframework.boot.context.properties.EnableConfigurationProperties
@@ -46,7 +48,7 @@ import java.time.Duration
 @Component
 @EnableConfigurationProperties(ActiveProjectNodeFolderStatJobProperties::class)
 class ActiveProjectNodeFolderStatJob(
-    properties: ActiveProjectNodeFolderStatJobProperties,
+    val properties: ActiveProjectNodeFolderStatJobProperties,
     executor: ThreadPoolTaskExecutor,
     private val activeProjectService: ActiveProjectService,
     private val mongoTemplate: MongoTemplate,
@@ -65,6 +67,14 @@ class ActiveProjectNodeFolderStatJob(
         return Criteria().and(FOLDER).`is`(false)
     }
 
+    override fun beforeRunProject(projectId: String) {
+        if (properties.userMemory) return
+        // 每次任务启动前要将redis上对应的key清理， 避免干扰
+        val key = KEY_PREFIX + StringPool.COLON +
+            FolderUtils.buildCacheKey(projectId = projectId, repoName = StringPool.EMPTY)
+        nodeFolderStat.removeRedisKey(key)
+    }
+
     override fun runRow(row: StatNode, context: JobContext) {
         require(context is NodeFolderJobContext)
         val node = nodeFolderStat.buildNode(
@@ -76,7 +86,12 @@ class ActiveProjectNodeFolderStatJob(
             folder = row.folder,
             size = row.size
         )
-        nodeFolderStat.collectNode(node, context)
+        nodeFolderStat.collectNode(
+            node = node,
+            context = context,
+            useMemory = properties.userMemory,
+            keyPrefix = KEY_PREFIX
+        )
     }
 
     override fun createJobContext(): NodeFolderJobContext {
@@ -94,8 +109,22 @@ class ActiveProjectNodeFolderStatJob(
      */
     override fun onRunProjectFinished(collection: String, projectId: String, context: JobContext) {
         require(context is NodeFolderJobContext)
-        logger.info("store memory cache to db with projectId $projectId")
-        nodeFolderStat.storeMemoryCacheToDB(context, collection, projectId)
+        if (!properties.userMemory) {
+            nodeFolderStat.updateRedisCache(
+                context = context,
+                force = true,
+                keyPrefix = KEY_PREFIX,
+                collectionName = null,
+                projectId = projectId
+            )
+        }
+
+        logger.info("store cache to db with projectId $projectId")
+        if (properties.userMemory) {
+            nodeFolderStat.storeMemoryCacheToDB(context, collection, projectId)
+        } else {
+            nodeFolderStat.storeRedisCacheToDB(context, KEY_PREFIX, collection, projectId)
+        }
     }
 
     /**
@@ -105,5 +134,6 @@ class ActiveProjectNodeFolderStatJob(
 
     companion object {
         private val logger = LoggerFactory.getLogger(ActiveProjectNodeFolderStatJob::class.java)
+        private const val KEY_PREFIX = "activeProjectNode"
     }
 }