feat: 添加配置TencentBlueKing#2420

lannoy0523 · Jul 25, 2024 · 29067dc · 29067dc
1 parent 9c5e83b
commit 29067dc
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 26 deletions.
diff --git a/.../main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt b/.../main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt
@@ -86,7 +86,8 @@ class ActiveProjectEmptyFolderCleanupJob(
                 row = node,
                 context = context,
                 keyPrefix = KEY_PREFIX,
-                useMemory = properties.userMemory
+                useMemory = properties.userMemory,
+                cacheNumLimit = properties.cacheNumLimit
             )
         } catch (e: Exception) {
             logger.error("run empty folder clean for Node $row failed, ${e.message}")
@@ -117,7 +118,8 @@ class ActiveProjectEmptyFolderCleanupJob(
                 force = true,
                 keyPrefix = KEY_PREFIX,
                 collectionName = null,
-                projectId = projectId
+                projectId = projectId,
+                cacheNumLimit = properties.cacheNumLimit
             )
         }
         if (properties.userMemory) {

diff --git a/.../src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt b/.../src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt
@@ -90,7 +90,8 @@ class ActiveProjectNodeFolderStatJob(
             node = node,
             context = context,
             useMemory = properties.userMemory,
-            keyPrefix = KEY_PREFIX
+            keyPrefix = KEY_PREFIX,
+            cacheNumLimit = properties.cacheNumLimit
         )
     }
 
@@ -115,7 +116,8 @@ class ActiveProjectNodeFolderStatJob(
                 force = true,
                 keyPrefix = KEY_PREFIX,
                 collectionName = null,
-                projectId = projectId
+                projectId = projectId,
+                cacheNumLimit = properties.cacheNumLimit
             )
         }
 

diff --git a/.../job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/EmptyFolderCleanup.kt b/.../job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/EmptyFolderCleanup.kt
@@ -89,8 +89,9 @@ class EmptyFolderCleanup(
         context: EmptyFolderCleanupJobContext,
         useMemory: Boolean,
         keyPrefix: String,
-        collectionName: String? = null
-    ) {
+        collectionName: String? = null,
+        cacheNumLimit: Long,
+        ) {
         if (row.folder) {
             val folderKey = FolderUtils.buildCacheKey(
                 collectionName = collectionName, projectId = row.projectId,
@@ -120,7 +121,8 @@ class EmptyFolderCleanup(
                 context = context,
                 keyPrefix = keyPrefix,
                 projectId = row.projectId,
-                collectionName = collectionName
+                collectionName = collectionName,
+                cacheNumLimit = cacheNumLimit
             )
         }
     }
@@ -133,9 +135,10 @@ class EmptyFolderCleanup(
         force: Boolean = false,
         keyPrefix: String,
         projectId: String = StringPool.EMPTY,
-        collectionName: String? = null
-    ) {
-        if (!force && context.folders.size < 100000) return
+        collectionName: String? = null,
+        cacheNumLimit: Long,
+        ) {
+        if (!force && context.folders.size < cacheNumLimit) return
         if (context.folders.isEmpty()) return
         val movedToRedis: MutableList<String> = mutableListOf()
         val storedFolderPrefix = if (collectionName.isNullOrEmpty()) {
@@ -172,6 +175,7 @@ class EmptyFolderCleanup(
         for (key in movedToRedis) {
             context.folders.remove(key)
         }
+        movedToRedis.clear()
     }
 
     fun emptyFolderHandlerWithMemory(

diff --git a/...ain/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt b/...ain/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt
@@ -102,7 +102,8 @@ class InactiveProjectEmptyFolderCleanupJob(
         )
         emptyFolderCleanup.collectEmptyFolderWithMemory(
             row = node, context = context, collectionName = collectionName,
-            keyPrefix = KEY_PREFIX, useMemory = properties.userMemory
+            keyPrefix = KEY_PREFIX, useMemory = properties.userMemory,
+            cacheNumLimit = properties.cacheNumLimit
         )
     }
 
@@ -132,6 +133,7 @@ class InactiveProjectEmptyFolderCleanupJob(
                 force = true,
                 keyPrefix = KEY_PREFIX,
                 collectionName = collectionName,
+                cacheNumLimit = properties.cacheNumLimit
             )
         }
         if (properties.userMemory) {

diff --git a/...rc/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt b/...rc/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt
@@ -111,7 +111,8 @@ class InactiveProjectNodeFolderStatJob(
             context = context,
             useMemory = properties.userMemory,
             keyPrefix = KEY_PREFIX,
-            collectionName = collectionName
+            collectionName = collectionName,
+            cacheNumLimit = properties.cacheNumLimit
         )
     }
 
@@ -148,6 +149,7 @@ class InactiveProjectNodeFolderStatJob(
                 force = true,
                 keyPrefix = KEY_PREFIX,
                 collectionName = collectionName,
+                cacheNumLimit = properties.cacheNumLimit
             )
         }
 

diff --git a/...kend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/NodeFolderStat.kt b/...kend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/NodeFolderStat.kt
@@ -82,7 +82,8 @@ class NodeFolderStat(
         useMemory: Boolean,
         keyPrefix: String,
         collectionName: String? = null,
-    ) {
+        cacheNumLimit: Long,
+        ) {
         //只统计非目录类节点；没有根目录这个节点，不需要统计
         if (node.path == PathUtils.ROOT) {
             return
@@ -99,15 +100,16 @@ class NodeFolderStat(
                 context = context,
                 collectionName = collectionName
             )
-            if (!useMemory) {
-                // 避免每次请求都去请求redis， 先将数据缓存在本地cache中，到达上限后更新到redis
-                updateRedisCache(
-                    context = context,
-                    collectionName = collectionName,
-                    keyPrefix = keyPrefix,
-                    projectId = node.projectId
-                )
-            }
+        }
+        if (!useMemory) {
+            // 避免每次请求都去请求redis， 先将数据缓存在本地cache中，到达上限后更新到redis
+            updateRedisCache(
+                context = context,
+                collectionName = collectionName,
+                keyPrefix = keyPrefix,
+                projectId = node.projectId,
+                cacheNumLimit = cacheNumLimit
+            )
         }
     }
 
@@ -138,9 +140,10 @@ class NodeFolderStat(
         keyPrefix: String,
         projectId: String = StringPool.EMPTY,
         force: Boolean = false,
-        collectionName: String?
-    ) {
-        if (!force && context.folderCache.size < 100000) return
+        collectionName: String?,
+        cacheNumLimit: Long,
+        ) {
+        if (!force && context.folderCache.size < cacheNumLimit) return
         if (context.folderCache.isEmpty()) return
         val movedToRedis: MutableList<String> = mutableListOf()
         val storedFolderPrefix = if (collectionName.isNullOrEmpty()) {
@@ -176,6 +179,7 @@ class NodeFolderStat(
         for (key in movedToRedis) {
             context.folderCache.remove(key)
         }
+        movedToRedis.clear()
     }
 
     fun removeRedisKey(key: String) {

diff --git a/...job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/StatJobProperties.kt b/...job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/StatJobProperties.kt
@@ -38,5 +38,6 @@ open class StatJobProperties(
     // 特殊仓库在每周第几天执行，默认周六
     var specialDay: Int = 6,
     var concurrencyNum: Int = 1,
-    var userMemory: Boolean = true
+    var userMemory: Boolean = true,
+    var cacheNumLimit: Long = 10000,
 ) : MongodbJobProperties()