diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt index 64019ee85f..237d4d6583 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt @@ -86,7 +86,8 @@ class ActiveProjectEmptyFolderCleanupJob( row = node, context = context, keyPrefix = KEY_PREFIX, - useMemory = properties.userMemory + useMemory = properties.userMemory, + cacheNumLimit = properties.cacheNumLimit ) } catch (e: Exception) { logger.error("run empty folder clean for Node $row failed, ${e.message}") @@ -117,7 +118,8 @@ class ActiveProjectEmptyFolderCleanupJob( force = true, keyPrefix = KEY_PREFIX, collectionName = null, - projectId = projectId + projectId = projectId, + cacheNumLimit = properties.cacheNumLimit ) } if (properties.userMemory) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt index ecf2d6bbc0..02733d070a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt @@ -90,7 +90,8 @@ class ActiveProjectNodeFolderStatJob( node = node, context = context, useMemory = properties.userMemory, - keyPrefix = KEY_PREFIX + keyPrefix = KEY_PREFIX, + cacheNumLimit = properties.cacheNumLimit ) } @@ -115,7 +116,8 @@ class ActiveProjectNodeFolderStatJob( force = true, keyPrefix = KEY_PREFIX, collectionName = null, - projectId = projectId + projectId = projectId, + cacheNumLimit = properties.cacheNumLimit ) } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/EmptyFolderCleanup.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/EmptyFolderCleanup.kt index d799648340..17889aa8c6 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/EmptyFolderCleanup.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/EmptyFolderCleanup.kt @@ -89,8 +89,9 @@ class EmptyFolderCleanup( context: EmptyFolderCleanupJobContext, useMemory: Boolean, keyPrefix: String, - collectionName: String? = null - ) { + collectionName: String? = null, + cacheNumLimit: Long, + ) { if (row.folder) { val folderKey = FolderUtils.buildCacheKey( collectionName = collectionName, projectId = row.projectId, @@ -120,7 +121,8 @@ class EmptyFolderCleanup( context = context, keyPrefix = keyPrefix, projectId = row.projectId, - collectionName = collectionName + collectionName = collectionName, + cacheNumLimit = cacheNumLimit ) } } @@ -133,9 +135,10 @@ class EmptyFolderCleanup( force: Boolean = false, keyPrefix: String, projectId: String = StringPool.EMPTY, - collectionName: String? = null - ) { - if (!force && context.folders.size < 100000) return + collectionName: String? = null, + cacheNumLimit: Long, + ) { + if (!force && context.folders.size < cacheNumLimit) return if (context.folders.isEmpty()) return val movedToRedis: MutableList = mutableListOf() val storedFolderPrefix = if (collectionName.isNullOrEmpty()) { @@ -172,6 +175,7 @@ class EmptyFolderCleanup( for (key in movedToRedis) { context.folders.remove(key) } + movedToRedis.clear() } fun emptyFolderHandlerWithMemory( diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt index 996918846e..e356c054aa 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt @@ -102,7 +102,8 @@ class InactiveProjectEmptyFolderCleanupJob( ) emptyFolderCleanup.collectEmptyFolderWithMemory( row = node, context = context, collectionName = collectionName, - keyPrefix = KEY_PREFIX, useMemory = properties.userMemory + keyPrefix = KEY_PREFIX, useMemory = properties.userMemory, + cacheNumLimit = properties.cacheNumLimit ) } @@ -132,6 +133,7 @@ class InactiveProjectEmptyFolderCleanupJob( force = true, keyPrefix = KEY_PREFIX, collectionName = collectionName, + cacheNumLimit = properties.cacheNumLimit ) } if (properties.userMemory) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt index a3a8c7f24e..5d6158ef8a 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt @@ -111,7 +111,8 @@ class InactiveProjectNodeFolderStatJob( context = context, useMemory = properties.userMemory, keyPrefix = KEY_PREFIX, - collectionName = collectionName + collectionName = collectionName, + cacheNumLimit = properties.cacheNumLimit ) } @@ -148,6 +149,7 @@ class InactiveProjectNodeFolderStatJob( force = true, keyPrefix = KEY_PREFIX, collectionName = collectionName, + cacheNumLimit = properties.cacheNumLimit ) } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/NodeFolderStat.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/NodeFolderStat.kt index 6790dacc99..94e815ea0b 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/NodeFolderStat.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/NodeFolderStat.kt @@ -82,7 +82,8 @@ class NodeFolderStat( useMemory: Boolean, keyPrefix: String, collectionName: String? = null, - ) { + cacheNumLimit: Long, + ) { //只统计非目录类节点;没有根目录这个节点,不需要统计 if (node.path == PathUtils.ROOT) { return @@ -99,15 +100,16 @@ class NodeFolderStat( context = context, collectionName = collectionName ) - if (!useMemory) { - // 避免每次请求都去请求redis, 先将数据缓存在本地cache中,到达上限后更新到redis - updateRedisCache( - context = context, - collectionName = collectionName, - keyPrefix = keyPrefix, - projectId = node.projectId - ) - } + } + if (!useMemory) { + // 避免每次请求都去请求redis, 先将数据缓存在本地cache中,到达上限后更新到redis + updateRedisCache( + context = context, + collectionName = collectionName, + keyPrefix = keyPrefix, + projectId = node.projectId, + cacheNumLimit = cacheNumLimit + ) } } @@ -138,9 +140,10 @@ class NodeFolderStat( keyPrefix: String, projectId: String = StringPool.EMPTY, force: Boolean = false, - collectionName: String? - ) { - if (!force && context.folderCache.size < 100000) return + collectionName: String?, + cacheNumLimit: Long, + ) { + if (!force && context.folderCache.size < cacheNumLimit) return if (context.folderCache.isEmpty()) return val movedToRedis: MutableList = mutableListOf() val storedFolderPrefix = if (collectionName.isNullOrEmpty()) { @@ -176,6 +179,7 @@ class NodeFolderStat( for (key in movedToRedis) { context.folderCache.remove(key) } + movedToRedis.clear() } fun removeRedisKey(key: String) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/StatJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/StatJobProperties.kt index efaad42831..95e95223aa 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/StatJobProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/StatJobProperties.kt @@ -38,5 +38,6 @@ open class StatJobProperties( // 特殊仓库在每周第几天执行,默认周六 var specialDay: Int = 6, var concurrencyNum: Int = 1, - var userMemory: Boolean = true + var userMemory: Boolean = true, + var cacheNumLimit: Long = 10000, ) : MongodbJobProperties() \ No newline at end of file