feat: 增加目录统计 TencentBlueKing#1012 (TencentBlueKing#1065)

* feat: v1 TencentBlueKing#1012 * feat: 统计目录size大小 TencentBlueKing#1012 * feat: 存储调整 TencentBlueKing#1012 * feat: 数据刷新调整 TencentBlueKing#1012 * feat: 数据刷新调整 TencentBlueKing#1012 * feat: 缓存数据存入调整 TencentBlueKing#1012 * feat: 目录节点大小统计调整 TencentBlueKing#1012 * feat: 目录统计表中增加节点数 TencentBlueKing#1012 * feat: 增加统计以及下载接口 TencentBlueKing#1012 * feat: 执行计划变更为每周6执行一次 TencentBlueKing#1012 * feat: 修复目录层级不全导致数据不准确的问题 TencentBlueKing#1012 * feat: 修复目录层级不全导致数据不准确的问题 TencentBlueKing#1012 * feat: 目录size存储回node表中 TencentBlueKing#1012 * feat: 删除多余代码 TencentBlueKing#1012 * feat: 删除多余代码 TencentBlueKing#1012 * feat: job增加回滚操作 TencentBlueKing#1012 * feat: 类名调整 TencentBlueKing#1012 * feat: 转换报错修复 TencentBlueKing#1012 * feat: 修改时间不更新 TencentBlueKing#1012 * feat: move节点更新逻辑调整 TencentBlueKing#1012 * feat: move节点更新逻辑调整 TencentBlueKing#1012 * feat: 没有根目录这个节点，所以忽略path为/的节点 TencentBlueKing#1012 * feat: 增加子job执行策略 TencentBlueKing#1012 * feat: 去除多余代码 TencentBlueKing#1012 * feat: 将实时计算结果更新到db中 TencentBlueKing#1012 * feat: 计算逻辑调整 TencentBlueKing#1012 * feat: 测试用例调整 TencentBlueKing#1012 * feat: 目录统计缓存逻辑调整 TencentBlueKing#1012 * feat: 使用redis存储缓存记录 TencentBlueKing#1012 * feat: 修复ERR unknown command 'KEYS' TencentBlueKing#1012 * feat: redis存储逻辑调整，避免使用keys或者scan命令报错问题 TencentBlueKing#1012 * feat: 代码调整 TencentBlueKing#1012 * feat: 批量操作 TencentBlueKing#1012 * feat: redis key逻辑调整 TencentBlueKing#1012 * feat: redis存储逻辑调整；执行计划支持每天执行部分表 TencentBlueKing#1012 * feat: 配置调整 TencentBlueKing#1012 * feat: 空列表判断 TencentBlueKing#1012 * feat: 写redis前，先写内存缓存，内存缓存满后再更新到redis TencentBlueKing#1012 * feat: nodenum取值调整 TencentBlueKing#1012 * feat: redis设置值方式调整 TencentBlueKing#1012 * feat: 存储数据读取redis调整 TencentBlueKing#1012 * feat: 新增节点下非目录节点数量字段 TencentBlueKing#1012
lannoy0523 · Aug 28, 2023 · aaf5ed4 · aaf5ed4
1 parent 2cb50a0
commit aaf5ed4
Show file tree

Hide file tree

Showing 26 changed files with 1,103 additions and 12 deletions.
diff --git a/...nd/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/EasyExcelUtils.kt b/...nd/analyst/biz-analyst/src/main/kotlin/com/tencent/bkrepo/analyst/utils/EasyExcelUtils.kt
@@ -1,3 +1,30 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
 package com.tencent.bkrepo.analyst.utils
 
 import com.alibaba.excel.EasyExcel

diff --git a/...fact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt b/...fact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/constant/ArtifactConstants.kt
@@ -112,6 +112,12 @@ const val PIPELINE = "pipeline"
  */
 const val REPORT = "report"
 
+
+/**
+ * 日志仓库
+ */
+const val LOG = "log"
+
 /**
  * 文件访问请求是否为直接下载
  */

diff --git a/...rtifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/path/PathUtils.kt b/...rtifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/path/PathUtils.kt
@@ -207,6 +207,19 @@ object PathUtils {
         return result
     }
 
+    /**
+     * 获取当前路径的所有上级目录列表
+     */
+    fun resolveAncestorFolder(fullPath: String): List<String> {
+        return resolveAncestor(fullPath).map {
+            if (it != ROOT) {
+                it.removeSuffix(StringPool.SLASH)
+            } else {
+                it
+            }
+        }
+    }
+
     /**
      * 根据fullPath解析文件名称，返回格式abc.txt
      *

diff --git a/src/backend/job/biz-job/build.gradle.kts b/src/backend/job/biz-job/build.gradle.kts
@@ -31,6 +31,7 @@ dependencies {
     implementation(project(":common:common-security"))
     implementation(project(":common:common-storage:storage-service"))
     implementation(project(":common:common-stream"))
+    implementation(project(":common:common-redis"))
     implementation(project(":repository:api-repository"))
     implementation(project(":helm:api-helm"))
     implementation(project(":oci:api-oci"))

diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt
@@ -51,6 +51,14 @@ const val CATEGORY = "category"
 const val CREATED_DATE = "createdDate"
 const val LAST_MODIFIED_DATE = "lastModifiedDate"
 const val DELETED_DATE = "deleted"
+const val FULLPATH = "fullPath"
+const val PATH = "path"
+
+/**
+ * 缓存类型
+ */
+const val REDIS_CACHE_TYPE = "redis"
+const val MEMORY_CACHE_TYPE = "memory"
 
 /**
  * metrics

diff --git a/...job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/CompositeMongoDbBatchJob.kt b/...job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/CompositeMongoDbBatchJob.kt
@@ -44,6 +44,12 @@ abstract class CompositeMongoDbBatchJob<T>(
         return CompositeJobContext(enabledChildJobs)
     }
 
+    override fun onRunCollectionFinished(collectionName: String, context: CompositeJobContext<T>) {
+        context.childJobs.forEach {
+            logException { it.onRunCollectionFinished(collectionName, context.childContext(it.getJobName())) }
+        }
+    }
+
     protected abstract fun createChildJobs(): List<ChildMongoDbBatchJob<T>>
 
     @Suppress("TooGenericExceptionCaught")

diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/MongoDbBatchJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/base/MongoDbBatchJob.kt
@@ -76,6 +76,12 @@ abstract class MongoDbBatchJob<Entity, Context : JobContext>(
 
     abstract fun entityClass(): Class<Entity>
 
+
+    /**
+     * 表执行结束回调
+     * */
+    open fun onRunCollectionFinished(collectionName: String, context: Context) {}
+
     private val batchSize: Int
         get() = properties.batchSize
 
@@ -162,6 +168,7 @@ abstract class MongoDbBatchJob<Entity, Context : JobContext>(
             } while (querySize == pageSize && isRunning())
         }.apply {
             val elapsedTime = HumanReadable.time(this)
+            onRunCollectionFinished(collectionName, context)
             logger.info("Job[${getJobName()}]: collection $collectionName run completed,sum [$sum] elapse $elapsedTime")
         }
     }

diff --git a/...nd/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/FolderChildContext.kt b/...nd/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/FolderChildContext.kt
@@ -0,0 +1,52 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2022 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.batch.context
+
+import com.tencent.bkrepo.job.MEMORY_CACHE_TYPE
+import com.tencent.bkrepo.job.batch.base.ChildJobContext
+import com.tencent.bkrepo.job.batch.base.JobContext
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.atomic.LongAdder
+
+class FolderChildContext(
+    parentContent: JobContext,
+    // 是否执行任务
+    var runFlag: Boolean = false,
+    // 缓存类型redis和内存：数据量级大的建议使用redis
+    var cacheType: String = MEMORY_CACHE_TYPE,
+    // 表对应项目记录： 主要用于redis缓存生成key使用
+    var projectMap: ConcurrentHashMap<String, MutableSet<String>> = ConcurrentHashMap(),
+    // 用于内存缓存下存储目录统计信息
+    var folderCache: ConcurrentHashMap<String, FolderMetrics> = ConcurrentHashMap()
+) : ChildJobContext(parentContent) {
+
+    data class FolderMetrics(
+        var nodeNum: LongAdder = LongAdder(),
+        var capSize: LongAdder = LongAdder()
+    )
+}
diff --git a/...b/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/ProjectRepoChildContext.kt b/...b/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/context/ProjectRepoChildContext.kt
@@ -41,7 +41,9 @@ class ProjectRepoChildContext(
                 .storageCredentials
                 ?.key ?: "default"
             val repo = repoMetrics.getOrPut(row.repoName) { RepoMetrics(row.repoName, credentialsKey) }
-            repo.size.add(row.size)
+            if (!row.folder) {
+                repo.size.add(row.size)
+            }
             repo.num.increment()
             repo.addFolderMetrics(row)
             repo.addExtensionMetrics(row)
@@ -94,7 +96,9 @@ class ProjectRepoChildContext(
             }
             val metric = folderMetrics.getOrPut(firstLevelPath) { FolderMetric(firstLevelPath) }
             metric.nodeNum.increment()
-            metric.capSize.add(row.size)
+            if (!row.folder) {
+                metric.capSize.add(row.size)
+            }
         }
 
         fun addExtensionMetrics(row: NodeStatCompositeMongoDbBatchJob.Node) {