Skip to content

Commit

Permalink
feat: 仓库大小统计调整 TencentBlueKing#1116 (TencentBlueKing#1128)
Browse files Browse the repository at this point in the history
* feat: 项目仓库容量统计存储单位由GIGABYTE变为BYTE TencentBlueKing#1116

* feat: 一次性分发任务支持统计仓库大小;repository提供项目仓库大小统计查询接口 TencentBlueKing#1116

* feat: 问题修复 TencentBlueKing#1116

* feat: 数据不存在异常修复 TencentBlueKing#1116

* feat: 去掉多余字段 TencentBlueKing#1116
  • Loading branch information
zacYL authored Sep 12, 2023
1 parent fb40117 commit 1d869df
Show file tree
Hide file tree
Showing 21 changed files with 273 additions and 17 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ class ProjectRepoChildContext(
return ProjectRepoStatChildJob.TProjectMetrics(
projectId = projectId,
nodeNum = nodeNum.toLong(),
capSize = capSize.toLong() / TO_GIGABYTE,
capSize = capSize.toLong(),
repoMetrics = repoMetrics
)
}
Expand Down Expand Up @@ -128,7 +128,7 @@ class ProjectRepoChildContext(
return ProjectRepoStatChildJob.TRepoMetrics(
repoName = repoName,
credentialsKey = credentialsKey,
size = size.toLong() / TO_GIGABYTE,
size = size.toLong(),
num = num.toLong()
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,7 @@ const val OPDATA_PROJECT_ID = "projectId"
const val OPDATA_REPOSITORY = "repository"
const val OPDATA_REPO_NAME = "repoName"
const val OPDATA_PATH = "path"


const val TO_GIGABYTE = 1024 * 1024 * 1024

const val OPDATA_FILE_EXTENSION_METRICS = "file_extension_metrics"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ data class RepoMetrics(
@ApiModelProperty("credentialsKey")
val credentialsKey: String? = "default",
@ApiModelProperty("size")
val size: Long,
var size: Long,
@ApiModelProperty("num")
val num: Long
)
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ package com.tencent.bkrepo.opdata.handler.impl

import com.tencent.bkrepo.opdata.constant.OPDATA_CAP_SIZE
import com.tencent.bkrepo.opdata.constant.OPDATA_GRAFANA_NUMBER
import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE
import com.tencent.bkrepo.opdata.handler.QueryHandler
import com.tencent.bkrepo.opdata.pojo.Columns
import com.tencent.bkrepo.opdata.pojo.QueryResult
Expand All @@ -58,7 +59,7 @@ class CapSizeHandler(
size += it.capSize
}
val column = Columns(OPDATA_CAP_SIZE, OPDATA_GRAFANA_NUMBER)
val row = listOf(size)
val row = listOf(size / TO_GIGABYTE)
val data = QueryResult(listOf(column), listOf(row), target.type)
result.add(data)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ import com.tencent.bkrepo.opdata.constant.OPDATA_GRAFANA_STRING
import com.tencent.bkrepo.opdata.constant.OPDATA_PIPELINE
import com.tencent.bkrepo.opdata.constant.OPDATA_PIPELINE_NUM
import com.tencent.bkrepo.opdata.constant.OPDATA_PIPELINE_SIZE
import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE
import com.tencent.bkrepo.opdata.handler.QueryHandler
import com.tencent.bkrepo.opdata.model.TProjectMetrics
import com.tencent.bkrepo.opdata.pojo.Columns
Expand Down Expand Up @@ -86,7 +87,7 @@ class ProjectListHandler(
}
val row = listOf(
repo.projectId, repo.nodeNum, repo.capSize,
customNum, customSize, pipelineNum, pipelineSize
customNum, customSize / TO_GIGABYTE, pipelineNum, pipelineSize / TO_GIGABYTE
)
rows.add(row)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

package com.tencent.bkrepo.opdata.handler.impl

import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE
import com.tencent.bkrepo.opdata.handler.QueryHandler
import com.tencent.bkrepo.opdata.pojo.Target
import com.tencent.bkrepo.opdata.pojo.enums.Metrics
Expand All @@ -52,8 +53,9 @@ class ProjectNodeSizeHandler(
val tmpMap = HashMap<String, Long>()
projects.forEach {
val projectId = it.projectId
if (it.capSize != 0L) {
tmpMap[projectId] = it.capSize
val gbSize = it.capSize / TO_GIGABYTE
if (gbSize != 0L) {
tmpMap[projectId] = gbSize
}
}
return convToDisplayData(tmpMap, result)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@

package com.tencent.bkrepo.opdata.handler.impl

import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE
import com.tencent.bkrepo.opdata.handler.QueryHandler
import com.tencent.bkrepo.opdata.pojo.Target
import com.tencent.bkrepo.opdata.pojo.enums.Metrics
Expand All @@ -54,8 +55,9 @@ class RepoNodeSizeHandler(
val projectId = it.projectId
it.repoMetrics.forEach {
val repoName = it.repoName
if (it.size != 0L) {
tmpMap["$projectId-$repoName"] = it.size
val gbSize = it.size / TO_GIGABYTE
if (gbSize != 0L) {
tmpMap["$projectId-$repoName"] = gbSize
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ package com.tencent.bkrepo.opdata.service

import com.tencent.bkrepo.common.api.pojo.Page
import com.tencent.bkrepo.common.mongo.dao.util.Pages
import com.tencent.bkrepo.opdata.constant.TO_GIGABYTE
import com.tencent.bkrepo.opdata.model.TProjectMetrics
import com.tencent.bkrepo.opdata.pojo.ProjectMetricsOption
import com.tencent.bkrepo.opdata.repository.ProjectMetricsRepository
Expand All @@ -47,6 +48,12 @@ class ProjectMetricsService (
} else {
projectMetricsRepository.findAllByOrderByCreatedDateDesc(pageRequest)
}
queryResult.content.forEach {
it.capSize = it.capSize / TO_GIGABYTE
it.repoMetrics.forEach { repo ->
repo.size = repo.size / TO_GIGABYTE
}
}
return Pages.ofResponse(pageRequest, queryResult.totalElements, queryResult.content)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -253,4 +253,14 @@ class LocalDataManager(
val repo = findRepoByName(projectId, repoName)
return getBlobDataByRange(sha256, range, repo)
}


/**
* 从项目仓库统计信息中找出对应仓库的大小
*/
fun getRepoMetricInfo(projectId: String, repoName: String): Long {
findRepoByName(projectId, repoName)
val projectMetrics = projectClient.getProjectMetrics(projectId).data ?: return 0
return projectMetrics.repoMetrics.firstOrNull { it.repoName == repoName }?.size ?: 0
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ class ReplicaTaskServiceImpl(
return when (replicaObjectType) {
ReplicaObjectType.PACKAGE -> computePackageSize(localProjectId, replicaTaskObjects)
ReplicaObjectType.PATH -> computeNodeSize(localProjectId, replicaTaskObjects)
ReplicaObjectType.REPOSITORY -> computeRepositorySize(localProjectId, replicaTaskObjects)
else -> -1
}
}
Expand All @@ -286,7 +287,7 @@ class ReplicaTaskServiceImpl(
val taskObject = replicaTaskObjects.first()
return taskObject.pathConstraints!!.sumOf {
val node = localDataManager.findNodeDetail(localProjectId, taskObject.localRepoName, it.path!!)
if (node.folder) {
if (node.folder && node.size == 0L) {
try {
localDataManager.listNode(localProjectId, taskObject.localRepoName, it.path!!).sumOf { n -> n.size }
} catch (e: Exception) {
Expand All @@ -299,6 +300,11 @@ class ReplicaTaskServiceImpl(
}
}

private fun computeRepositorySize(localProjectId: String, replicaTaskObjects: List<ReplicaObjectInfo>): Long {
val taskObject = replicaTaskObjects.first()
return localDataManager.getRepoMetricInfo(localProjectId, taskObject.localRepoName)
}

private fun validateRequest(request: ReplicaTaskCreateRequest) {
with(request) {
// 验证任务是否存在
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import com.tencent.bkrepo.common.api.pojo.Page
import com.tencent.bkrepo.common.api.pojo.Response
import com.tencent.bkrepo.repository.pojo.project.ProjectCreateRequest
import com.tencent.bkrepo.repository.pojo.project.ProjectInfo
import com.tencent.bkrepo.repository.pojo.project.ProjectMetricsInfo
import com.tencent.bkrepo.repository.pojo.project.ProjectRangeQueryRequest
import io.swagger.annotations.Api
import io.swagger.annotations.ApiOperation
Expand Down Expand Up @@ -69,4 +70,8 @@ interface ProjectClient {
@ApiOperation("创建项目")
@PostMapping("/create")
fun createProject(@RequestBody request: ProjectCreateRequest): Response<ProjectInfo>

@ApiOperation("项目仓库统计信息列表")
@PostMapping("/metrics/{name}")
fun getProjectMetrics(@ApiParam(value = "项目名") @PathVariable name: String): Response<ProjectMetricsInfo?>
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.repository.pojo.project

import com.tencent.bkrepo.repository.pojo.repo.RepoMetricsInfo
import io.swagger.annotations.ApiModelProperty
import java.time.LocalDateTime

data class ProjectMetricsInfo(
@ApiModelProperty("projectId")
var projectId: String,
@ApiModelProperty("nodeNum")
var nodeNum: Long,
@ApiModelProperty("capSize")
var capSize: Long,
@ApiModelProperty("repoMetrics")
val repoMetrics: List<RepoMetricsInfo>,
@ApiModelProperty("createdDate")
val createdDate: LocalDateTime? = LocalDateTime.now()
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
* documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
* permit persons to whom the Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all copies or substantial portions of
* the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
* LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
* NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.repository.pojo.repo

import io.swagger.annotations.ApiModelProperty

data class RepoMetricsInfo(
@ApiModelProperty("repoName")
val repoName: String,
@ApiModelProperty("credentialsKey")
val credentialsKey: String? = "default",
@ApiModelProperty("size")
var size: Long,
@ApiModelProperty("num")
val num: Long
)
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import com.tencent.bkrepo.common.service.util.ResponseBuilder
import com.tencent.bkrepo.repository.api.ProjectClient
import com.tencent.bkrepo.repository.pojo.project.ProjectCreateRequest
import com.tencent.bkrepo.repository.pojo.project.ProjectInfo
import com.tencent.bkrepo.repository.pojo.project.ProjectMetricsInfo
import com.tencent.bkrepo.repository.pojo.project.ProjectRangeQueryRequest
import com.tencent.bkrepo.repository.service.repo.ProjectService
import org.springframework.web.bind.annotation.RestController
Expand Down Expand Up @@ -64,4 +65,8 @@ class ProjectController(
override fun createProject(request: ProjectCreateRequest): Response<ProjectInfo> {
return ResponseBuilder.success(projectService.createProject(request))
}

override fun getProjectMetrics(name: String): Response<ProjectMetricsInfo?> {
return ResponseBuilder.success(projectService.getProjectMetricsInfo(name))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import com.tencent.bkrepo.common.service.util.ResponseBuilder
import com.tencent.bkrepo.repository.pojo.project.ProjectCreateRequest
import com.tencent.bkrepo.repository.pojo.project.ProjectInfo
import com.tencent.bkrepo.repository.pojo.project.ProjectListOption
import com.tencent.bkrepo.repository.pojo.project.ProjectMetricsInfo
import com.tencent.bkrepo.repository.pojo.project.ProjectSearchOption
import com.tencent.bkrepo.repository.pojo.project.ProjectUpdateRequest
import com.tencent.bkrepo.repository.pojo.project.UserProjectCreateRequest
Expand Down Expand Up @@ -141,4 +142,14 @@ class UserProjectController(
): Response<Void> {
return this.createProject(userId, userProjectRequest)
}

@ApiOperation("项目仓库统计信息列表")
@GetMapping("/metrics/{projectId}")
fun projectMetricsList(
@ApiParam(value = "项目ID", required = true)
@PathVariable projectId: String
): Response<ProjectMetricsInfo?> {
permissionManager.checkProjectPermission(PermissionAction.READ, projectId)
return ResponseBuilder.success(projectService.getProjectMetricsInfo(projectId))
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
* A copy of the MIT License is included in this file.
*
*
* Terms of the MIT License:
* ---------------------------------------------------
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/

package com.tencent.bkrepo.repository.dao.repository

import com.tencent.bkrepo.repository.model.TProjectMetrics
import org.springframework.data.mongodb.repository.MongoRepository
import org.springframework.stereotype.Repository

@Repository
interface ProjectMetricsRepository : MongoRepository<TProjectMetrics, String>{
fun findByProjectId(projectId:String): TProjectMetrics?
}
Loading

0 comments on commit 1d869df

Please sign in to comment.