From 6b72ef7f169d3e1c0c8ec130d3bba35b4f466892 Mon Sep 17 00:00:00 2001 From: zacYL <100330102+zacYL@users.noreply.github.com> Date: Thu, 25 Jul 2024 11:31:40 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E6=81=A2=E5=A4=8D=E5=8A=9F=E8=83=BD=20#2337?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: 增加自动恢复功能 #2337 * feat: 统计任务锁时间变更 #2337 * feat: 代码调整#2337 * feat: 引用清理调整#2337 * feat: 删除多余代码#2337 * feat: package不进行最后修改时间判断#2337 --- .../common/artifact/event/base/EventType.kt | 1 + .../event/node/NodeSeparationRecoveryEvent.kt | 49 +++++ src/backend/job/biz-job/build.gradle.kts | 1 + .../com/tencent/bkrepo/job/Constants.kt | 1 - .../ActiveProjectEmptyFolderCleanupJob.kt | 2 +- .../stat/ActiveProjectNodeFolderStatJob.kt | 2 +- .../InactiveProjectEmptyFolderCleanupJob.kt | 2 +- .../stat/InactiveProjectNodeFolderStatJob.kt | 2 +- .../task/stat/ProjectRepoMetricsStatJob.kt | 2 +- .../properties/DataSeparationJobProperties.kt | 2 +- .../separation/config/DataSeparationConfig.kt | 2 + .../job/separation/dao/SeparationNodeDao.kt | 9 + .../SeparationRecoveryEventConsumer.kt | 196 ++++++++++++++++++ .../job/separation/pojo/RecoveryNodeInfo.kt | 35 ++++ .../separation/pojo/RecoveryVersionInfo.kt | 35 ++++ .../service/RepoSpecialDataSeparator.kt | 7 + .../service/impl/AbstractHandler.kt | 7 +- .../service/impl/DataSeparatorImpl.kt | 5 +- .../service/impl/SeparationTaskServiceImpl.kt | 42 ++-- .../MavenRepoSpecialDataSeparatorHandler.kt | 47 +++-- .../repo/RepoSpecialSeparationMappings.kt | 9 + .../job/separation/util/SeparationUtils.kt | 6 + .../exception/MavenArtifactFormatException.kt | 0 .../bkrepo/maven/pojo/MavenRepoConf.kt | 0 .../tencent/bkrepo/maven/pojo/MavenVersion.kt | 0 .../bkrepo/maven/util/MavenGAVCUtils.kt | 0 .../bkrepo/maven/util/MavenStringUtils.kt | 28 ++- .../tencent/bkrepo/maven/util/MavenUtil.kt | 72 +++++++ .../maven/artifact/MavenArtifactConfigurer.kt | 3 + .../repository/MavenLocalRepository.kt | 56 ++++- .../bkrepo/maven/config/MavenProperties.kt | 38 ++++ .../tencent/bkrepo/maven/util/MavenUtil.kt | 45 ---- 32 files changed, 595 insertions(+), 111 deletions(-) create mode 100644 src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/node/NodeSeparationRecoveryEvent.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/listener/SeparationRecoveryEventConsumer.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/pojo/RecoveryNodeInfo.kt create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/pojo/RecoveryVersionInfo.kt rename src/backend/maven/{biz-maven => api-maven}/src/main/kotlin/com/tencent/bkrepo/maven/exception/MavenArtifactFormatException.kt (100%) rename src/backend/maven/{biz-maven => api-maven}/src/main/kotlin/com/tencent/bkrepo/maven/pojo/MavenRepoConf.kt (100%) rename src/backend/maven/{biz-maven => api-maven}/src/main/kotlin/com/tencent/bkrepo/maven/pojo/MavenVersion.kt (100%) rename src/backend/maven/{biz-maven => api-maven}/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenGAVCUtils.kt (100%) rename src/backend/maven/{biz-maven => api-maven}/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenStringUtils.kt (85%) create mode 100644 src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenUtil.kt create mode 100644 src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/config/MavenProperties.kt delete mode 100644 src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenUtil.kt diff --git a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/base/EventType.kt b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/base/EventType.kt index 8898954126..cfa36010c8 100644 --- a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/base/EventType.kt +++ b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/base/EventType.kt @@ -51,6 +51,7 @@ enum class EventType(val msgKey: String) { NODE_DOWNLOADED("artifact.event.node-downloaded"), NODE_CLEAN("artifact.event.node-clean"), NODE_UPDATE_ACCESS_DATE("artifact.event.node-update-access-date"), + NODE_SEPARATION_RECOVERY("artifact.event.node-separation-recovery"), // METADATA METADATA_DELETED("artifact.event.metadata-deleted"), diff --git a/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/node/NodeSeparationRecoveryEvent.kt b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/node/NodeSeparationRecoveryEvent.kt new file mode 100644 index 0000000000..bd61159481 --- /dev/null +++ b/src/backend/common/common-artifact/artifact-api/src/main/kotlin/com/tencent/bkrepo/common/artifact/event/node/NodeSeparationRecoveryEvent.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.artifact.event.node + +import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.common.artifact.event.base.EventType + +/** + * 将冷节点自动恢复 + */ +class NodeSeparationRecoveryEvent( + override val projectId: String, + override val repoName: String, + override val resourceKey: String, + override val userId: String, + val repoType: String, +) : ArtifactEvent( + type = EventType.NODE_SEPARATION_RECOVERY, + projectId = projectId, + repoName = repoName, + resourceKey = resourceKey, + userId = userId, + data = mapOf("repoType" to repoType) +) diff --git a/src/backend/job/biz-job/build.gradle.kts b/src/backend/job/biz-job/build.gradle.kts index fcc09deb21..f38fdd1334 100644 --- a/src/backend/job/biz-job/build.gradle.kts +++ b/src/backend/job/biz-job/build.gradle.kts @@ -39,6 +39,7 @@ dependencies { implementation(project(":repository:api-repository")) implementation(project(":helm:api-helm")) implementation(project(":oci:api-oci")) + implementation(project(":maven:api-maven")) implementation(project(":replication:api-replication")) implementation(project(":archive:api-archive")) implementation(project(":common:common-operate:operate-service")) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt index f6af4c7aeb..e552aed048 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/Constants.kt @@ -106,7 +106,6 @@ const val SEPARATE = "SEPARATE" const val PACKAGE_COLLECTION_NAME = "package" const val PACKAGE_VERSION_COLLECTION_NAME = "package_version" const val PACKAGE_DOWNLOADS_COLLECTION_NAME = "package_downloads" -const val FILE_REFERENCE_COLLECTION_NAME = "file_reference" const val SEPARATION_TASK_COLLECTION_NAME = "separation_task" const val PACKAGE_VERSION = "version" diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt index 237d4d6583..68921d03c9 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectEmptyFolderCleanupJob.kt @@ -95,7 +95,7 @@ class ActiveProjectEmptyFolderCleanupJob( } override fun getLockAtMostFor(): Duration { - return Duration.ofDays(1) + return Duration.ofDays(14) } override fun createJobContext(): EmptyFolderCleanupJobContext { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt index 02733d070a..cb5b9fe17d 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ActiveProjectNodeFolderStatJob.kt @@ -132,7 +132,7 @@ class ActiveProjectNodeFolderStatJob( /** * 最长加锁时间 */ - override fun getLockAtMostFor(): Duration = Duration.ofDays(1) + override fun getLockAtMostFor(): Duration = Duration.ofDays(14) companion object { private val logger = LoggerFactory.getLogger(ActiveProjectNodeFolderStatJob::class.java) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt index e356c054aa..f45baa1d15 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectEmptyFolderCleanupJob.kt @@ -108,7 +108,7 @@ class InactiveProjectEmptyFolderCleanupJob( } override fun getLockAtMostFor(): Duration { - return Duration.ofDays(1) + return Duration.ofDays(14) } override fun createJobContext(): EmptyFolderCleanupJobContext { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt index 5d6158ef8a..c7bc361dbe 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/InactiveProjectNodeFolderStatJob.kt @@ -83,7 +83,7 @@ class InactiveProjectNodeFolderStatJob( /** * 最长加锁时间 */ - override fun getLockAtMostFor(): Duration = Duration.ofDays(1) + override fun getLockAtMostFor(): Duration = Duration.ofDays(14) fun statProjectCheck( projectId: String, diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ProjectRepoMetricsStatJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ProjectRepoMetricsStatJob.kt index e981e8a070..23a5d1e7eb 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ProjectRepoMetricsStatJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/stat/ProjectRepoMetricsStatJob.kt @@ -130,7 +130,7 @@ open class ProjectRepoMetricsStatJob( } override fun getLockAtMostFor(): Duration { - return Duration.ofDays(1) + return Duration.ofDays(14) } override fun onRunCollectionFinished(collectionName: String, context: JobContext) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/DataSeparationJobProperties.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/DataSeparationJobProperties.kt index 975d5349d2..040957dd3c 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/DataSeparationJobProperties.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/properties/DataSeparationJobProperties.kt @@ -36,5 +36,5 @@ class DataSeparationJobProperties( override var enabled: Boolean = false, override var cron: String = Scheduled.CRON_DISABLED, // 任务处于running 状态超过多久没有更新数据，则判断任务已经中断 - var waitTime: Duration = Duration.ofMillis(60), + var waitTime: Duration = Duration.ofMinutes(120) ) : MongodbJobProperties(enabled) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/config/DataSeparationConfig.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/config/DataSeparationConfig.kt index 2b61037a42..0a5eeb7ca6 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/config/DataSeparationConfig.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/config/DataSeparationConfig.kt @@ -69,6 +69,8 @@ data class DataSeparationConfig( var restoreTaskConcurrency: Int = Runtime.getRuntime().availableProcessors() * 2, // 允许同时执行的fix任务数 var fixTaskConcurrency: Int = Runtime.getRuntime().availableProcessors() * 2, + // 是否允许自动恢复 + var enableAutoRecovery: Boolean = false ) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/dao/SeparationNodeDao.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/dao/SeparationNodeDao.kt index 7d55671cf6..5317c1a3cb 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/dao/SeparationNodeDao.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/dao/SeparationNodeDao.kt @@ -64,6 +64,15 @@ import java.time.LocalDateTime @Repository class SeparationNodeDao : MonthRangeShardingMongoDao() { + fun findOneByFullPath( + projectId: String, repoName: String, + fullPath: String, separationDate: LocalDateTime + ): TSeparationNode? { + return this.findOne(SeparationQueryHelper.fullPathQuery( + projectId, repoName, fullPath, separationDate + )) + } + fun findOne( projectId: String, repoName: String, versionPath: String, separationDate: LocalDateTime diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/listener/SeparationRecoveryEventConsumer.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/listener/SeparationRecoveryEventConsumer.kt new file mode 100644 index 0000000000..e37c88fe9a --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/listener/SeparationRecoveryEventConsumer.kt @@ -0,0 +1,196 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.separation.listener + +import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent +import com.tencent.bkrepo.common.artifact.event.base.EventType +import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.job.RESTORE +import com.tencent.bkrepo.job.separation.config.DataSeparationConfig +import com.tencent.bkrepo.job.separation.dao.SeparationNodeDao +import com.tencent.bkrepo.job.separation.dao.SeparationPackageDao +import com.tencent.bkrepo.job.separation.dao.SeparationPackageVersionDao +import com.tencent.bkrepo.job.separation.model.TSeparationNode +import com.tencent.bkrepo.job.separation.model.TSeparationPackageVersion +import com.tencent.bkrepo.job.separation.pojo.NodeFilterInfo +import com.tencent.bkrepo.job.separation.pojo.PackageFilterInfo +import com.tencent.bkrepo.job.separation.pojo.RecoveryNodeInfo +import com.tencent.bkrepo.job.separation.pojo.RecoveryVersionInfo +import com.tencent.bkrepo.job.separation.pojo.SeparationContent +import com.tencent.bkrepo.job.separation.pojo.task.SeparationTaskRequest +import com.tencent.bkrepo.job.separation.service.SeparationTaskService +import com.tencent.bkrepo.job.separation.service.impl.repo.RepoSpecialSeparationMappings +import org.slf4j.LoggerFactory +import org.springframework.messaging.Message +import org.springframework.stereotype.Component +import java.time.format.DateTimeFormatter +import java.util.function.Consumer + +/** + * 消费降冷自动恢复事件 + */ +@Component("separationRecovery") +class SeparationRecoveryEventConsumer( + private val separationTaskService: SeparationTaskService, + private val dataSeparationConfig: DataSeparationConfig, + private val separationPackageDao: SeparationPackageDao, + private val separationPackageVersionDao: SeparationPackageVersionDao, + private val separationNodeDao: SeparationNodeDao +) : Consumer> { + + /** + * 允许接收的事件类型 + */ + private val acceptTypes = setOf( + EventType.NODE_SEPARATION_RECOVERY, + ) + + override fun accept(message: Message) { + if (!dataSeparationConfig.enableAutoRecovery) return + if (!acceptTypes.contains(message.payload.type)) { + return + } + logger.info("current separation recovery message header is ${message.headers}") + doSeparationRecovery(message.payload) + } + + private fun doSeparationRecovery(event: ArtifactEvent) { + val recoveryNodeInfo = buildRecoveryNodeInfo(event) + val task = when (recoveryNodeInfo.repoType) { + RepositoryType.MAVEN.name -> { + val recoveryVersionInfo = RepoSpecialSeparationMappings.getRecoveryPackageVersionData(recoveryNodeInfo) + val version = recoveryVersionCheck(recoveryVersionInfo) ?: return + buildVersionSeparationTaskRequest(recoveryVersionInfo, version) + } + RepositoryType.GENERIC.name -> { + val node = recoveryNodeCheck(recoveryNodeInfo) ?: return + buildNodeSeparationTaskRequest(recoveryNodeInfo, node) + } + else -> { + null + } + } + if (task != null) { + separationTaskService.createSeparationTask(task) + } + } + + private fun buildRecoveryNodeInfo(event: ArtifactEvent): RecoveryNodeInfo { + return RecoveryNodeInfo( + projectId = event.projectId, + repoName = event.repoName, + fullPath = event.resourceKey, + repoType = event.data["repoType"].toString() + ) + } + + private fun recoveryNodeCheck(recoveryNodeInfo: RecoveryNodeInfo): TSeparationNode? { + with(recoveryNodeInfo) { + val separateDates = separationTaskService.findDistinctSeparationDate(projectId, repoName) + if (separateDates.isEmpty()) return null + separateDates.forEach { + val separationNode = separationNodeDao.findOneByFullPath(projectId, repoName, fullPath, it) + if (separationNode != null) { + return separationNode + } + } + return null + } + } + + private fun recoveryVersionCheck(recoveryVersionInfo: RecoveryVersionInfo): TSeparationPackageVersion? { + with(recoveryVersionInfo) { + val separateDates = separationTaskService.findDistinctSeparationDate(projectId, repoName) + if (separateDates.isEmpty()) return null + val separationPackage = separationPackageDao.findByKey(projectId, repoName, packageKey) ?: return null + separateDates.forEach { + val version = separationPackageVersionDao.findByName(separationPackage.id!!, version, it) + if (version != null) { + return version + } + } + return null + } + } + + private fun buildVersionSeparationTaskRequest( + recoveryVersionInfo: RecoveryVersionInfo, + separationPackageVersion: TSeparationPackageVersion + ): SeparationTaskRequest { + return SeparationTaskRequest( + projectId = recoveryVersionInfo.projectId, + repoName = recoveryVersionInfo.repoName, + type = RESTORE, + separateAt = separationPackageVersion.separationDate.format(DateTimeFormatter.ISO_DATE_TIME), + content = buildSeparationContent(recoveryVersionInfo.packageKey, separationPackageVersion.name)!! + ) + } + + private fun buildNodeSeparationTaskRequest( + recoveryNodeInfo: RecoveryNodeInfo, + separationNode: TSeparationNode + ): SeparationTaskRequest { + return SeparationTaskRequest( + projectId = recoveryNodeInfo.projectId, + repoName = recoveryNodeInfo.repoName, + type = RESTORE, + separateAt = separationNode.separationDate.format(DateTimeFormatter.ISO_DATE_TIME), + content = buildSeparationContent(fullPath = separationNode.fullPath)!! + ) + } + + private fun buildSeparationContent( + packageKey: String? = null, + version: String? = null, + fullPath: String? = null + ): SeparationContent? { + if (packageKey.isNullOrEmpty() && version.isNullOrEmpty() && fullPath.isNullOrEmpty()) return null + return if (fullPath.isNullOrEmpty()) { + SeparationContent( + packages = mutableListOf( + PackageFilterInfo( + packageKey = packageKey, + versions = listOf(version!!) + ) + ) + ) + } else { + SeparationContent( + paths = mutableListOf( + NodeFilterInfo( + path = fullPath + ) + ) + ) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(SeparationRecoveryEventConsumer::class.java) + } +} \ No newline at end of file diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/pojo/RecoveryNodeInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/pojo/RecoveryNodeInfo.kt new file mode 100644 index 0000000000..4e05e3e3d3 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/pojo/RecoveryNodeInfo.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.separation.pojo + +data class RecoveryNodeInfo( + val projectId: String, + val repoName: String, + val fullPath: String, + val repoType: String, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/pojo/RecoveryVersionInfo.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/pojo/RecoveryVersionInfo.kt new file mode 100644 index 0000000000..4e1bde5269 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/pojo/RecoveryVersionInfo.kt @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.separation.pojo + +data class RecoveryVersionInfo( + val projectId: String, + val repoName: String, + val packageKey: String, + val version: String, +) diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/RepoSpecialDataSeparator.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/RepoSpecialDataSeparator.kt index 904dbf3a29..2bba321151 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/RepoSpecialDataSeparator.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/RepoSpecialDataSeparator.kt @@ -28,6 +28,8 @@ package com.tencent.bkrepo.job.separation.service import com.tencent.bkrepo.common.artifact.pojo.RepositoryType +import com.tencent.bkrepo.job.separation.pojo.RecoveryNodeInfo +import com.tencent.bkrepo.job.separation.pojo.RecoveryVersionInfo import com.tencent.bkrepo.job.separation.pojo.VersionSeparationInfo /** @@ -76,4 +78,9 @@ interface RepoSpecialDataSeparator { * 根据条件删除已恢复的仓库特定冷数据 */ fun removeRestoredRepoSpecialData(versionSeparationInfo: VersionSeparationInfo) + + /** + * 获取待恢复节点对应的包版本信息 + */ + fun getRecoveryPackageVersionData(recoveryInfo: RecoveryNodeInfo): RecoveryVersionInfo } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt index 9b46870b05..588ea5dfd4 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt @@ -31,7 +31,6 @@ import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 -import com.tencent.bkrepo.job.FILE_REFERENCE_COLLECTION_NAME import com.tencent.bkrepo.job.separation.dao.SeparationFailedRecordDao import com.tencent.bkrepo.job.separation.dao.SeparationTaskDao import com.tencent.bkrepo.job.separation.pojo.NodeFilterInfo @@ -42,6 +41,7 @@ import com.tencent.bkrepo.job.separation.pojo.record.SeparationContext import com.tencent.bkrepo.job.separation.pojo.record.SeparationProgress import com.tencent.bkrepo.job.separation.pojo.task.SeparationCount import com.tencent.bkrepo.job.separation.pojo.task.SeparationTaskState +import com.tencent.bkrepo.job.separation.util.SeparationUtils.getFileReferenceCollectionName import com.tencent.bkrepo.repository.constant.SYSTEM_USER import org.slf4j.LoggerFactory import org.springframework.dao.DuplicateKeyException @@ -113,15 +113,16 @@ open class AbstractHandler( } fun increment(sha256: String, credentialsKey: String?, inc: Long) { + val collectionName = getFileReferenceCollectionName(sha256) val criteria = Criteria.where(FileReferenceInfo::sha256.name).`is`(sha256) .and(FileReferenceInfo::credentialsKey.name).`is`(credentialsKey) val query = Query(criteria) val update = Update().inc(FileReferenceInfo::count.name, inc) try { - mongoTemplate.upsert(query, update, FILE_REFERENCE_COLLECTION_NAME) + mongoTemplate.upsert(query, update, collectionName) } catch (exception: DuplicateKeyException) { // retry because upsert operation is not atomic - mongoTemplate.upsert(query, update, FILE_REFERENCE_COLLECTION_NAME) + mongoTemplate.upsert(query, update, collectionName) } logger.info("Increment hot node reference [$inc] of file [$sha256] on credentialsKey [$credentialsKey].") } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/DataSeparatorImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/DataSeparatorImpl.kt index 2b90dde29e..e39c723677 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/DataSeparatorImpl.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/DataSeparatorImpl.kt @@ -126,7 +126,7 @@ class DataSeparatorImpl( ) { with(context) { validatePackageParams(pkg) - val criteria = buildPackageCriteria(projectId, repoName, separationDate, pkg) + val criteria = buildPackageCriteria(projectId, repoName, pkg) val pageSize = BATCH_SIZE var querySize: Int var lastId = ObjectId(MIN_OBJECT_ID) @@ -150,11 +150,10 @@ class DataSeparatorImpl( private fun buildPackageCriteria( projectId: String, repoName: String, - separationDate: LocalDateTime, pkg: PackageFilterInfo? + pkg: PackageFilterInfo? ): Criteria { val criteria = Criteria.where(PROJECT_ID).isEqualTo(projectId) .and(REPO_NAME).isEqualTo(repoName) - .and(LAST_MODIFIED_DATE).lte(separationDate) if (pkg != null) { if (pkg.packageKey.isNullOrEmpty()) { if (!pkg.packageKeyRegex.isNullOrEmpty()) { diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt index a9f36102b2..3323c81257 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/SeparationTaskServiceImpl.kt @@ -80,19 +80,13 @@ class SeparationTaskServiceImpl( separationTaskDao.save(task) } RESTORE -> { - val separatedDates = findDistinctSeparationDate(projectId, repoName) - if (separatedDates.isEmpty()) { - logger.warn("no cold data has been stored in $projectId|$repoName") - throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, SeparationTaskRequest::type.name) - } - separatedDates.forEach { - val task = buildSeparationTask(request, it) - separationTaskDao.save(task) - } + createRestoreTask(request) } else -> { logger.warn("unsupported task type $type") - throw BadRequestException(CommonMessageCode.PARAMETER_INVALID, SeparationTaskRequest::type.name) + throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, SeparationTaskRequest::type.name + ) } } @@ -130,6 +124,28 @@ class SeparationTaskServiceImpl( return exist || failedExist } + private fun createRestoreTask(request: SeparationTaskRequest) { + with(request) { + if (separateAt.isNullOrEmpty()) { + val separatedDates = findDistinctSeparationDate(projectId, repoName) + if (separatedDates.isEmpty()) { + logger.warn("no cold data has been stored in $projectId|$repoName") + throw BadRequestException( + CommonMessageCode.PARAMETER_INVALID, SeparationTaskRequest::type.name + ) + } + separatedDates.forEach { + val task = buildSeparationTask(request, it) + separationTaskDao.save(task) + } + } else { + val date = LocalDateTime.parse(separateAt, DateTimeFormatter.ISO_DATE_TIME) + val task = buildSeparationTask(request, date) + separationTaskDao.save(task) + } + } + } + private fun getRepoInfo(projectId: String, repoName: String): RepositoryDetail { val repo = repositoryClient.getRepoDetail(projectId, repoName).data ?: run { @@ -194,10 +210,12 @@ class SeparationTaskServiceImpl( } private fun buildSeparationTask( - request: SeparationTaskRequest, restoreDate: LocalDateTime? = null + request: SeparationTaskRequest, + restoreDate: LocalDateTime? = null, + userId: String = SecurityUtils.getUserId() ): TSeparationTask { with(request) { - val userId = SecurityUtils.getUserId() + val userId = userId val date = restoreDate ?: LocalDateTime.parse(separateAt, DateTimeFormatter.ISO_DATE_TIME) val separateDate = LocalDateTime.of(date.toLocalDate(), LocalTime.MAX) return TSeparationTask( diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/MavenRepoSpecialDataSeparatorHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/MavenRepoSpecialDataSeparatorHandler.kt index 07dee6fbab..9ab5e315f6 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/MavenRepoSpecialDataSeparatorHandler.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/MavenRepoSpecialDataSeparatorHandler.kt @@ -27,7 +27,6 @@ package com.tencent.bkrepo.job.separation.service.impl.repo -import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.util.PackageKeys @@ -38,6 +37,8 @@ import com.tencent.bkrepo.job.separation.dao.SeparationNodeDao import com.tencent.bkrepo.job.separation.dao.repo.SeparationMavenMetadataDao import com.tencent.bkrepo.job.separation.model.TSeparationNode import com.tencent.bkrepo.job.separation.model.repo.TSeparationMavenMetadataRecord +import com.tencent.bkrepo.job.separation.pojo.RecoveryNodeInfo +import com.tencent.bkrepo.job.separation.pojo.RecoveryVersionInfo import com.tencent.bkrepo.job.separation.pojo.VersionSeparationInfo import com.tencent.bkrepo.job.separation.pojo.query.MavenMetadata import com.tencent.bkrepo.job.separation.pojo.query.NodeBaseInfo @@ -45,7 +46,10 @@ import com.tencent.bkrepo.job.separation.pojo.query.NodeDetailInfo import com.tencent.bkrepo.job.separation.service.RepoSpecialDataSeparator import com.tencent.bkrepo.job.separation.util.SeparationUtils import com.tencent.bkrepo.job.separation.util.SeparationUtils.getNodeCollectionName -import org.apache.commons.lang3.StringUtils +import com.tencent.bkrepo.maven.util.MavenGAVCUtils.toMavenGAVC +import com.tencent.bkrepo.maven.util.MavenStringUtils.formatSeparator +import com.tencent.bkrepo.maven.util.MavenUtil.extractGroupIdAndArtifactId +import com.tencent.bkrepo.maven.util.MavenUtil.extractPath import org.bson.types.ObjectId import org.slf4j.LoggerFactory import org.springframework.data.domain.PageRequest @@ -104,7 +108,7 @@ class MavenRepoSpecialDataSeparatorHandler( accessCheck: Boolean ): MutableMap { with(versionSeparationInfo) { - val packagePath = extractPath(packageKey) + val packagePath = PathUtils.normalizeFullPath(extractPath(packageKey)) val versionPath = PathUtils.combinePath(packagePath, version) val nodeCollectionName = getNodeCollectionName(projectId) // 目录节点保持不变，空目录节点清理job会对降冷的仓库禁用 @@ -142,7 +146,7 @@ class MavenRepoSpecialDataSeparatorHandler( override fun getRestoreNodesOfVersion(versionSeparationInfo: VersionSeparationInfo): MutableMap { with(versionSeparationInfo) { - val packagePath = extractPath(packageKey) + val packagePath = PathUtils.normalizeFullPath(extractPath(packageKey)) val versionPath = PathUtils.combinePath(packagePath, version) var pageNumber = 0 val pageSize = BATCH_SIZE @@ -203,6 +207,22 @@ class MavenRepoSpecialDataSeparatorHandler( } } + override fun getRecoveryPackageVersionData(recoveryInfo: RecoveryNodeInfo): RecoveryVersionInfo { + with(recoveryInfo) { + val mavenGAVC = fullPath.toMavenGAVC() + val version = mavenGAVC.version + val artifactId = mavenGAVC.artifactId + val groupId = mavenGAVC.groupId.formatSeparator("/", ".") + val packageKey = PackageKeys.ofGav(groupId, artifactId) + return RecoveryVersionInfo( + projectId = projectId, + repoName = repoName, + packageKey = packageKey, + version = version + ) + } + } + private fun pathQuery( projectId: String, repoName: String, versionPath: String, separationDate: LocalDateTime ): Query { @@ -271,25 +291,6 @@ class MavenRepoSpecialDataSeparatorHandler( } } - /** - * 提取出对应的artifactId和groupId - */ - private fun extractGroupIdAndArtifactId(packageKey: String): Pair { - val params = PackageKeys.resolveGav(packageKey) - val artifactId = params.split(":").last() - val groupId = params.split(":").first() - return Pair(artifactId, groupId) - } - - /** - * 获取对应package存储的节点路径 - */ - private fun extractPath(packageKey: String): String { - val (artifactId, groupId) = extractGroupIdAndArtifactId(packageKey) - return PathUtils.UNIX_SEPARATOR + - StringUtils.join(groupId.split(CharPool.DOT), PathUtils.UNIX_SEPARATOR) + "/$artifactId" - } - companion object { private val logger = LoggerFactory.getLogger(MavenRepoSpecialDataSeparatorHandler::class.java) private const val MAVEN_METADATA_COLLECTION_NAME = "maven_metadata" diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/RepoSpecialSeparationMappings.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/RepoSpecialSeparationMappings.kt index 8fb099c39a..ab21f4ce7b 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/RepoSpecialSeparationMappings.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/repo/RepoSpecialSeparationMappings.kt @@ -29,6 +29,8 @@ package com.tencent.bkrepo.job.separation.service.impl.repo import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.service.util.SpringContextUtils +import com.tencent.bkrepo.job.separation.pojo.RecoveryNodeInfo +import com.tencent.bkrepo.job.separation.pojo.RecoveryVersionInfo import com.tencent.bkrepo.job.separation.pojo.VersionSeparationInfo import com.tencent.bkrepo.job.separation.service.RepoSpecialDataSeparator @@ -82,4 +84,11 @@ object RepoSpecialSeparationMappings { check(mapper != null) { "mapper[${versionSeparationInfo.type}] not found" } mapper.removeRestoredRepoSpecialData(versionSeparationInfo) } + + fun getRecoveryPackageVersionData(recoveryInfo: RecoveryNodeInfo): RecoveryVersionInfo { + val type = RepositoryType.ofValueOrDefault(recoveryInfo.repoType) + val mapper = mappers[type] + check(mapper != null) { "mapper[${type}] not found" } + return mapper.getRecoveryPackageVersionData(recoveryInfo) + } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/util/SeparationUtils.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/util/SeparationUtils.kt index d80ae08377..463806c0c6 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/util/SeparationUtils.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/util/SeparationUtils.kt @@ -38,12 +38,18 @@ import java.util.concurrent.TimeUnit object SeparationUtils { private const val NODE_COLLECTION_PREFIX = "node_" + private const val FILE_REFERENCE_COLLECTION_NAME_PREFIX = "file_reference_" fun getNodeCollectionName(projectId: String): String { return NODE_COLLECTION_PREFIX + HashShardingUtils.shardingSequenceFor(projectId, SHARDING_COUNT).toString() } + fun getFileReferenceCollectionName(sha256: String): String { + return FILE_REFERENCE_COLLECTION_NAME_PREFIX + + HashShardingUtils.shardingSequenceFor(sha256, SHARDING_COUNT).toString() + } + /** * 创建线程池 */ diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/exception/MavenArtifactFormatException.kt b/src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/exception/MavenArtifactFormatException.kt similarity index 100% rename from src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/exception/MavenArtifactFormatException.kt rename to src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/exception/MavenArtifactFormatException.kt diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/pojo/MavenRepoConf.kt b/src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/pojo/MavenRepoConf.kt similarity index 100% rename from src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/pojo/MavenRepoConf.kt rename to src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/pojo/MavenRepoConf.kt diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/pojo/MavenVersion.kt b/src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/pojo/MavenVersion.kt similarity index 100% rename from src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/pojo/MavenVersion.kt rename to src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/pojo/MavenVersion.kt diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenGAVCUtils.kt b/src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenGAVCUtils.kt similarity index 100% rename from src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenGAVCUtils.kt rename to src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenGAVCUtils.kt diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenStringUtils.kt b/src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenStringUtils.kt similarity index 85% rename from src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenStringUtils.kt rename to src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenStringUtils.kt index 889c838d77..2f9ce47354 100644 --- a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenStringUtils.kt +++ b/src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenStringUtils.kt @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -10,23 +10,19 @@ * * Terms of the MIT License: * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ package com.tencent.bkrepo.maven.util diff --git a/src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenUtil.kt b/src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenUtil.kt new file mode 100644 index 0000000000..33192d1609 --- /dev/null +++ b/src/backend/maven/api-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenUtil.kt @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.maven.util + +import com.google.common.io.ByteStreams +import com.google.common.io.CharStreams +import com.tencent.bkrepo.common.artifact.util.PackageKeys +import org.apache.commons.lang3.StringUtils +import java.io.InputStream +import java.io.InputStreamReader +import java.nio.charset.StandardCharsets + +object MavenUtil { + private const val MAX_DIGEST_CHARS_NEEDED = 128 + + /** + * 从流中导出摘要 + * */ + fun extractDigest(inputStream: InputStream): String { + inputStream.use { + val reader = InputStreamReader( + ByteStreams + .limit(inputStream, MAX_DIGEST_CHARS_NEEDED.toLong()), + StandardCharsets.UTF_8 + ) + return CharStreams.toString(reader) + } + } + + /** + * 提取出对应的artifactId和groupId + */ + fun extractGroupIdAndArtifactId(packageKey: String): Pair { + val params = PackageKeys.resolveGav(packageKey) + val artifactId = params.split(":").last() + val groupId = params.split(":").first() + return Pair(artifactId, groupId) + } + + /** + * 获取对应package存储的节点路径 + */ + fun extractPath(packageKey: String): String { + val (artifactId, groupId) = extractGroupIdAndArtifactId(packageKey) + return StringUtils.join(groupId.split("."), "/") + "/$artifactId" + } +} diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/MavenArtifactConfigurer.kt b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/MavenArtifactConfigurer.kt index b8e87e866c..be460559f6 100644 --- a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/MavenArtifactConfigurer.kt +++ b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/MavenArtifactConfigurer.kt @@ -38,12 +38,15 @@ import com.tencent.bkrepo.common.service.util.SpringContextUtils import com.tencent.bkrepo.maven.artifact.repository.MavenLocalRepository import com.tencent.bkrepo.maven.artifact.repository.MavenRemoteRepository import com.tencent.bkrepo.maven.artifact.repository.MavenVirtualRepository +import com.tencent.bkrepo.maven.config.MavenProperties +import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.stereotype.Component /** * Maven 依赖源配置 */ @Component +@EnableConfigurationProperties(MavenProperties::class) class MavenArtifactConfigurer : ArtifactConfigurerSupport() { override fun getRepositoryType() = RepositoryType.MAVEN diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt index 0f47b7153e..00583ee35e 100644 --- a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt +++ b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/artifact/repository/MavenLocalRepository.kt @@ -36,6 +36,7 @@ import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.util.toJsonString import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.api.ArtifactInfo +import com.tencent.bkrepo.common.artifact.event.node.NodeSeparationRecoveryEvent import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContext @@ -54,6 +55,7 @@ import com.tencent.bkrepo.common.service.util.HeaderUtils import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.maven.artifact.MavenArtifactInfo import com.tencent.bkrepo.maven.artifact.MavenDeleteArtifactInfo +import com.tencent.bkrepo.maven.config.MavenProperties import com.tencent.bkrepo.maven.constants.FULL_PATH import com.tencent.bkrepo.maven.constants.MAVEN_METADATA_FILE_NAME import com.tencent.bkrepo.maven.constants.METADATA_KEY_ARTIFACT_ID @@ -128,7 +130,8 @@ import java.util.regex.Pattern @Component class MavenLocalRepository( private val stageClient: StageClient, - private val mavenMetadataService: MavenMetadataService + private val mavenMetadataService: MavenMetadataService, + private val mavenProperties: MavenProperties, ) : LocalRepository() { @Value("\${maven.domain:http://127.0.0.1:25803}") @@ -710,7 +713,20 @@ class MavenLocalRepository( */ override fun onDownload(context: ArtifactDownloadContext): ArtifactResource? { with(context) { - val node = getNodeInfoForDownload(context) + val originalFullPath = artifactInfo.getArtifactFullPath() + val node = try { + getNodeInfoForDownload(context) + } catch (e: MavenArtifactNotFoundException) { + sendSeparationRecoveryEvent( + artifactInfo.projectId, artifactInfo.repoName, originalFullPath, context.repo.type.name + ) + throw e + } + if (node == null) { + sendSeparationRecoveryEvent( + artifactInfo.projectId, artifactInfo.repoName, originalFullPath, context.repo.type.name + ) + } node?.metadata?.get(HashType.SHA1.ext)?.let { response.addHeader(X_CHECKSUM_SHA1, it.toString()) } @@ -1305,6 +1321,42 @@ class MavenLocalRepository( } } + private fun sendSeparationRecoveryEvent( + projectId: String, + repoName: String, + fullPath: String, + repoType: String, + ) { + if (!mavenProperties.autoRecovery) return + if (mavenProperties.recoveryTopic.isNullOrEmpty()) return + val event = buildNodeSeparationRecoveryEvent( + projectId = projectId, + repoName = repoName, + fullPath = fullPath, + repoType = repoType + ) + messageSupplier.delegateToSupplier( + data = event, + topic = mavenProperties.recoveryTopic!!, + key = event.getFullResourceKey(), + ) + } + + private fun buildNodeSeparationRecoveryEvent( + projectId: String, + repoName: String, + fullPath: String, + repoType: String, + ): NodeSeparationRecoveryEvent { + return NodeSeparationRecoveryEvent( + projectId = projectId, + repoName = repoName, + resourceKey = fullPath, + userId = SecurityUtils.getUserId(), + repoType = repoType + ) + } + companion object { private val logger: Logger = LoggerFactory.getLogger(MavenLocalRepository::class.java) private val formatter: DateTimeFormatter = DateTimeFormatter.ofPattern("yyyyMMdd.HHmmss") diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/config/MavenProperties.kt b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/config/MavenProperties.kt new file mode 100644 index 0000000000..e21d0d476b --- /dev/null +++ b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/config/MavenProperties.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.maven.config + +import org.springframework.boot.context.properties.ConfigurationProperties + +@ConfigurationProperties(prefix = "maven") +data class MavenProperties( + // 是否从冷表中进行恢复 + var autoRecovery: Boolean = false, + // 处理节点恢复事件topic + var recoveryTopic: String? = null +) diff --git a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenUtil.kt b/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenUtil.kt deleted file mode 100644 index 684b988d3c..0000000000 --- a/src/backend/maven/biz-maven/src/main/kotlin/com/tencent/bkrepo/maven/util/MavenUtil.kt +++ /dev/null @@ -1,45 +0,0 @@ -package com.tencent.bkrepo.maven.util - -import com.google.common.io.ByteStreams -import com.google.common.io.CharStreams -import com.tencent.bkrepo.common.artifact.util.PackageKeys -import org.apache.commons.lang3.StringUtils -import java.io.InputStream -import java.io.InputStreamReader -import java.nio.charset.StandardCharsets - -object MavenUtil { - private const val MAX_DIGEST_CHARS_NEEDED = 128 - - /** - * 从流中导出摘要 - * */ - fun extractDigest(inputStream: InputStream): String { - inputStream.use { - val reader = InputStreamReader( - ByteStreams - .limit(inputStream, MAX_DIGEST_CHARS_NEEDED.toLong()), - StandardCharsets.UTF_8 - ) - return CharStreams.toString(reader) - } - } - - /** - * 提取出对应的artifactId和groupId - */ - fun extractGroupIdAndArtifactId(packageKey: String): Pair { - val params = PackageKeys.resolveGav(packageKey) - val artifactId = params.split(":").last() - val groupId = params.split(":").first() - return Pair(artifactId, groupId) - } - - /** - * 获取对应package存储的节点路径 - */ - fun extractPath(packageKey: String): String { - val (artifactId, groupId) = extractGroupIdAndArtifactId(packageKey) - return StringUtils.join(groupId.split("."), "/") + "/$artifactId" - } -}