feat: 支持跨云迁移仓库存储 TencentBlueKing#1987

* feat: 支持为项目配置默认使用的存储凭据 TencentBlueKing#1987 * feat: 支持创建仓库数据迁移任务 TencentBlueKing#1987 * feat: 支持执行迁移任务 TencentBlueKing#1987 * feat: 支持迁移结束后矫正新增node TencentBlueKing#1987 * feat: 限制仓库迁移存储时不能清理node或修复文件引用数据 TencentBlueKing#1987 * feat: 保存迁移失败的node TencentBlueKing#1987 * feat: 保存迁移失败的node TencentBlueKing#1987 * feat: 支持重新迁移失败的node TencentBlueKing#1987 * feat: 支持定时执行处于可执行状态的迁移任务 TencentBlueKing#1987 * feat: 增加创建迁移任务接口 TencentBlueKing#1987 * feat: 增加创建迁移任务接口 TencentBlueKing#1987 * feat: 设置传输线程池满时直接在任务执行线程中传输 TencentBlueKing#1987 * feat: 支持任务中断后自动重试 TencentBlueKing#1987 * feat: 移除无用代码 TencentBlueKing#1987 * feat: 支持更新项目存储凭据为默认存储 TencentBlueKing#1987 * feat: 增加更新存储凭据单元测试 TencentBlueKing#1987 * feat: 增加待迁移制品遍历工具单元测试 TencentBlueKing#1987 * feat: 增加NodeCommonUtils单元测试 TencentBlueKing#1987 * feat: 修复任务执行结束后未从执行列表中移除 TencentBlueKing#1987 * feat: 修复迁移执行器未记录失败node，增加单元测试 TencentBlueKing#1987 * feat: 修复记录的已迁移制品数错误 TencentBlueKing#1987 * feat: 取消修改待迁移与已迁移制品数时修改最后修改时间 TencentBlueKing#1987 * feat: 拆分迁移任务执行器代码 TencentBlueKing#1987 * feat: 拆分数据传输线程池 TencentBlueKing#1987 * feat: 调整任务执行器获取方式 TencentBlueKing#1987 * feat: 修复失败节点重复迁移，调整保存失败节点方式 TencentBlueKing#1987 * feat: 调整迁移配置默认值 TencentBlueKing#1987 * feat: 修复任务状态回滚失败 TencentBlueKing#1987 * feat: 修复单元测试错误 TencentBlueKing#1987 * feat: 过滤FAKE_SHA256节点 TencentBlueKing#1987 * feat: 增加MigrateRepoStorageService测试 TencentBlueKing#1987 * feat: 增加迁移任务执行器测试 TencentBlueKing#1987 * feat: 优化迁移任务执行器代码结构，修复单元测试错误 TencentBlueKing#1987 * feat: 修复单元测试错误 TencentBlueKing#1987 * feat: 修复单元测试错误 TencentBlueKing#1987 * feat: 调整为根据nodeId遍历待迁移制品 TencentBlueKing#1987 * feat: 增加断点继续迁移测试 TencentBlueKing#1987 * feat: 增加迁移任务查询缓存 TencentBlueKing#1987 * feat: 修复单元测试错误 TencentBlueKing#1987 * feat: 修复单元测试错误 TencentBlueKing#1987 * feat: 修复单元测试错误 TencentBlueKing#1987 * feat: 修复单元测试错误 TencentBlueKing#1987 * feat: 移除迁移任务缓存 TencentBlueKing#1987
lannoy0523 · Apr 22, 2024 · 4c10420 · 4c10420
1 parent 548eb36
commit 4c10420
Show file tree

Hide file tree

Showing 60 changed files with 3,808 additions and 444 deletions.
diff --git a/...rvice/src/main/kotlin/com/tencent/bkrepo/common/service/actuator/ActuatorConfiguration.kt b/...rvice/src/main/kotlin/com/tencent/bkrepo/common/service/actuator/ActuatorConfiguration.kt
@@ -75,6 +75,6 @@ class ActuatorConfiguration {
         private const val SERVICE_NAME = "\${service.prefix:}\${spring.application.name}\${service.suffix:}"
         private const val SERVER_HOST = "\${spring.cloud.client.ip-address}"
         private const val SERVER_PORT = "\${server.port}"
-        private const val SERVICE_INSTANCE_ID = "${SERVICE_NAME}-${SERVER_PORT}-${SERVER_HOST}"
+        const val SERVICE_INSTANCE_ID = "${SERVICE_NAME}-${SERVER_PORT}-${SERVER_HOST}"
     }
 }
diff --git a/.../biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt b/.../biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt
@@ -46,6 +46,7 @@ import com.tencent.bkrepo.job.batch.context.DeletedNodeCleanupJobContext
 import com.tencent.bkrepo.job.batch.utils.MongoShardingUtils
 import com.tencent.bkrepo.job.batch.utils.TimeUtils
 import com.tencent.bkrepo.job.config.properties.DeletedNodeCleanupJobProperties
+import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService
 import com.tencent.bkrepo.repository.api.RepositoryClient
 import org.slf4j.LoggerFactory
 import org.springframework.boot.context.properties.EnableConfigurationProperties
@@ -69,7 +70,8 @@ import java.util.concurrent.TimeUnit
 class DeletedNodeCleanupJob(
     private val properties: DeletedNodeCleanupJobProperties,
     private val clusterProperties: ClusterProperties,
-    private val repositoryClient: RepositoryClient
+    private val repositoryClient: RepositoryClient,
+    private val migrateRepoStorageService: MigrateRepoStorageService,
 ) : DefaultContextMongoDbJob<DeletedNodeCleanupJob.Node>(properties) {
 
     data class Node(
@@ -122,6 +124,12 @@ class DeletedNodeCleanupJob(
 
     override fun run(row: Node, collectionName: String, context: JobContext) {
         require(context is DeletedNodeCleanupJobContext)
+        // 仓库正在迁移时删除node会导致迁移任务分页查询数据重复或缺失，需要等迁移完后再执行清理
+        if (migrateRepoStorageService.migrating(row.projectId, row.repoName)) {
+            logger.info("repo[${row.projectId}/${row.repoName}] storage was migrating, skip clean node[${row.sha256}]")
+            return
+        }
+
         if (row.folder) {
             cleanupFolderNode(context, row.id, collectionName)
         } else {

diff --git a/.../biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/MigrateRepoStorageJob.kt b/.../biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/MigrateRepoStorageJob.kt
@@ -0,0 +1,56 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2024 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.batch.task.other
+
+import com.tencent.bkrepo.job.batch.base.DefaultContextJob
+import com.tencent.bkrepo.job.batch.base.JobContext
+import com.tencent.bkrepo.job.config.properties.MigrateRepoStorageJobProperties
+import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService
+import org.slf4j.LoggerFactory
+import org.springframework.boot.context.properties.EnableConfigurationProperties
+import org.springframework.stereotype.Component
+
+@Component
+@EnableConfigurationProperties(MigrateRepoStorageJobProperties::class)
+class MigrateRepoStorageJob(
+    properties: MigrateRepoStorageJobProperties,
+    private val migrateRepoStorageService: MigrateRepoStorageService
+) : DefaultContextJob(properties) {
+    override fun doStart0(jobContext: JobContext) {
+        migrateRepoStorageService.rollbackInterruptedTaskState()
+        while (true) {
+            migrateRepoStorageService.tryExecuteTask()
+                ?.let { logger.info("execute migrate task[$it]") }
+                ?: break
+        }
+    }
+
+    companion object {
+        private val logger = LoggerFactory.getLogger(MigrateRepoStorageJob::class.java)
+    }
+}
diff --git a/...backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtils.kt b/...backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtils.kt
@@ -6,6 +6,7 @@ import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID
 import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils
 import com.tencent.bkrepo.job.BATCH_SIZE
 import com.tencent.bkrepo.job.SHARDING_COUNT
+import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService
 import org.bson.types.ObjectId
 import org.springframework.data.domain.Sort
 import java.time.LocalDateTime
@@ -23,9 +24,11 @@ import java.util.function.Consumer
 @Component
 class NodeCommonUtils(
     mongoTemplate: MongoTemplate,
+    migrateRepoStorageService: MigrateRepoStorageService,
 ) {
     init {
         Companion.mongoTemplate = mongoTemplate
+        Companion.migrateRepoStorageService = migrateRepoStorageService
     }
 
     data class Node(
@@ -41,6 +44,7 @@ class NodeCommonUtils(
 
     companion object {
         lateinit var mongoTemplate: MongoTemplate
+        lateinit var migrateRepoStorageService: MigrateRepoStorageService
         private const val COLLECTION_NAME_PREFIX = "node_"
         private val workPool = ThreadPoolExecutor(
             Runtime.getRuntime().availableProcessors(),
@@ -69,6 +73,10 @@ class NodeCommonUtils(
                 val find = mongoTemplate.find(query, Node::class.java, collection)
                     .distinctBy { it.projectId + it.repoName }
                     .any {
+                        // node正在迁移时无法判断是否存在于存储[storageCredentialsKey]上
+                        if (migrateRepoStorageService.migrating(it.projectId, it.repoName)) {
+                            throw IllegalStateException("repo[${it.projectId}/${it.repoName}] was migrating")
+                        }
                         val repo = RepositoryCommonUtils.getRepositoryDetail(it.projectId, it.repoName)
                         repo.storageCredentials?.key == storageCredentialsKey
                     }

diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/JobConfig.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/config/JobConfig.kt
@@ -29,6 +29,7 @@ package com.tencent.bkrepo.job.config
 
 import com.tencent.bkrepo.common.security.http.core.HttpAuthSecurity
 import com.tencent.bkrepo.job.executor.BlockThreadPoolTaskExecutorDecorator
+import com.tencent.bkrepo.job.migrate.config.MigrateRepoStorageProperties
 import org.springframework.boot.autoconfigure.task.TaskExecutionProperties
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.context.annotation.Bean
@@ -39,7 +40,7 @@ import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor
  * Job配置
  * */
 @Configuration
-@EnableConfigurationProperties(JobProperties::class)
+@EnableConfigurationProperties(JobProperties::class, MigrateRepoStorageProperties::class)
 class JobConfig {
     @Bean
     fun blockThreadPoolTaskExecutorDecorator(

diff --git a/...c/main/kotlin/com/tencent/bkrepo/job/config/properties/MigrateRepoStorageJobProperties.kt b/...c/main/kotlin/com/tencent/bkrepo/job/config/properties/MigrateRepoStorageJobProperties.kt
@@ -0,0 +1,35 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2024 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.config.properties
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+
+@ConfigurationProperties("job.migrate-repo-storage")
+class MigrateRepoStorageJobProperties(
+    override var cron: String = "0 0/5 * * * ?",
+) : BatchJobProperties()
diff --git a/...rc/main/kotlin/com/tencent/bkrepo/job/controller/user/UserMigrateRepoStorageController.kt b/...rc/main/kotlin/com/tencent/bkrepo/job/controller/user/UserMigrateRepoStorageController.kt
@@ -0,0 +1,56 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2024 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.bkrepo.job.controller.user
+
+import com.tencent.bkrepo.common.api.pojo.Response
+import com.tencent.bkrepo.common.security.permission.Principal
+import com.tencent.bkrepo.common.security.permission.PrincipalType
+import com.tencent.bkrepo.common.security.util.SecurityUtils
+import com.tencent.bkrepo.common.service.util.ResponseBuilder
+import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService
+import com.tencent.bkrepo.job.migrate.pojo.CreateMigrateRepoStorageTaskRequest
+import com.tencent.bkrepo.job.migrate.pojo.MigrateRepoStorageTask
+import org.springframework.web.bind.annotation.PostMapping
+import org.springframework.web.bind.annotation.RequestBody
+import org.springframework.web.bind.annotation.RequestMapping
+import org.springframework.web.bind.annotation.RestController
+
+@RestController
+@RequestMapping("/api/job/migrate")
+@Principal(type = PrincipalType.ADMIN)
+class UserMigrateRepoStorageController(
+    private val migrateRepoStorageService: MigrateRepoStorageService
+) {
+    @PostMapping
+    fun migrate(
+        @RequestBody request: CreateMigrateRepoStorageTaskRequest
+    ): Response<MigrateRepoStorageTask> {
+        val task = migrateRepoStorageService.createTask(request.copy(operator = SecurityUtils.getUserId()))
+        return ResponseBuilder.success(task)
+    }
+}