From b47fc3667dc9c6083a1f45622e8e25c50e74f426 Mon Sep 17 00:00:00 2001 From: kunlongli <16629885+cnlkl@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:32:17 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E8=BF=81=E7=A7=BB?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1job=E6=AF=8F=E6=AC=A1=E5=8F=AA=E8=A7=A6?= =?UTF-8?q?=E5=8F=91=E4=B8=80=E4=B8=AA=E4=BB=BB=E5=8A=A1=20#2058?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: 修复迁移任务job每次只触发一个任务 #2058 * fix: 修复迁移任务job每次只触发一个任务 #2058 * fix: 设置仓库在迁移时抛出RepoMigratingException #2058 --- .../task/clean/FileReferenceCleanupJob.kt | 7 ++++- .../batch/task/other/MigrateRepoStorageJob.kt | 5 ++-- .../bkrepo/job/batch/utils/NodeCommonUtils.kt | 3 +- .../job/exception/RepoMigratingException.kt | 30 +++++++++++++++++++ .../migrate/executor/MigrateExecutorTest.kt | 4 +-- 5 files changed, 42 insertions(+), 7 deletions(-) create mode 100644 src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/exception/RepoMigratingException.kt diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/FileReferenceCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/FileReferenceCleanupJob.kt index 67e31fe01d..039304b1f5 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/FileReferenceCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/FileReferenceCleanupJob.kt @@ -48,6 +48,7 @@ import com.tencent.bkrepo.job.batch.context.FileJobContext import com.tencent.bkrepo.job.batch.utils.NodeCommonUtils import com.tencent.bkrepo.job.config.properties.FileReferenceCleanupJobProperties import com.tencent.bkrepo.job.exception.JobExecuteException +import com.tencent.bkrepo.job.exception.RepoMigratingException import com.tencent.bkrepo.repository.api.StorageCredentialsClient import com.tencent.bkrepo.repository.constant.SYSTEM_USER import org.springframework.boot.context.properties.EnableConfigurationProperties @@ -145,7 +146,11 @@ class FileReferenceCleanupJob( } mongoTemplate.remove(Query(Criteria(ID).isEqualTo(id)), collectionName) } catch (e: Exception) { - throw JobExecuteException("Failed to delete file[$sha256] on [$storageCredentials].", e) + if (e is RepoMigratingException) { + logger.info(e.message) + } else { + throw JobExecuteException("Failed to delete file[$sha256] on [${storageCredentials?.key}].", e) + } } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/MigrateRepoStorageJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/MigrateRepoStorageJob.kt index 44cbae3f0a..e08c2a989f 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/MigrateRepoStorageJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/other/MigrateRepoStorageJob.kt @@ -44,9 +44,8 @@ class MigrateRepoStorageJob( override fun doStart0(jobContext: JobContext) { migrateRepoStorageService.rollbackInterruptedTaskState() while (true) { - migrateRepoStorageService.tryExecuteTask() - ?.let { logger.info("execute migrate task[$it]") } - ?: break + val executedTask = migrateRepoStorageService.tryExecuteTask() ?: break + logger.info("execute migrate task[$executedTask]") } } diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtils.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtils.kt index 302451be02..21bba79364 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtils.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/utils/NodeCommonUtils.kt @@ -6,6 +6,7 @@ import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils import com.tencent.bkrepo.job.BATCH_SIZE import com.tencent.bkrepo.job.SHARDING_COUNT +import com.tencent.bkrepo.job.exception.RepoMigratingException import com.tencent.bkrepo.job.migrate.MigrateRepoStorageService import org.bson.types.ObjectId import org.springframework.data.domain.Sort @@ -75,7 +76,7 @@ class NodeCommonUtils( .any { // node正在迁移时无法判断是否存在于存储[storageCredentialsKey]上 if (migrateRepoStorageService.migrating(it.projectId, it.repoName)) { - throw IllegalStateException("repo[${it.projectId}/${it.repoName}] was migrating") + throw RepoMigratingException("repo[${it.projectId}/${it.repoName}] was migrating") } val repo = RepositoryCommonUtils.getRepositoryDetail(it.projectId, it.repoName) repo.storageCredentials?.key == storageCredentialsKey diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/exception/RepoMigratingException.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/exception/RepoMigratingException.kt new file mode 100644 index 0000000000..f273a16375 --- /dev/null +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/exception/RepoMigratingException.kt @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.job.exception + +class RepoMigratingException(msg: String): IllegalStateException(msg) diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt index d7e4c8d236..d55e4653bb 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt @@ -61,10 +61,10 @@ class MigrateExecutorTest @Autowired constructor( val task = migrateTaskService.findTask(UT_PROJECT_ID, UT_REPO_NAME)!! assertEquals(MigrateRepoStorageTaskState.MIGRATING.name, task.state) assertNotNull(task.startDate) - assertTrue(executingTaskRecorder.executing(task.id!!)) // 等待任务执行结束 - Thread.sleep(2000L) + Thread.sleep(500L) + assertTrue(executingTaskRecorder.executing(task.id!!)) context.waitAllTransferFinished() assertTaskFinished(task.id!!, nodeCount) }