From 57d922426c312f7f02b79efa4cd52ec3758772d4 Mon Sep 17 00:00:00 2001 From: yaoxuwan Date: Fri, 16 Aug 2024 10:36:19 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20blockNode=E6=8A=BD=E5=8F=96=E8=87=B3?= =?UTF-8?q?=E5=85=AC=E5=85=B1=E6=A8=A1=E5=9D=97=20#2413?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: blockNode抽取至公共模块 #2413 * feat: blockNode抽取至公共模块 #2413 * feat: blockNode抽取至公共模块 #2413 * feat: blockNode抽取至公共模块 #2413 --- .../bkrepo/config/FsNodeDefaultImpl.kt | 65 -------- .../common/api/mongo}/ShardingDocument.kt | 2 +- .../bkrepo/common/api/mongo}/ShardingKey.kt | 2 +- .../artifact-service/build.gradle.kts | 1 + .../manager/NodeResourceFactoryImpl.kt | 6 +- .../manager/resource/FsNodeResource.kt | 16 +- .../build.gradle.kts} | 16 +- .../metadata-api/build.gradle.kts} | 20 +-- .../common/metadata/constant/Constant.kt | 34 +++++ .../metadata-service/build.gradle.kts | 40 +++++ .../metadata/MetadataAutoConfiguration.kt | 37 +++++ .../metadata/condition/ReactiveCondition.kt | 43 ++++++ .../metadata/condition/SyncCondition.kt | 43 ++++++ .../common/metadata/dao/BlockNodeDao.kt | 38 +++++ .../common/metadata/dao/RBlockNodeDao.kt | 38 +++++ .../common/metadata/model/TBlockNode.kt | 64 ++++++++ .../service/blocknode/BlockNodeService.kt | 99 ++++++++++++ .../service/blocknode/RBlockNodeService.kt | 98 ++++++++++++ .../impl/AbstractBlockNodeService.kt | 141 ++++++++++++++++++ .../impl/DefaultBlockNodeServiceImpl.kt | 26 ++++ .../impl/RAbstractBlockNodeService.kt | 141 ++++++++++++++++++ .../metadata/util/BlockNodeQueryHelper.kt | 103 +++++++++++++ .../main/resources/META-INF/spring.factories | 29 ++++ .../reactive/dao/ShardingMongoReactiveDao.kt | 7 +- .../mongo/dao/sharding/ShardingMongoDao.kt | 2 + .../operate/service/model/TOperateLog.kt | 4 +- .../bkrepo/fs/server/api/FsNodeClient.kt | 69 --------- .../bkrepo/fs/server/constant/Constants.kt | 5 - .../fs/boot-fs-server/build.gradle.kts | 1 + .../fs/server/config/RouteConfiguration.kt | 2 - .../bkrepo/fs/server/handler/ClientHandler.kt | 4 +- .../server/handler/NodeOperationsHandler.kt | 4 +- .../server/handler/service/FsNodeHandler.kt | 21 +-- .../bkrepo/fs/server/model/TBlockNode.kt | 64 -------- .../server/repository/BlockNodeRepository.kt | 35 ----- .../fs/server/service/BlockNodeService.kt | 85 ----------- .../fs/server/service/BlockNodeServiceImpl.kt | 117 ++------------- .../fs/server/service/FileNodeService.kt | 26 +--- .../fs/server/service/FileOperationService.kt | 2 +- .../fs/server/storage/CoStorageManager.kt | 6 +- .../bkrepo/fs/service/BlockNodeServiceTest.kt | 14 +- .../batch/task/archive/IdleNodeArchiveJob.kt | 2 +- .../job/batch/task/archive/SystemGcJob.kt | 2 +- .../batch/task/clean/DeletedNodeCleanupJob.kt | 2 +- .../job/migrate/executor/BaseTaskExecutor.kt | 2 +- .../job/separation/model/TSeparationNode.kt | 4 +- .../model/TSeparationPackageVersion.kt | 4 +- .../repo/TSeparationMavenMetadataRecord.kt | 4 +- .../service/impl/AbstractHandler.kt | 4 +- .../job/service/impl/ArchiveJobServiceImpl.kt | 2 +- .../migrate/executor/MigrateExecutorTest.kt | 2 +- .../commitedge/CenterClusterReplicator.kt | 2 +- .../replica/type/AbstractReplicaService.kt | 2 +- .../replica/type/edge/EdgeReplicaTaskJob.kt | 2 +- .../biz-repository/build.gradle.kts | 1 + .../bkrepo/repository/model/TFileReference.kt | 4 +- .../tencent/bkrepo/repository/model/TNode.kt | 4 +- .../service/blocknode/BlockNodeServiceImpl.kt | 56 +++++++ .../service/fs/impl/FsServiceImpl.kt | 4 +- .../service/node/impl/NodeBaseService.kt | 8 +- .../service/node/impl/NodeRestoreSupport.kt | 14 +- .../service/node/impl/NodeServiceImpl.kt | 6 +- .../center/CommitEdgeCenterNodeServiceImpl.kt | 6 +- .../node/impl/edge/EdgeNodeBaseService.kt | 6 +- .../node/impl/edge/EdgeNodeServiceImpl.kt | 6 +- .../repo/impl/RepositoryServiceImpl.kt | 2 +- .../bkrepo/repository/util/NodeQueryHelper.kt | 2 +- .../repository/service/ServiceBaseTest.kt | 10 +- src/backend/settings.gradle.kts | 1 + .../bkrepo/webhook/model/TWebHookLog.kt | 2 +- 70 files changed, 1139 insertions(+), 597 deletions(-) delete mode 100644 src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt rename src/backend/common/{common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao => common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo}/ShardingDocument.kt (96%) rename src/backend/common/{common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding => common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo}/ShardingKey.kt (97%) rename src/backend/common/{common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingDocument.kt => common-metadata/build.gradle.kts} (81%) rename src/backend/common/{common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingKey.kt => common-metadata/metadata-api/build.gradle.kts} (78%) create mode 100644 src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt create mode 100644 src/backend/common/common-metadata/metadata-service/build.gradle.kts create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/RBlockNodeDao.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt create mode 100644 src/backend/common/common-metadata/metadata-service/src/main/resources/META-INF/spring.factories delete mode 100644 src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt delete mode 100644 src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/TBlockNode.kt delete mode 100644 src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/repository/BlockNodeRepository.kt delete mode 100644 src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeService.kt create mode 100644 src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt diff --git a/src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt b/src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt deleted file mode 100644 index b7f145c119..0000000000 --- a/src/backend/boot-assembly/src/main/kotlin/com/tencent/bkrepo/config/FsNodeDefaultImpl.kt +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.config - -import com.tencent.bkrepo.common.api.exception.MethodNotAllowedException -import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.storage.pojo.RegionResource -import com.tencent.bkrepo.fs.server.api.FsNodeClient -import org.springframework.stereotype.Service - -/** - * fs-server提供的是reactive的实现，与boot-assembly冲突， - * 所以这里提供一个默认实现 - * */ -@Service -class FsNodeDefaultImpl : FsNodeClient { - override fun listBlockResources( - projectId: String, - repoName: String, - path: String, - startPos: Long, - endPos: Long - ): Response> { - throw MethodNotAllowedException() - } - - override fun restoreBlockResources( - projectId: String, - repoName: String, - fullPath: String, - nodeCreateDate: String, - nodeDeleteDate: String - ): Response { - throw MethodNotAllowedException() - } - - override fun deleteBlockResources(projectId: String, repoName: String, fullPath: String) { - throw MethodNotAllowedException() - } -} diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingDocument.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingDocument.kt similarity index 96% rename from src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingDocument.kt rename to src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingDocument.kt index ab2231b5a2..2a8bdbb2eb 100644 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingDocument.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingDocument.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.common.mongo.reactive.dao +package com.tencent.bkrepo.common.api.mongo /** * 分表Document diff --git a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingKey.kt b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingKey.kt similarity index 97% rename from src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingKey.kt rename to src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingKey.kt index 3085091b08..a7376a2cf8 100644 --- a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingKey.kt +++ b/src/backend/common/common-api/src/main/kotlin/com/tencent/bkrepo/common/api/mongo/ShardingKey.kt @@ -29,7 +29,7 @@ * SOFTWARE. */ -package com.tencent.bkrepo.common.mongo.dao.sharding +package com.tencent.bkrepo.common.api.mongo /** * 分表字段 diff --git a/src/backend/common/common-artifact/artifact-service/build.gradle.kts b/src/backend/common/common-artifact/artifact-service/build.gradle.kts index a043e46bb7..91e92bbf99 100644 --- a/src/backend/common/common-artifact/artifact-service/build.gradle.kts +++ b/src/backend/common/common-artifact/artifact-service/build.gradle.kts @@ -43,6 +43,7 @@ dependencies { api(project(":common:common-operate:operate-service")) api(project(":common:common-stream")) api(project(":common:common-metrics-push")) + api(project(":common:common-metadata:metadata-service")) api("org.springframework.boot:spring-boot-starter-aop") api("io.micrometer:micrometer-registry-prometheus") diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt index a4a87467c4..22fd07c76c 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/NodeResourceFactoryImpl.kt @@ -36,11 +36,11 @@ import com.tencent.bkrepo.common.artifact.manager.resource.LocalNodeResource import com.tencent.bkrepo.common.artifact.manager.resource.NodeResource import com.tencent.bkrepo.common.artifact.manager.resource.RemoteNodeResource import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.service.cluster.ClusterInfo import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.replication.api.ClusterNodeClient import com.tencent.bkrepo.replication.exception.ReplicationMessageCode @@ -51,7 +51,7 @@ class NodeResourceFactoryImpl( private val clusterProperties: ClusterProperties, private val storageService: StorageService, private val storageCredentialsClient: StorageCredentialsClient, - private val fsNodeClient: FsNodeClient, + private val blockNodeService: BlockNodeService, private val clusterNodeClient: ClusterNodeClient, private val archiveClient: ArchiveClient, ) : NodeResourceFactory { @@ -71,7 +71,7 @@ class NodeResourceFactoryImpl( ): NodeResource { val digest = nodeInfo.sha256.orEmpty() if (isFsFile(nodeInfo)) { - return FsNodeResource(nodeInfo, fsNodeClient, range, storageService, storageCredentials) + return FsNodeResource(nodeInfo, blockNodeService, range, storageService, storageCredentials) } if (clusterProperties.role == ClusterNodeType.EDGE && clusterProperties.architecture != ClusterArchitecture.COMMIT_EDGE) { diff --git a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/FsNodeResource.kt b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/FsNodeResource.kt index e5d0033cad..d36cf4a595 100644 --- a/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/FsNodeResource.kt +++ b/src/backend/common/common-artifact/artifact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/resource/FsNodeResource.kt @@ -29,9 +29,10 @@ package com.tencent.bkrepo.common.artifact.manager.resource import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.fs.server.api.FsNodeClient +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo /** @@ -39,20 +40,17 @@ import com.tencent.bkrepo.repository.pojo.node.NodeInfo * */ class FsNodeResource( private val node: NodeInfo, - private val fsNodeClient: FsNodeClient, + private val blockNodeService: BlockNodeService, private val range: Range, private val storageService: StorageService, private val storageCredentials: StorageCredentials? ) : AbstractNodeResource() { override fun getArtifactInputStream(): ArtifactInputStream? { with(node) { - val blocks = fsNodeClient.listBlockResources( - projectId = projectId, - repoName = repoName, - path = fullPath, - startPos = range.start, - endPos = range.end - ).data ?: emptyList() + val blocks = blockNodeService.info( + nodeDetail = NodeDetail(this), + range = range + ) return storageService.load(blocks, range, storageCredentials) } } diff --git a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingDocument.kt b/src/backend/common/common-metadata/build.gradle.kts similarity index 81% rename from src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingDocument.kt rename to src/backend/common/common-metadata/build.gradle.kts index 78dfa669de..6971910f79 100644 --- a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingDocument.kt +++ b/src/backend/common/common-metadata/build.gradle.kts @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -28,17 +28,3 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - -package com.tencent.bkrepo.common.mongo.dao.sharding - -/** - * 分表Document - */ -@Retention(AnnotationRetention.RUNTIME) -@Target(AnnotationTarget.CLASS) -annotation class ShardingDocument( - /** - * collection name - */ - val collection: String = "" -) diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingKey.kt b/src/backend/common/common-metadata/metadata-api/build.gradle.kts similarity index 78% rename from src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingKey.kt rename to src/backend/common/common-metadata/metadata-api/build.gradle.kts index 59dc10b1d7..6971910f79 100644 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingKey.kt +++ b/src/backend/common/common-metadata/metadata-api/build.gradle.kts @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * - * Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. * * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. * @@ -28,21 +28,3 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ - -package com.tencent.bkrepo.common.mongo.reactive.dao - -/** - * 分表字段 - */ -@Retention(AnnotationRetention.RUNTIME) -@Target(AnnotationTarget.FIELD) -annotation class ShardingKey( - /** - * 分表字段 - */ - val column: String = "", - /** - * 分表数，power of 2 - */ - val count: Int = 1 -) diff --git a/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt new file mode 100644 index 0000000000..3f63c95f1d --- /dev/null +++ b/src/backend/common/common-metadata/metadata-api/src/main/kotlin/com/tencent/bkrepo/common/metadata/constant/Constant.kt @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.constant + +const val SHARDING_COUNT = 256 +const val ID = "_id" + +const val FAKE_SHA256 = "0000000000000000000000000000000000000000000000000000000000000000" +const val FAKE_MD5 = "00000000000000000000000000000000" diff --git a/src/backend/common/common-metadata/metadata-service/build.gradle.kts b/src/backend/common/common-metadata/metadata-service/build.gradle.kts new file mode 100644 index 0000000000..8fba1d187c --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/build.gradle.kts @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +dependencies { + api(project(":common:common-artifact:artifact-api")) + api(project(":common:common-metadata:metadata-api")) + api(project(":common:common-storage:storage-api")) + api(project(":repository:api-repository")) + + compileOnly(project(":common:common-mongo-reactive")) + compileOnly(project(":common:common-mongo")) +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt new file mode 100644 index 0000000000..ccaffc9669 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/MetadataAutoConfiguration.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata + +import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication +import org.springframework.context.annotation.ComponentScan +import org.springframework.context.annotation.Configuration + +@Configuration +@ConditionalOnWebApplication +@ComponentScan(basePackages = ["com.tencent.bkrepo.common.metadata"]) +class MetadataAutoConfiguration diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt new file mode 100644 index 0000000000..3d8e63eb14 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/ReactiveCondition.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.condition + +import org.springframework.context.annotation.Condition +import org.springframework.context.annotation.ConditionContext +import org.springframework.core.type.AnnotatedTypeMetadata + +class ReactiveCondition : Condition { + override fun matches(context: ConditionContext, metadata: AnnotatedTypeMetadata): Boolean { + try { + context.classLoader?.loadClass("com.mongodb.reactivestreams.client.MongoClient") + return true + } catch (e: ClassNotFoundException) { + return false + } + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt new file mode 100644 index 0000000000..9514e30e35 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/condition/SyncCondition.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.condition + +import org.springframework.context.annotation.Condition +import org.springframework.context.annotation.ConditionContext +import org.springframework.core.type.AnnotatedTypeMetadata + +class SyncCondition : Condition { + override fun matches(context: ConditionContext, metadata: AnnotatedTypeMetadata): Boolean { + try { + context.classLoader?.loadClass("com.mongodb.client.MongoClient") + return true + } catch (e: ClassNotFoundException) { + return false + } + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt new file mode 100644 index 0000000000..3e93f78bab --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/BlockNodeDao.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.dao + +import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.mongo.dao.sharding.HashShardingMongoDao +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Repository + +@Repository +@Conditional(SyncCondition::class) +class BlockNodeDao : HashShardingMongoDao() diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/RBlockNodeDao.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/RBlockNodeDao.kt new file mode 100644 index 0000000000..7ccb332148 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/RBlockNodeDao.kt @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.dao + +import com.tencent.bkrepo.common.metadata.condition.ReactiveCondition +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingMongoReactiveDao +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Repository + +@Repository +@Conditional(ReactiveCondition::class) +class RBlockNodeDao : ShardingMongoReactiveDao() diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt new file mode 100644 index 0000000000..4ec14a7c41 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/model/TBlockNode.kt @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.model + +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.metadata.constant.SHARDING_COUNT +import com.tencent.bkrepo.common.metadata.model.TBlockNode.Companion.BLOCK_IDX +import com.tencent.bkrepo.common.metadata.model.TBlockNode.Companion.BLOCK_IDX_DEF +import com.tencent.bkrepo.common.api.mongo.ShardingKey +import org.springframework.data.mongodb.core.index.CompoundIndex +import org.springframework.data.mongodb.core.index.CompoundIndexes +import java.time.LocalDateTime + +/** + * 块节点 + * */ +@ShardingDocument("block_node") +@CompoundIndexes( + CompoundIndex(name = BLOCK_IDX, def = BLOCK_IDX_DEF) +) +data class TBlockNode( + var id: String? = null, + var createdBy: String, + var createdDate: LocalDateTime, + val nodeFullPath: String, + val startPos: Long, + var sha256: String, + val projectId: String, + @ShardingKey(count = SHARDING_COUNT) + val repoName: String, + val size: Long, + val endPos: Long = startPos + size - 1, + var deleted: LocalDateTime? = null +) { + companion object { + const val BLOCK_IDX = "start_pos_idx" + const val BLOCK_IDX_DEF = "{'projectId': 1, 'repoName': 1,'nodeFullPath':1, 'startPos': 1, 'isDeleted': 1}" + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt new file mode 100644 index 0000000000..3557450932 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/BlockNodeService.kt @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.service.blocknode + +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.storage.pojo.RegionResource +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import java.time.LocalDateTime + +interface BlockNodeService { + + /** + * 查询出范围内的分块 + * */ + fun listBlocks( + range: Range, + projectId: String, + repoName: String, + fullPath: String, + createdDate: String + ): List + + /** + * 创建分块 + * */ + fun createBlock( + blockNode: TBlockNode, + storageCredentials: StorageCredentials? + ): TBlockNode + + /** + * 删除旧分块，即删除非指定的nodeCurrentSha256的分块。 + * 如果未指定nodeCurrentSha256，则删除节点所有分块 + * @param projectId 项目id + * @param repoName 仓库名 + * @param fullPath 文件路径 + * */ + fun deleteBlocks( + projectId: String, + repoName: String, + fullPath: String + ) + + /** + * 移动文件对应分块 + */ + fun moveBlocks( + projectId: String, + repoName: String, + fullPath: String, + dstFullPath: String + ) + + /** + * 恢复文件对应分块 + */ + fun restoreBlocks( + projectId: String, + repoName: String, + fullPath: String, + nodeCreateDate: LocalDateTime, + nodeDeleteDate: LocalDateTime + ) + + /** + * 查询节点对应范围的分块资源 + */ + fun info( + nodeDetail: NodeDetail, + range: Range + ): List +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt new file mode 100644 index 0000000000..15c6f77559 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/RBlockNodeService.kt @@ -0,0 +1,98 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.service.blocknode + +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.storage.pojo.RegionResource +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import java.time.LocalDateTime + +interface RBlockNodeService { + /** + * 查询出范围内的分块 + * */ + suspend fun listBlocks( + range: Range, + projectId: String, + repoName: String, + fullPath: String, + createdDate: String + ): List + + /** + * 创建分块 + * */ + suspend fun createBlock( + blockNode: TBlockNode, + storageCredentials: StorageCredentials? + ): TBlockNode + + /** + * 删除旧分块，即删除非指定的nodeCurrentSha256的分块。 + * 如果未指定nodeCurrentSha256，则删除节点所有分块 + * @param projectId 项目id + * @param repoName 仓库名 + * @param fullPath 文件路径 + * */ + suspend fun deleteBlocks( + projectId: String, + repoName: String, + fullPath: String + ) + + /** + * 移动文件对应分块 + */ + suspend fun moveBlocks( + projectId: String, + repoName: String, + fullPath: String, + dstFullPath: String + ) + + /** + * 恢复文件对应分块 + */ + suspend fun restoreBlocks( + projectId: String, + repoName: String, + fullPath: String, + nodeCreateDate: LocalDateTime, + nodeDeleteDate: LocalDateTime + ) + + /** + * 查询节点对应范围的分块资源 + */ + suspend fun info( + nodeDetail: NodeDetail, + range: Range + ): List +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt new file mode 100644 index 0000000000..edb6cfeb7a --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt @@ -0,0 +1,141 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.service.blocknode.impl + +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.constant.ID +import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService +import com.tencent.bkrepo.common.metadata.util.BlockNodeQueryHelper +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.storage.pojo.RegionResource +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.isEqualTo +import java.time.LocalDateTime + +abstract class AbstractBlockNodeService( + private val blockNodeDao: BlockNodeDao +) : BlockNodeService { + + override fun createBlock(blockNode: TBlockNode, storageCredentials: StorageCredentials?): TBlockNode { + with(blockNode) { + val bn = blockNodeDao.save(blockNode) + incFileRef(bn.sha256, storageCredentials?.key) + logger.info("Create block node[$projectId/$repoName$nodeFullPath-$startPos] ,sha256[$sha256] success.") + return bn + } + } + + override fun listBlocks( + range: Range, + projectId: String, + repoName: String, + fullPath: String, + createdDate: String + ): List { + val query = BlockNodeQueryHelper.listQuery(projectId, repoName, fullPath, createdDate, range) + return blockNodeDao.find(query) + } + + override fun deleteBlocks( + projectId: String, + repoName: String, + fullPath: String + ) { + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) + val update = BlockNodeQueryHelper.deleteUpdate() + blockNodeDao.updateMulti(Query(criteria), update) + logger.info("Delete node blocks[$projectId/$repoName$fullPath] success.") + } + + override fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { + val nodeDetail = getNodeDetail(projectId, repoName, dstFullPath) + if (nodeDetail.folder) { + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, true) + val blocks = blockNodeDao.find(Query(criteria)) + blocks.forEach { + val update = Update().set(TBlockNode::nodeFullPath.name, it.nodeFullPath.replace(fullPath, dstFullPath)) + val query = Query(Criteria.where(ID).isEqualTo(it.id).and(TBlockNode::repoName).isEqualTo(repoName)) + blockNodeDao.updateMulti(query, update) + } + } else { + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) + val update = BlockNodeQueryHelper.moveUpdate(dstFullPath) + blockNodeDao.updateMulti(Query(criteria), update) + } + logger.info("Move node[$projectId/$repoName$fullPath] to node[$projectId/$repoName$dstFullPath] success.") + } + + + + override fun restoreBlocks( + projectId: String, + repoName: String, + fullPath: String, + nodeCreateDate: LocalDateTime, + nodeDeleteDate: LocalDateTime + ) { + val criteria = + BlockNodeQueryHelper.deletedCriteria(projectId, repoName, fullPath, nodeCreateDate, nodeDeleteDate) + val update = BlockNodeQueryHelper.restoreUpdate() + val result = blockNodeDao.updateMulti(Query(criteria), update) + logger.info("Restore ${result.modifiedCount} blocks node[$projectId/$repoName$fullPath] " + + "between $nodeCreateDate and $nodeDeleteDate success.") + } + + override fun info(nodeDetail: NodeDetail, range: Range): List { + with(nodeDetail) { + val blocks = listBlocks(range, projectId, repoName, fullPath, createdDate) + val blockResources = mutableListOf() + if (sha256 != null && sha256 != FAKE_SHA256) { + val nodeData = RegionResource(sha256!!, 0, size, 0, size) + blockResources.add(nodeData) + } + blocks.forEach { + val res = RegionResource(it.sha256, it.startPos, it.size, 0, it.size) + blockResources.add(res) + } + return blockResources + } + } + + abstract fun incFileRef(sha256: String, credentialsKey: String?) + + abstract fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail + + companion object { + private val logger = LoggerFactory.getLogger(AbstractBlockNodeService::class.java) + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt new file mode 100644 index 0000000000..688bd865b4 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/DefaultBlockNodeServiceImpl.kt @@ -0,0 +1,26 @@ +package com.tencent.bkrepo.common.metadata.service.blocknode.impl + +import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException +import com.tencent.bkrepo.common.metadata.condition.SyncCondition +import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao +import com.tencent.bkrepo.repository.api.FileReferenceClient +import com.tencent.bkrepo.repository.api.NodeClient +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import org.springframework.context.annotation.Conditional +import org.springframework.stereotype.Service + +@Service +@Conditional(SyncCondition::class) +class DefaultBlockNodeServiceImpl( + blockNodeDao: BlockNodeDao, + private val fileReferenceClient: FileReferenceClient, + private val nodeClient: NodeClient +) : AbstractBlockNodeService(blockNodeDao) { + override fun incFileRef(sha256: String, credentialsKey: String?) { + fileReferenceClient.increment(sha256, credentialsKey) + } + + override fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { + return nodeClient.getNodeDetail(projectId, repoName, fullPath).data ?: throw NodeNotFoundException(fullPath) + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt new file mode 100644 index 0000000000..4e48e22f5b --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/RAbstractBlockNodeService.kt @@ -0,0 +1,141 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.service.blocknode.impl + +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.constant.ID +import com.tencent.bkrepo.common.metadata.dao.RBlockNodeDao +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService +import com.tencent.bkrepo.common.metadata.util.BlockNodeQueryHelper +import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.storage.pojo.RegionResource +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import org.slf4j.LoggerFactory +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.isEqualTo +import java.time.LocalDateTime + +abstract class RAbstractBlockNodeService( + private val rBlockNodeDao: RBlockNodeDao +) : RBlockNodeService { + + override suspend fun createBlock(blockNode: TBlockNode, storageCredentials: StorageCredentials?): TBlockNode { + with(blockNode) { + val bn = rBlockNodeDao.save(blockNode) + incFileRef(bn.sha256, storageCredentials?.key) + logger.info("Create block node[$projectId/$repoName$nodeFullPath-$startPos] ,sha256[$sha256] success.") + return bn + } + } + + override suspend fun listBlocks( + range: Range, + projectId: String, + repoName: String, + fullPath: String, + createdDate: String + ): List { + val query = BlockNodeQueryHelper.listQuery(projectId, repoName, fullPath, createdDate, range) + return rBlockNodeDao.find(query) + } + + override suspend fun deleteBlocks( + projectId: String, + repoName: String, + fullPath: String + ) { + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) + val update = BlockNodeQueryHelper.deleteUpdate() + rBlockNodeDao.updateMulti(Query(criteria), update) + logger.info("Delete node blocks[$projectId/$repoName$fullPath] success.") + } + + override suspend fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { + val nodeDetail = getNodeDetail(projectId, repoName, dstFullPath) + if (nodeDetail.folder) { + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, true) + val blocks = rBlockNodeDao.find(Query(criteria)) + blocks.forEach { + val update = BlockNodeQueryHelper.moveUpdate(it.nodeFullPath.replace(fullPath, dstFullPath)) + val query = Query(Criteria.where(ID).isEqualTo(it.id).and(TBlockNode::repoName).isEqualTo(repoName)) + rBlockNodeDao.updateMulti(query, update) + } + } else { + val criteria = BlockNodeQueryHelper.fullPathCriteria(projectId, repoName, fullPath, false) + val update = BlockNodeQueryHelper.moveUpdate(dstFullPath) + rBlockNodeDao.updateMulti(Query(criteria), update) + } + logger.info("Move node[$projectId/$repoName$fullPath] to node[$projectId/$repoName$dstFullPath] success.") + } + + + override suspend fun restoreBlocks( + projectId: String, + repoName: String, + fullPath: String, + nodeCreateDate: LocalDateTime, + nodeDeleteDate: LocalDateTime + ) { + val criteria = + BlockNodeQueryHelper.deletedCriteria(projectId, repoName, fullPath, nodeCreateDate, nodeDeleteDate) + val update = BlockNodeQueryHelper.restoreUpdate() + val result = rBlockNodeDao.updateMulti(Query(criteria), update) + logger.info( + "Restore ${result.modifiedCount} blocks node[$projectId/$repoName$fullPath] " + + "between $nodeCreateDate and $nodeDeleteDate success." + ) + } + + override suspend fun info(nodeDetail: NodeDetail, range: Range): List { + with(nodeDetail) { + val blocks = listBlocks(range, projectId, repoName, fullPath, createdDate) + val blockResources = mutableListOf() + if (sha256 != null && sha256 != FAKE_SHA256) { + val nodeData = RegionResource(sha256!!, 0, size, 0, size) + blockResources.add(nodeData) + } + blocks.forEach { + val res = RegionResource(it.sha256, it.startPos, it.size, 0, it.size) + blockResources.add(res) + } + return blockResources + } + } + + abstract suspend fun incFileRef(sha256: String, credentialsKey: String?) + + abstract suspend fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail + + companion object { + private val logger = LoggerFactory.getLogger(RAbstractBlockNodeService::class.java) + } +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt new file mode 100644 index 0000000000..396e6b14e5 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/util/BlockNodeQueryHelper.kt @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.common.metadata.util + +import com.tencent.bkrepo.common.api.util.EscapeUtils +import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import org.springframework.data.domain.Sort +import org.springframework.data.mongodb.core.query.Criteria +import org.springframework.data.mongodb.core.query.Query +import org.springframework.data.mongodb.core.query.Update +import org.springframework.data.mongodb.core.query.and +import org.springframework.data.mongodb.core.query.gt +import org.springframework.data.mongodb.core.query.isEqualTo +import org.springframework.data.mongodb.core.query.lt +import org.springframework.data.mongodb.core.query.where +import java.time.LocalDateTime + +object BlockNodeQueryHelper { + + fun listQuery( + projectId: String, + repoName: String, + fullPath: String, + createdDate: String, + range: Range + ): Query { + val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId).isEqualTo(projectId) + .and(TBlockNode::repoName).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + .and(TBlockNode::createdDate).gt(LocalDateTime.parse(createdDate)) + .norOperator( + TBlockNode::startPos.gt(range.end), + TBlockNode::endPos.lt(range.start) + ) + val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) + return query + } + + fun fullPathCriteria(projectId: String, repoName: String, fullPath: String, deep: Boolean): Criteria { + val criteria = if (deep) { + where(TBlockNode::nodeFullPath).regex("^${EscapeUtils.escapeRegex(fullPath)}/") + } else { + where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + } + return criteria + .and(TBlockNode::projectId).isEqualTo(projectId) + .and(TBlockNode::repoName).isEqualTo(repoName) + .and(TBlockNode::deleted).isEqualTo(null) + } + + fun deletedCriteria( + projectId: String, + repoName: String, + fullPath: String, + nodeCreateDate: LocalDateTime, + nodeDeleteDate: LocalDateTime + ): Criteria { + return where(TBlockNode::nodeFullPath).isEqualTo(fullPath) + .and(TBlockNode::projectId.name).isEqualTo(projectId) + .and(TBlockNode::repoName.name).isEqualTo(repoName) + .and(TBlockNode::createdDate).gt(nodeCreateDate).lt(nodeDeleteDate) + } + + fun deleteUpdate(): Update { + return Update().set(TBlockNode::deleted.name, LocalDateTime.now()) + } + + fun moveUpdate(dstFullPath: String): Update { + return Update().set(TBlockNode::nodeFullPath.name, dstFullPath) + } + + fun restoreUpdate(): Update { + return Update().set(TBlockNode::deleted.name, null) + } + +} diff --git a/src/backend/common/common-metadata/metadata-service/src/main/resources/META-INF/spring.factories b/src/backend/common/common-metadata/metadata-service/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..4a292034f6 --- /dev/null +++ b/src/backend/common/common-metadata/metadata-service/src/main/resources/META-INF/spring.factories @@ -0,0 +1,29 @@ +# +# Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. +# +# Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. +# +# BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. +# +# A copy of the MIT License is included in this file. +# +# +# Terms of the MIT License: +# --------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +# LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bkrepo.common.metadata.MetadataAutoConfiguration diff --git a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt index 2dc45cacf5..e9351b3a0b 100644 --- a/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt +++ b/src/backend/common/common-mongo-reactive/src/main/kotlin/com/tencent/bkrepo/common/mongo/reactive/dao/ShardingMongoReactiveDao.kt @@ -28,10 +28,10 @@ package com.tencent.bkrepo.common.mongo.reactive.dao import com.mongodb.BasicDBList +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.common.mongo.util.MongoIndexResolver import com.tencent.bkrepo.common.mongo.util.ShardingUtils -import java.lang.reflect.Field -import javax.annotation.PostConstruct import org.apache.commons.lang3.reflect.FieldUtils import org.bson.Document import org.slf4j.LoggerFactory @@ -40,8 +40,9 @@ import org.springframework.beans.factory.annotation.Value import org.springframework.core.annotation.AnnotationUtils import org.springframework.data.mongodb.core.ReactiveMongoOperations import org.springframework.data.mongodb.core.index.IndexDefinition -import org.springframework.data.mongodb.core.indexOps import org.springframework.data.mongodb.core.query.Query +import java.lang.reflect.Field +import javax.annotation.PostConstruct abstract class ShardingMongoReactiveDao : AbstractMongoReactiveDao() { diff --git a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt b/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt index a771a1075c..ab19360ee1 100644 --- a/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt +++ b/src/backend/common/common-mongo/src/main/kotlin/com/tencent/bkrepo/common/mongo/dao/sharding/ShardingMongoDao.kt @@ -32,6 +32,8 @@ package com.tencent.bkrepo.common.mongo.dao.sharding import com.mongodb.BasicDBList +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.common.mongo.dao.AbstractMongoDao import com.tencent.bkrepo.common.mongo.dao.util.MongoIndexResolver import com.tencent.bkrepo.common.mongo.dao.util.sharding.ShardingUtils diff --git a/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TOperateLog.kt b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TOperateLog.kt index b80bff887c..e3a7a99c48 100644 --- a/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TOperateLog.kt +++ b/src/backend/common/common-operate/operate-service/src/main/kotlin/com/tencent/bkrepo/common/operate/service/model/TOperateLog.kt @@ -31,8 +31,8 @@ package com.tencent.bkrepo.common.operate.service.model -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.common.operate.service.model.TOperateLog.Companion.RESOURCE_KEY_IDX import com.tencent.bkrepo.common.operate.service.model.TOperateLog.Companion.RESOURCE_KEY_IDX_DEF import org.springframework.data.mongodb.core.index.CompoundIndex diff --git a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt deleted file mode 100644 index 9853d201bf..0000000000 --- a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/api/FsNodeClient.kt +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.fs.server.api - -import com.tencent.bkrepo.common.api.constant.FS_SERVER_SERVICE_NAME -import com.tencent.bkrepo.common.api.pojo.Response -import com.tencent.bkrepo.common.storage.pojo.RegionResource -import org.springframework.cloud.openfeign.FeignClient -import org.springframework.web.bind.annotation.DeleteMapping -import org.springframework.web.bind.annotation.GetMapping -import org.springframework.web.bind.annotation.PathVariable -import org.springframework.web.bind.annotation.PostMapping -import org.springframework.web.bind.annotation.RequestMapping -import org.springframework.web.bind.annotation.RequestParam - -@FeignClient(FS_SERVER_SERVICE_NAME, contextId = "BlockClient", primary = false) -@RequestMapping("/service/block") -interface FsNodeClient { - - @GetMapping("/list/{projectId}/{repoName}") - fun listBlockResources( - @PathVariable projectId: String, - @PathVariable repoName: String, - @RequestParam path: String, - @RequestParam startPos: Long, - @RequestParam endPos: Long - ): Response> - - @PostMapping("/restore/{projectId}/{repoName}") - fun restoreBlockResources( - @PathVariable projectId: String, - @PathVariable repoName: String, - @RequestParam fullPath: String, - @RequestParam nodeCreateDate: String, - @RequestParam nodeDeleteDate: String, - ): Response - - @DeleteMapping("/delete/{projectId}/{repoName}") - fun deleteBlockResources( - @PathVariable projectId: String, - @PathVariable repoName: String, - @RequestParam fullPath: String - ) -} diff --git a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt index 50a8db58a7..d2f9c3b457 100644 --- a/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt +++ b/src/backend/fs/api-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/constant/Constants.kt @@ -33,8 +33,3 @@ const val JWT_CLAIMS_REPOSITORY = "repository" const val JWT_CLAIMS_PERMIT = "permit" const val FS_ATTR_KEY = "fs:attr" - -const val FAKE_SHA256 = "0000000000000000000000000000000000000000000000000000000000000000" -const val FAKE_MD5 = "00000000000000000000000000000000" - -const val ID = "_id" diff --git a/src/backend/fs/boot-fs-server/build.gradle.kts b/src/backend/fs/boot-fs-server/build.gradle.kts index 143faa5d33..a990c5da36 100644 --- a/src/backend/fs/boot-fs-server/build.gradle.kts +++ b/src/backend/fs/boot-fs-server/build.gradle.kts @@ -24,6 +24,7 @@ dependencies { api(project(":common:common-stream")) api(project(":fs:api-fs-server")) implementation("com.github.ben-manes.caffeine:caffeine:2.9.3") + implementation(project(":common:common-metadata:metadata-service")) testImplementation("org.mockito.kotlin:mockito-kotlin") testImplementation("de.flapdoodle.embed:de.flapdoodle.embed.mongo") diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt index b6968a9c2f..2457a1685c 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/config/RouteConfiguration.kt @@ -86,8 +86,6 @@ class RouteConfiguration( "/service/block".nest { GET("/list$DEFAULT_MAPPING_URI", fsNodeHandler::listBlocks) - POST("/restore$DEFAULT_MAPPING_URI", fsNodeHandler::restoreBlock) - DELETE("/delete$DEFAULT_MAPPING_URI", fsNodeHandler::deleteBlock) } "/node".nest { diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/ClientHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/ClientHandler.kt index bb87d8be8c..e5bddf9533 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/ClientHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/ClientHandler.kt @@ -29,9 +29,9 @@ package com.tencent.bkrepo.fs.server.handler import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_SIZE -import com.tencent.bkrepo.fs.server.request.ClientCreateRequest import com.tencent.bkrepo.fs.server.pojo.ClientListRequest import com.tencent.bkrepo.fs.server.pojo.DailyClientListRequest +import com.tencent.bkrepo.fs.server.request.ClientCreateRequest import com.tencent.bkrepo.fs.server.request.ClientPushMetricsRequest import com.tencent.bkrepo.fs.server.service.ClientService import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder @@ -100,4 +100,4 @@ class ClientHandler( clientService.pushMetrics(pushMetricsRequest) return ReactiveResponseBuilder.success() } -} \ No newline at end of file +} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt index 7d2f8519ae..bc20036819 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/NodeOperationsHandler.kt @@ -30,11 +30,11 @@ package com.tencent.bkrepo.fs.server.handler import com.github.benmanes.caffeine.cache.Caffeine import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.storage.core.overlay.OverlayRangeUtils import com.tencent.bkrepo.fs.server.api.RGenericClient import com.tencent.bkrepo.fs.server.api.RRepositoryClient -import com.tencent.bkrepo.fs.server.constant.FAKE_MD5 -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder import com.tencent.bkrepo.fs.server.model.NodeAttribute diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt index 829a5fc5f3..8e1dd3682b 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/handler/service/FsNodeHandler.kt @@ -29,10 +29,7 @@ package com.tencent.bkrepo.fs.server.handler.service import com.tencent.bkrepo.common.artifact.stream.Range import com.tencent.bkrepo.fs.server.api.RRepositoryClient -import com.tencent.bkrepo.fs.server.request.service.DeleteBlocksRequest import com.tencent.bkrepo.fs.server.request.service.ListBlocksRequest -import com.tencent.bkrepo.fs.server.request.service.RestoreBlocksRequest -import com.tencent.bkrepo.fs.server.service.BlockNodeService import com.tencent.bkrepo.fs.server.service.FileNodeService import com.tencent.bkrepo.fs.server.utils.ReactiveResponseBuilder import kotlinx.coroutines.reactor.awaitSingle @@ -42,8 +39,7 @@ import org.springframework.web.reactive.function.server.buildAndAwait class FsNodeHandler( private val rRepositoryClient: RRepositoryClient, - private val fileNodeService: FileNodeService, - private val blockNodeService: BlockNodeService + private val fileNodeService: FileNodeService ) { suspend fun listBlocks(request: ServerRequest): ServerResponse { @@ -57,19 +53,4 @@ class FsNodeHandler( return ReactiveResponseBuilder.success(fileNodeService.info(nodeDetail, range)) } } - - suspend fun restoreBlock(request: ServerRequest): ServerResponse { - with(RestoreBlocksRequest(request)) { - blockNodeService.restoreBlocks(projectId, repoName, fullPath, nodeCreateDate, nodeDeleteDate) - return ReactiveResponseBuilder.success() - } - } - - suspend fun deleteBlock(request: ServerRequest): ServerResponse { - with(DeleteBlocksRequest(request)) { - blockNodeService.deleteBlocks(projectId, repoName, fullPath) - return ReactiveResponseBuilder.success() - } - - } } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/TBlockNode.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/TBlockNode.kt deleted file mode 100644 index bafc4b9284..0000000000 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/model/TBlockNode.kt +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.fs.server.model - -import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingDocument -import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingKey -import com.tencent.bkrepo.fs.server.model.TBlockNode.Companion.BLOCK_IDX -import com.tencent.bkrepo.fs.server.model.TBlockNode.Companion.BLOCK_IDX_DEF -import com.tencent.bkrepo.repository.constant.SHARDING_COUNT -import org.springframework.data.mongodb.core.index.CompoundIndex -import org.springframework.data.mongodb.core.index.CompoundIndexes -import java.time.LocalDateTime - -/** - * 块节点 - * */ -@ShardingDocument("block_node") -@CompoundIndexes( - CompoundIndex(name = BLOCK_IDX, def = BLOCK_IDX_DEF) -) -data class TBlockNode( - var id: String? = null, - var createdBy: String, - var createdDate: LocalDateTime, - val nodeFullPath: String, - val startPos: Long, - var sha256: String, - val projectId: String, - @ShardingKey(count = SHARDING_COUNT) - val repoName: String, - val size: Long, - val endPos: Long = startPos + size - 1, - var deleted: LocalDateTime? = null -) { - companion object { - const val BLOCK_IDX = "start_pos_idx" - const val BLOCK_IDX_DEF = "{'projectId': 1, 'repoName': 1,'nodeFullPath':1, 'startPos': 1, 'isDeleted': 1}" - } -} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/repository/BlockNodeRepository.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/repository/BlockNodeRepository.kt deleted file mode 100644 index 2e6d607082..0000000000 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/repository/BlockNodeRepository.kt +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.fs.server.repository - -import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingMongoReactiveDao -import com.tencent.bkrepo.fs.server.model.TBlockNode -import org.springframework.stereotype.Repository - -@Repository -class BlockNodeRepository : ShardingMongoReactiveDao() diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeService.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeService.kt deleted file mode 100644 index aba1d01b58..0000000000 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeService.kt +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. - * - * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. - * - * A copy of the MIT License is included in this file. - * - * - * Terms of the MIT License: - * --------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT - * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package com.tencent.bkrepo.fs.server.service - -import com.tencent.bkrepo.common.artifact.stream.Range -import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.fs.server.model.TBlockNode -import java.time.LocalDateTime - -/** - * 块服务 - * */ -interface BlockNodeService { - /** - * 查询出范围内的分块 - * */ - suspend fun listBlocks( - range: Range, - projectId: String, - repoName: String, - fullPath: String, - createdDate: String - ): List - - /** - * 创建分块 - * */ - suspend fun createBlock( - blockNode: TBlockNode, - storageCredentials: StorageCredentials? - ): TBlockNode - - /** - * 删除旧分块，即删除非指定的nodeCurrentSha256的分块。 - * 如果未指定nodeCurrentSha256，则删除节点所有分块 - * @param projectId 项目id - * @param repoName 仓库名 - * @param fullPath 文件路径 - * */ - suspend fun deleteBlocks( - projectId: String, - repoName: String, - fullPath: String - ) - - suspend fun moveBlocks( - projectId: String, - repoName: String, - fullPath: String, - dstFullPath: String - ) - - suspend fun restoreBlocks( - projectId: String, - repoName: String, - fullPath: String, - nodeCreateDate: LocalDateTime, - nodeDeleteDate: LocalDateTime - ) -} diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt index 51fd09ee1b..2523c43a5e 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/BlockNodeServiceImpl.kt @@ -27,124 +27,27 @@ package com.tencent.bkrepo.fs.server.service -import com.tencent.bkrepo.common.api.util.EscapeUtils import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException -import com.tencent.bkrepo.common.artifact.stream.Range -import com.tencent.bkrepo.common.storage.credentials.StorageCredentials +import com.tencent.bkrepo.common.metadata.dao.RBlockNodeDao +import com.tencent.bkrepo.common.metadata.service.blocknode.impl.RAbstractBlockNodeService import com.tencent.bkrepo.fs.server.api.RRepositoryClient -import com.tencent.bkrepo.fs.server.constant.ID -import com.tencent.bkrepo.fs.server.model.TBlockNode -import com.tencent.bkrepo.fs.server.repository.BlockNodeRepository +import com.tencent.bkrepo.repository.pojo.node.NodeDetail import kotlinx.coroutines.reactor.awaitSingle -import org.slf4j.LoggerFactory -import org.springframework.data.domain.Sort -import org.springframework.data.mongodb.core.query.Criteria -import org.springframework.data.mongodb.core.query.Query -import org.springframework.data.mongodb.core.query.Update -import org.springframework.data.mongodb.core.query.and -import org.springframework.data.mongodb.core.query.gt -import org.springframework.data.mongodb.core.query.isEqualTo -import org.springframework.data.mongodb.core.query.lt -import org.springframework.data.mongodb.core.query.where -import java.time.LocalDateTime /** * 文件块服务 * */ class BlockNodeServiceImpl( - private val blockNodeRepository: BlockNodeRepository, + rBlockNodeDao: RBlockNodeDao, private val rRepositoryClient: RRepositoryClient -) : BlockNodeService { +) : RAbstractBlockNodeService(rBlockNodeDao) { - override suspend fun createBlock( - blockNode: TBlockNode, - storageCredentials: StorageCredentials? - ): TBlockNode { - with(blockNode) { - val bn = blockNodeRepository.save(blockNode) - rRepositoryClient.increment(blockNode.sha256, storageCredentials?.key).awaitSingle() - logger.info("Create block node[$projectId/$repoName$nodeFullPath-$startPos] ,sha256[$sha256] success.") - return bn - } + override suspend fun incFileRef(sha256: String, credentialsKey: String?) { + rRepositoryClient.increment(sha256, credentialsKey).awaitSingle() } - override suspend fun listBlocks( - range: Range, - projectId: String, - repoName: String, - fullPath: String, - createdDate: String - ): List { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - .and(TBlockNode::createdDate).gt(LocalDateTime.parse(createdDate)) - .norOperator( - TBlockNode::startPos.gt(range.end), - TBlockNode::endPos.lt(range.start) - ) - val query = Query(criteria).with(Sort.by(TBlockNode::createdDate.name)) - return blockNodeRepository.find(query) - } - - override suspend fun deleteBlocks( - projectId: String, - repoName: String, - fullPath: String - ) { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - val update = Update().set(TBlockNode::deleted.name, LocalDateTime.now()) - blockNodeRepository.updateMulti(Query(criteria), update) - logger.info("Delete node blocks[$projectId/$repoName$fullPath] success.") - } - - override suspend fun moveBlocks(projectId: String, repoName: String, fullPath: String, dstFullPath: String) { - val nodeDetail = rRepositoryClient.getNodeDetail(projectId, repoName, dstFullPath).awaitSingle().data - ?: throw NodeNotFoundException(dstFullPath) - if (nodeDetail.folder) { - val criteria = where(TBlockNode::nodeFullPath).regex("^${EscapeUtils.escapeRegex(fullPath)}/") - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - val blocks = blockNodeRepository.find(Query(criteria)) - blocks.forEach { - val update = Update().set(TBlockNode::nodeFullPath.name, it.nodeFullPath.replace(fullPath, dstFullPath)) - val query = Query(Criteria.where(ID).isEqualTo(it.id).and(TBlockNode::repoName).isEqualTo(repoName)) - blockNodeRepository.updateMulti(query, update) - } - } else { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::deleted).isEqualTo(null) - val update = Update().set(TBlockNode::nodeFullPath.name, dstFullPath) - blockNodeRepository.updateMulti(Query(criteria), update) - } - logger.info("Move node[$projectId/$repoName$fullPath] to node[$projectId/$repoName$dstFullPath] success.") - } - - override suspend fun restoreBlocks( - projectId: String, - repoName: String, - fullPath: String, - nodeCreateDate: LocalDateTime, - nodeDeleteDate: LocalDateTime - ) { - val criteria = where(TBlockNode::nodeFullPath).isEqualTo(fullPath) - .and(TBlockNode::projectId.name).isEqualTo(projectId) - .and(TBlockNode::repoName.name).isEqualTo(repoName) - .and(TBlockNode::createdDate).gt(nodeCreateDate).lt(nodeDeleteDate) - val update = Update().set(TBlockNode::deleted.name, null) - val result = blockNodeRepository.updateMulti(Query(criteria), update) - logger.info("Restore ${result.modifiedCount} blocks node[$projectId/$repoName$fullPath] " + - "between $nodeCreateDate and $nodeDeleteDate success.") - } - - companion object { - private val logger = LoggerFactory.getLogger(BlockNodeServiceImpl::class.java) + override suspend fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { + return rRepositoryClient.getNodeDetail(projectId, repoName, fullPath).awaitSingle().data + ?: throw NodeNotFoundException(fullPath) } } diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt index bbe381c47c..c62022e7a2 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileNodeService.kt @@ -30,26 +30,22 @@ package com.tencent.bkrepo.fs.server.service import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.RegionResource -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.fs.server.storage.CoStorageManager import com.tencent.bkrepo.repository.pojo.node.NodeDetail class FileNodeService( - private val blockNodeService: BlockNodeService, + private val blockNodeService: RBlockNodeService, private val coStorageManager: CoStorageManager ) { /** * 读取指定范围的文件 * 使用块数据overlay节点数据，当节点数据不存在时，则合并块，同时块之间空的地方补0 - * @param projectId 项目id - * @param repoName 仓库名 - * @param fullPath 节点路径 + * @param nodeDetail 节点详情 * @param storageCredentials 仓库的存储实例 - * @param digest 节点的sha56,当节点不存在时，可以为null - * @param size 节点的大小，当节点不存在时，可以为null * @param range 需要读取的文件范围 * */ suspend fun read( @@ -57,7 +53,7 @@ class FileNodeService( storageCredentials: StorageCredentials?, range: Range ): ArtifactInputStream? { - val blocks = info(nodeDetail, range) + val blocks = blockNodeService.info(nodeDetail, range) return coStorageManager.loadArtifactInputStream(blocks, range, storageCredentials) } @@ -65,19 +61,7 @@ class FileNodeService( nodeDetail: NodeDetail, range: Range ): List { - with(nodeDetail) { - val blocks = blockNodeService.listBlocks(range, projectId, repoName, fullPath, createdDate) - val blockResources = mutableListOf() - if (sha256 != null && sha256 != FAKE_SHA256) { - val nodeData = RegionResource(sha256!!, 0, size, 0, size) - blockResources.add(nodeData) - } - blocks.forEach { - val res = RegionResource(it.sha256, it.startPos, it.size, 0, it.size) - blockResources.add(res) - } - return blockResources - } + return blockNodeService.info(nodeDetail, range) } suspend fun deleteNodeBlocks(projectId: String, repoName: String, nodeFullPath: String) { diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt index 83bbcf5470..75b1e48e21 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/service/FileOperationService.kt @@ -33,7 +33,7 @@ import com.tencent.bkrepo.fs.server.api.RRepositoryClient import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.fs.server.context.ReactiveArtifactContextHolder import com.tencent.bkrepo.fs.server.model.NodeAttribute -import com.tencent.bkrepo.fs.server.model.TBlockNode +import com.tencent.bkrepo.common.metadata.model.TBlockNode import com.tencent.bkrepo.fs.server.request.BlockRequest import com.tencent.bkrepo.fs.server.request.FlushRequest import com.tencent.bkrepo.fs.server.storage.CoStorageManager diff --git a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt index 8fe79860e7..aacda5f035 100644 --- a/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt +++ b/src/backend/fs/boot-fs-server/src/main/kotlin/com/tencent/bkrepo/fs/server/storage/CoStorageManager.kt @@ -30,17 +30,17 @@ package com.tencent.bkrepo.fs.server.storage import com.tencent.bkrepo.common.artifact.api.ArtifactFile import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.storage.pojo.RegionResource import com.tencent.bkrepo.fs.server.RepositoryCache -import com.tencent.bkrepo.fs.server.model.TBlockNode -import com.tencent.bkrepo.fs.server.service.BlockNodeService import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext class CoStorageManager( - private val blockNodeService: BlockNodeService, + private val blockNodeService: RBlockNodeService, private val storageService: StorageService ) { diff --git a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt index 8ff6833fdf..6e408e410b 100644 --- a/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt +++ b/src/backend/fs/boot-fs-server/src/test/kotlin/com/tencent/com/bkrepo/fs/service/BlockNodeServiceTest.kt @@ -4,11 +4,11 @@ import com.tencent.bkrepo.common.api.constant.StringPool import com.tencent.bkrepo.common.api.message.CommonMessageCode import com.tencent.bkrepo.common.api.pojo.Response import com.tencent.bkrepo.common.artifact.stream.Range +import com.tencent.bkrepo.common.metadata.dao.RBlockNodeDao +import com.tencent.bkrepo.common.metadata.model.TBlockNode +import com.tencent.bkrepo.common.metadata.service.blocknode.RBlockNodeService import com.tencent.bkrepo.common.storage.credentials.FileSystemCredentials import com.tencent.bkrepo.fs.server.api.RRepositoryClient -import com.tencent.bkrepo.fs.server.model.TBlockNode -import com.tencent.bkrepo.fs.server.repository.BlockNodeRepository -import com.tencent.bkrepo.fs.server.service.BlockNodeService import com.tencent.bkrepo.repository.pojo.node.NodeDetail import com.tencent.bkrepo.repository.pojo.node.NodeInfo import com.tencent.com.bkrepo.fs.UT_PROJECT_ID @@ -36,7 +36,7 @@ import reactor.core.publisher.Mono import java.time.LocalDateTime @DataMongoTest -@Import(BlockNodeRepository::class) +@Import(RBlockNodeDao::class) @SpringBootConfiguration @EnableAutoConfiguration @TestPropertySource(locations = ["classpath:bootstrap-ut.properties"]) @@ -46,16 +46,16 @@ class BlockNodeServiceTest { lateinit var rRepositoryClient: RRepositoryClient @Autowired - lateinit var blockNodeService: BlockNodeService + lateinit var blockNodeService: RBlockNodeService @Autowired - lateinit var blockNodeRepository: BlockNodeRepository + lateinit var blockNodeDao: RBlockNodeDao private val storageCredentials = FileSystemCredentials() @BeforeEach fun beforeEach() { val criteria = where(TBlockNode::repoName).isEqualTo(UT_REPO_NAME) - runBlocking { blockNodeRepository.remove(Query(criteria)) } + runBlocking { blockNodeDao.remove(Query(criteria)) } } @DisplayName("测试创建块") diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt index 652c05dc55..e9769d0f3d 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/IdleNodeArchiveJob.kt @@ -30,8 +30,8 @@ package com.tencent.bkrepo.job.batch.task.archive import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.constant.ArchiveStorageClass import com.tencent.bkrepo.archive.request.CreateArchiveFileRequest +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.base.MongoDbBatchJob import com.tencent.bkrepo.job.batch.context.NodeContext diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/SystemGcJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/SystemGcJob.kt index 1f225c805a..5684aa9130 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/SystemGcJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/archive/SystemGcJob.kt @@ -33,12 +33,12 @@ import com.tencent.bkrepo.archive.request.CompressFileRequest import com.tencent.bkrepo.common.api.collection.groupBySimilar import com.tencent.bkrepo.common.api.constant.retry import com.tencent.bkrepo.common.api.util.HumanReadable +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.mongo.constant.MIN_OBJECT_ID import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils import com.tencent.bkrepo.common.service.exception.RemoteErrorCodeException import com.tencent.bkrepo.common.storage.credentials.StorageCredentials -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.base.DefaultContextJob import com.tencent.bkrepo.job.batch.base.JobContext diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt index 1464c76a63..f2a245312b 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/batch/task/clean/DeletedNodeCleanupJob.kt @@ -34,9 +34,9 @@ import com.google.common.util.concurrent.UncheckedExecutionException import com.mongodb.client.result.DeleteResult import com.tencent.bkrepo.common.api.constant.CharPool import com.tencent.bkrepo.common.artifact.exception.RepoNotFoundException +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.constant.ID import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.DELETED_DATE import com.tencent.bkrepo.job.NAME import com.tencent.bkrepo.job.PROJECT diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt index cd7c859a56..9fa97b33f2 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/migrate/executor/BaseTaskExecutor.kt @@ -28,10 +28,10 @@ package com.tencent.bkrepo.job.migrate.executor import com.tencent.bkrepo.common.api.util.HumanReadable +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.service.actuator.ActuatorConfiguration.Companion.SERVICE_INSTANCE_ID import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.storage.monitor.measureThroughput -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.migrate.config.MigrateRepoStorageProperties import com.tencent.bkrepo.job.migrate.dao.MigrateFailedNodeDao import com.tencent.bkrepo.job.migrate.dao.MigrateRepoStorageTaskDao diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationNode.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationNode.kt index ac63fec108..967c04bdff 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationNode.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationNode.kt @@ -54,8 +54,8 @@ package com.tencent.bkrepo.job.separation.model -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.job.separation.model.TSeparationNode.Companion.SEPARATION_FOLDER_IDX import com.tencent.bkrepo.job.separation.model.TSeparationNode.Companion.SEPARATION_FOLDER_IDX_DEF import com.tencent.bkrepo.job.separation.model.TSeparationNode.Companion.SEPARATION_FULL_PATH_IDX diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationPackageVersion.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationPackageVersion.kt index 6f9cb4b13f..0c33d8c9c5 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationPackageVersion.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/TSeparationPackageVersion.kt @@ -54,8 +54,8 @@ package com.tencent.bkrepo.job.separation.model -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.job.separation.model.TSeparationPackageVersion.Companion.SEPARATION_VERSION_NAME_IDX import com.tencent.bkrepo.job.separation.model.TSeparationPackageVersion.Companion.SEPARATION_VERSION_NAME_IDX_DEF import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/repo/TSeparationMavenMetadataRecord.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/repo/TSeparationMavenMetadataRecord.kt index 99fe6f6b23..68d21ab8a7 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/repo/TSeparationMavenMetadataRecord.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/model/repo/TSeparationMavenMetadataRecord.kt @@ -54,8 +54,8 @@ package com.tencent.bkrepo.job.separation.model.repo -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import org.springframework.data.mongodb.core.index.CompoundIndex import org.springframework.data.mongodb.core.index.CompoundIndexes import java.time.LocalDateTime diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt index 588ea5dfd4..682b756cbb 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/separation/service/impl/AbstractHandler.kt @@ -29,8 +29,8 @@ package com.tencent.bkrepo.job.separation.service.impl import com.tencent.bkrepo.common.api.exception.NotFoundException import com.tencent.bkrepo.common.api.message.CommonMessageCode +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.constant.ID -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.separation.dao.SeparationFailedRecordDao import com.tencent.bkrepo.job.separation.dao.SeparationTaskDao import com.tencent.bkrepo.job.separation.pojo.NodeFilterInfo @@ -218,4 +218,4 @@ open class AbstractHandler( companion object { private val logger = LoggerFactory.getLogger(AbstractHandler::class.java) } -} \ No newline at end of file +} diff --git a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt index ec6378970d..7d54f2d38c 100644 --- a/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt +++ b/src/backend/job/biz-job/src/main/kotlin/com/tencent/bkrepo/job/service/impl/ArchiveJobServiceImpl.kt @@ -4,9 +4,9 @@ import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.archive.constant.ArchiveStorageClass import com.tencent.bkrepo.archive.request.ArchiveFileRequest import com.tencent.bkrepo.archive.request.UncompressFileRequest +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.dao.util.sharding.HashShardingUtils import com.tencent.bkrepo.common.query.util.MongoEscapeUtils -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.job.BATCH_SIZE import com.tencent.bkrepo.job.SHARDING_COUNT import com.tencent.bkrepo.job.batch.context.NodeContext diff --git a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt index 65a6ec3c7d..b6bc6460f4 100644 --- a/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt +++ b/src/backend/job/biz-job/src/test/kotlin/com/tencent/bkrepo/job/migrate/executor/MigrateExecutorTest.kt @@ -1,6 +1,6 @@ package com.tencent.bkrepo.job.migrate.executor -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.job.UT_PROJECT_ID import com.tencent.bkrepo.job.UT_REPO_NAME import com.tencent.bkrepo.job.UT_STORAGE_CREDENTIALS_KEY diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterClusterReplicator.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterClusterReplicator.kt index ad22b0261a..5b045fc5a8 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterClusterReplicator.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/replicator/commitedge/CenterClusterReplicator.kt @@ -27,9 +27,9 @@ package com.tencent.bkrepo.replication.replica.replicator.commitedge +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeCenterCondition -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.replica.context.ReplicaContext diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/AbstractReplicaService.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/AbstractReplicaService.kt index da14276aa5..a0de01d3ea 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/AbstractReplicaService.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/AbstractReplicaService.kt @@ -32,7 +32,7 @@ import com.tencent.bkrepo.common.api.constant.DEFAULT_PAGE_NUMBER import com.tencent.bkrepo.common.api.pojo.ClusterNodeType import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.replication.manager.LocalDataManager import com.tencent.bkrepo.replication.pojo.metrics.ReplicationRecord import com.tencent.bkrepo.replication.pojo.record.ExecutionResult diff --git a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt index f2158cdd37..abcbbb1cd9 100644 --- a/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt +++ b/src/backend/replication/biz-replication/src/main/kotlin/com/tencent/bkrepo/replication/replica/type/edge/EdgeReplicaTaskJob.kt @@ -33,12 +33,12 @@ import com.tencent.bkrepo.common.api.util.readJsonString import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException import com.tencent.bkrepo.common.artifact.exception.PackageNotFoundException import com.tencent.bkrepo.common.artifact.exception.VersionNotFoundException +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeEdgeCondition import com.tencent.bkrepo.common.service.feign.FeignClientFactory import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace import com.tencent.bkrepo.common.service.util.UrlUtils -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.replication.api.cluster.ClusterReplicaTaskClient import com.tencent.bkrepo.replication.config.ReplicationProperties import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus diff --git a/src/backend/repository/biz-repository/build.gradle.kts b/src/backend/repository/biz-repository/build.gradle.kts index c3ac8bd752..e992c620ab 100644 --- a/src/backend/repository/biz-repository/build.gradle.kts +++ b/src/backend/repository/biz-repository/build.gradle.kts @@ -42,4 +42,5 @@ dependencies { testImplementation("org.mockito.kotlin:mockito-kotlin") testImplementation("io.mockk:mockk") implementation("com.tencent.bk.sdk:bk-notice-java-sdk:${Versions.Notice}") + implementation(project(":common:common-metadata:metadata-service")) } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TFileReference.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TFileReference.kt index b249753af7..cc9750d100 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TFileReference.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TFileReference.kt @@ -31,8 +31,8 @@ package com.tencent.bkrepo.repository.model -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.repository.constant.SHARDING_COUNT import org.springframework.data.mongodb.core.index.CompoundIndex import org.springframework.data.mongodb.core.index.CompoundIndexes diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt index 7023dc5b9d..cf6f57bee1 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/model/TNode.kt @@ -31,8 +31,8 @@ package com.tencent.bkrepo.repository.model -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingDocument -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingDocument +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.repository.constant.SHARDING_COUNT import com.tencent.bkrepo.repository.model.TNode.Companion.ARCHIVED_IDX import com.tencent.bkrepo.repository.model.TNode.Companion.ARCHIVED_IDX_DEF diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt new file mode 100644 index 0000000000..bd8d5b9f31 --- /dev/null +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/blocknode/BlockNodeServiceImpl.kt @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.bkrepo.repository.service.blocknode + +import com.tencent.bkrepo.common.artifact.exception.NodeNotFoundException +import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao +import com.tencent.bkrepo.common.metadata.service.blocknode.impl.AbstractBlockNodeService +import com.tencent.bkrepo.repository.dao.NodeDao +import com.tencent.bkrepo.repository.pojo.node.NodeDetail +import com.tencent.bkrepo.repository.service.file.FileReferenceService +import com.tencent.bkrepo.repository.service.node.impl.NodeBaseService +import org.springframework.context.annotation.Primary +import org.springframework.stereotype.Service + +@Service +@Primary +class BlockNodeServiceImpl( + blockNodeDao: BlockNodeDao, + private val fileReferenceService: FileReferenceService, + private val nodeDao: NodeDao +) : AbstractBlockNodeService(blockNodeDao) { + + override fun incFileRef(sha256: String, credentialsKey: String?) { + fileReferenceService.increment(sha256, credentialsKey) + } + + override fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail { + val node = nodeDao.findNode(projectId, repoName, fullPath) ?: throw NodeNotFoundException(fullPath) + return NodeBaseService.convertToDetail(node)!! + } +} diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/FsServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/FsServiceImpl.kt index d8575b05aa..edaff62bc8 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/FsServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/fs/impl/FsServiceImpl.kt @@ -31,10 +31,10 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.api.util.Preconditions import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils +import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.service.util.SpringContextUtils import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition -import com.tencent.bkrepo.fs.server.constant.FAKE_MD5 -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.model.TNode import com.tencent.bkrepo.repository.pojo.node.NodeDetail diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt index 36d9573364..6758011107 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeBaseService.kt @@ -42,6 +42,9 @@ import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.constant.FAKE_MD5 +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.query.model.Sort import com.tencent.bkrepo.common.security.manager.PermissionManager @@ -51,9 +54,6 @@ import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publi import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.constant.BinderType import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient -import com.tencent.bkrepo.fs.server.constant.FAKE_MD5 -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.constant.SYSTEM_USER import com.tencent.bkrepo.repository.dao.NodeDao @@ -105,7 +105,7 @@ abstract class NodeBaseService( open val servicePermissionClient: ServicePermissionClient, open val routerControllerClient: RouterControllerClient, open val routerControllerProperties: RouterControllerProperties, - open val fsNodeClient: FsNodeClient + open val blockNodeService: BlockNodeService ) : NodeService { @Autowired diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt index 1ff2302842..111193026a 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeRestoreSupport.kt @@ -35,9 +35,9 @@ import com.tencent.bkrepo.common.api.exception.ErrorCodeException import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils.isRoot +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.security.util.SecurityUtils -import com.tencent.bkrepo.fs.server.api.FsNodeClient -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.model.TNode @@ -73,7 +73,7 @@ open class NodeRestoreSupport( ) : NodeRestoreOperation { val nodeDao: NodeDao = nodeBaseService.nodeDao - val fsNodeClient: FsNodeClient = nodeBaseService.fsNodeClient + val blockNodeService: BlockNodeService = nodeBaseService.blockNodeService override fun getDeletedNodeDetail(artifact: ArtifactInfo): List { with(artifact) { @@ -178,12 +178,12 @@ open class NodeRestoreSupport( val deletedNode = nodeDao.findAndModify(query, nodeRestoreUpdate(), option, TNode::class.java) if (deletedNode?.sha256 == FAKE_SHA256 || deletedNode?.metadata?.find { it.key == FS_ATTR_KEY } != null) { try { - fsNodeClient.restoreBlockResources( + blockNodeService.restoreBlocks( projectId = projectId, repoName = repoName, fullPath = fullPath, - nodeCreateDate = deletedNode.createdDate.toString(), - nodeDeleteDate = deletedNode.deleted!!.toString() + nodeCreateDate = deletedNode.createdDate, + nodeDeleteDate = deletedNode.deleted!! ) } catch (e: Exception) { logger.error("restore block resources failed: $projectId/$repoName/$fullPath", e) @@ -202,7 +202,7 @@ open class NodeRestoreSupport( with(node) { val query = nodeQuery(projectId, repoName, fullPath) if (node.sha256 == FAKE_SHA256 || node.metadata?.find { it.key == FS_ATTR_KEY } != null) { - fsNodeClient.deleteBlockResources(projectId, repoName, fullPath) + blockNodeService.deleteBlocks(projectId, repoName, fullPath) } nodeDao.updateFirst(query, NodeQueryHelper.nodeDeleteUpdate(userId)) } diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt index 98a7bd8351..86bc0a7987 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/NodeServiceImpl.kt @@ -31,10 +31,10 @@ import com.tencent.bkrepo.archive.api.ArchiveClient import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.artifact.api.ArtifactInfo import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.dao.RepositoryDao @@ -76,7 +76,7 @@ class NodeServiceImpl( override val servicePermissionClient: ServicePermissionClient, override val routerControllerClient: RouterControllerClient, override val routerControllerProperties: RouterControllerProperties, - override val fsNodeClient: FsNodeClient, + override val blockNodeService: BlockNodeService, private val archiveClient: ArchiveClient, ) : NodeBaseService( nodeDao, @@ -90,7 +90,7 @@ class NodeServiceImpl( servicePermissionClient, routerControllerClient, routerControllerProperties, - fsNodeClient + blockNodeService ) { override fun computeSize( diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt index 1702ae0a57..3ded79ab00 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/center/CommitEdgeCenterNodeServiceImpl.kt @@ -35,12 +35,12 @@ import com.tencent.bkrepo.common.artifact.message.ArtifactMessageCode import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties import com.tencent.bkrepo.common.artifact.util.ClusterUtils +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.security.util.SecurityUtils import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeCenterCondition import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.dao.RepositoryDao @@ -84,7 +84,7 @@ class CommitEdgeCenterNodeServiceImpl( override val servicePermissionClient: ServicePermissionClient, override val routerControllerClient: RouterControllerClient, override val routerControllerProperties: RouterControllerProperties, - override val fsNodeClient: FsNodeClient, + override val blockNodeService: BlockNodeService, val clusterProperties: ClusterProperties, val archiveClient: ArchiveClient, ) : NodeServiceImpl( @@ -99,7 +99,7 @@ class CommitEdgeCenterNodeServiceImpl( servicePermissionClient, routerControllerClient, routerControllerProperties, - fsNodeClient, + blockNodeService, archiveClient ) { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt index 0f7084ed58..bcc9388a07 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeBaseService.kt @@ -30,11 +30,11 @@ package com.tencent.bkrepo.repository.service.node.impl.edge import com.tencent.bkrepo.auth.api.ServicePermissionClient import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties import com.tencent.bkrepo.common.artifact.util.ClusterUtils.reportMetadataToCenter +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.service.feign.FeignClientFactory import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.repository.api.cluster.ClusterNodeClient import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.dao.NodeDao @@ -62,7 +62,7 @@ abstract class EdgeNodeBaseService( override val routerControllerClient: RouterControllerClient, override val servicePermissionClient: ServicePermissionClient, override val routerControllerProperties: RouterControllerProperties, - override val fsNodeClient: FsNodeClient, + override val blockNodeService: BlockNodeService, open val clusterProperties: ClusterProperties ) : NodeBaseService( nodeDao, @@ -76,7 +76,7 @@ abstract class EdgeNodeBaseService( servicePermissionClient, routerControllerClient, routerControllerProperties, - fsNodeClient + blockNodeService ) { val centerNodeClient: ClusterNodeClient by lazy { diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt index 493476f95c..cf723ba665 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/node/impl/edge/EdgeNodeServiceImpl.kt @@ -34,11 +34,11 @@ import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties import com.tencent.bkrepo.common.artifact.util.ClusterUtils.ignoreException import com.tencent.bkrepo.common.artifact.util.ClusterUtils.nodeLevelNotFoundError import com.tencent.bkrepo.common.artifact.util.ClusterUtils.repoLevelNotFoundError +import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService import com.tencent.bkrepo.common.service.cluster.condition.CommitEdgeEdgeCondition import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.dao.NodeDao import com.tencent.bkrepo.repository.dao.RepositoryDao @@ -89,7 +89,7 @@ class EdgeNodeServiceImpl( override val clusterProperties: ClusterProperties, override val routerControllerClient: RouterControllerClient, override val routerControllerProperties: RouterControllerProperties, - override val fsNodeClient: FsNodeClient, + override val blockNodeService: BlockNodeService, val archiveClient: ArchiveClient, ) : EdgeNodeBaseService( nodeDao, @@ -103,7 +103,7 @@ class EdgeNodeServiceImpl( routerControllerClient, servicePermissionClient, routerControllerProperties, - fsNodeClient, + blockNodeService, clusterProperties, ) { override fun computeSize( diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt index 386499d830..69d6aaa315 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/service/repo/impl/RepositoryServiceImpl.kt @@ -49,6 +49,7 @@ import com.tencent.bkrepo.common.artifact.pojo.configuration.composite.ProxyConf import com.tencent.bkrepo.common.artifact.pojo.configuration.local.LocalConfiguration import com.tencent.bkrepo.common.artifact.pojo.configuration.remote.RemoteConfiguration import com.tencent.bkrepo.common.artifact.pojo.configuration.virtual.VirtualConfiguration +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.mongo.dao.AbstractMongoDao.Companion.ID import com.tencent.bkrepo.common.mongo.dao.util.Pages import com.tencent.bkrepo.common.security.util.RsaUtils @@ -57,7 +58,6 @@ import com.tencent.bkrepo.common.service.cluster.condition.DefaultCondition import com.tencent.bkrepo.common.service.util.SpringContextUtils.Companion.publishEvent import com.tencent.bkrepo.common.storage.credentials.StorageCredentials import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.repository.config.RepositoryProperties import com.tencent.bkrepo.repository.dao.RepositoryDao import com.tencent.bkrepo.repository.dao.repository.ProjectMetricsRepository diff --git a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt index d0cb05bb8f..f634a784ef 100644 --- a/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt +++ b/src/backend/repository/biz-repository/src/main/kotlin/com/tencent/bkrepo/repository/util/NodeQueryHelper.kt @@ -32,9 +32,9 @@ import com.tencent.bkrepo.common.artifact.path.PathUtils import com.tencent.bkrepo.common.artifact.path.PathUtils.escapeRegex import com.tencent.bkrepo.common.artifact.path.PathUtils.toFullPath import com.tencent.bkrepo.common.artifact.path.PathUtils.toPath +import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256 import com.tencent.bkrepo.common.query.enums.OperationType import com.tencent.bkrepo.common.query.util.MongoEscapeUtils -import com.tencent.bkrepo.fs.server.constant.FAKE_SHA256 import com.tencent.bkrepo.repository.model.TNode import com.tencent.bkrepo.repository.pojo.metadata.MetadataModel import com.tencent.bkrepo.repository.pojo.node.NodeListOption diff --git a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt index 614c400906..928b6455c7 100644 --- a/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt +++ b/src/backend/repository/biz-repository/src/test/kotlin/com/tencent/bkrepo/repository/service/ServiceBaseTest.kt @@ -43,6 +43,7 @@ import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory import com.tencent.bkrepo.common.artifact.pojo.RepositoryType import com.tencent.bkrepo.common.artifact.pojo.configuration.local.LocalConfiguration import com.tencent.bkrepo.common.artifact.router.RouterControllerProperties +import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao import com.tencent.bkrepo.common.security.http.core.HttpAuthProperties import com.tencent.bkrepo.common.security.manager.PermissionManager import com.tencent.bkrepo.common.security.manager.ci.CIPermissionManager @@ -52,7 +53,6 @@ import com.tencent.bkrepo.common.service.util.SpringContextUtils import com.tencent.bkrepo.common.storage.core.StorageProperties import com.tencent.bkrepo.common.storage.core.StorageService import com.tencent.bkrepo.common.stream.event.supplier.MessageSupplier -import com.tencent.bkrepo.fs.server.api.FsNodeClient import com.tencent.bkrepo.repository.UT_PROJECT_ID import com.tencent.bkrepo.repository.UT_REPO_DESC import com.tencent.bkrepo.repository.UT_REPO_DISPLAY @@ -97,7 +97,8 @@ import org.springframework.test.context.TestPropertySource HttpAuthProperties::class, SpringContextUtils::class, NodeDao::class, - RouterControllerProperties::class + RouterControllerProperties::class, + BlockNodeDao::class ) @ComponentScan("com.tencent.bkrepo.repository.service") @TestPropertySource(locations = ["classpath:bootstrap-ut.properties", "classpath:center-ut.properties"]) @@ -133,9 +134,6 @@ open class ServiceBaseTest { @MockBean lateinit var routerControllerClient: RouterControllerClient - @MockBean - lateinit var fsNodeClient: FsNodeClient - @Autowired lateinit var springContextUtils: SpringContextUtils @@ -177,8 +175,6 @@ open class ServiceBaseTest { whenever(messageSupplier.delegateToSupplier(any(), anyOrNull(), anyString(), anyOrNull(), any())) .then {} whenever(resourcePermissionListener.handle(any())).then {} - whenever(fsNodeClient.restoreBlockResources(anyString(), anyString(), anyString(), anyString(), anyString())) - .then {} } fun initRepoForUnitTest( diff --git a/src/backend/settings.gradle.kts b/src/backend/settings.gradle.kts index 25925627e1..4d0fae6d4f 100644 --- a/src/backend/settings.gradle.kts +++ b/src/backend/settings.gradle.kts @@ -91,3 +91,4 @@ includeAll(":archive") includeAll(":s3") includeAll(":router-controller") includeAll(":media") +includeAll(":common:common-metadata") diff --git a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/model/TWebHookLog.kt b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/model/TWebHookLog.kt index 07905bad86..010bc3ac53 100644 --- a/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/model/TWebHookLog.kt +++ b/src/backend/webhook/biz-webhook/src/main/kotlin/com/tencent/bkrepo/webhook/model/TWebHookLog.kt @@ -28,7 +28,7 @@ package com.tencent.bkrepo.webhook.model import com.tencent.bkrepo.common.artifact.event.base.EventType -import com.tencent.bkrepo.common.mongo.dao.sharding.ShardingKey +import com.tencent.bkrepo.common.api.mongo.ShardingKey import com.tencent.bkrepo.webhook.constant.WebHookRequestStatus import org.springframework.data.mongodb.core.mapping.Document import java.time.LocalDateTime