feat: FileReference抽取到common-metadata TencentBlueKing#2470

* feat: FileReference抽取到common-metadata TencentBlueKing#2470 * feat: FileReference抽取到common-metadata TencentBlueKing#2470
lannoy0523 · Sep 11, 2024 · c68401b · c68401b
1 parent 337187d
commit c68401b
Show file tree

Hide file tree

Showing 64 changed files with 473 additions and 249 deletions.
diff --git a/src/backend/archive/biz-archive/build.gradle.kts b/src/backend/archive/biz-archive/build.gradle.kts
@@ -6,10 +6,11 @@ dependencies {
     api(project(":common:common-service"))
     api(project(":common:common-mongo"))
     api(project(":common:common-mongo-reactive"))
+    api(project(":common:common-metadata:metadata-service"))
     implementation("io.micrometer:micrometer-registry-prometheus")
     implementation("org.apache.tika:tika-core")
     testImplementation("org.springframework.boot:spring-boot-starter-test")
     testImplementation("de.flapdoodle.embed:de.flapdoodle.embed.mongo")
     testImplementation("org.mockito.kotlin:mockito-kotlin")
     testImplementation("io.mockk:mockk")
-}
+}
diff --git a/...hive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManager.kt b/...hive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManager.kt
@@ -3,12 +3,12 @@ package com.tencent.bkrepo.archive.core.compress
 import com.google.common.util.concurrent.ThreadFactoryBuilder
 import com.tencent.bkrepo.archive.CompressStatus
 import com.tencent.bkrepo.archive.config.ArchiveProperties
+import com.tencent.bkrepo.archive.core.TaskResult
 import com.tencent.bkrepo.archive.core.provider.FileProviderFactory
 import com.tencent.bkrepo.archive.core.provider.FileTask
+import com.tencent.bkrepo.archive.core.provider.PriorityFileProvider
 import com.tencent.bkrepo.archive.event.StorageFileCompressedEvent
 import com.tencent.bkrepo.archive.event.StorageFileUncompressedEvent
-import com.tencent.bkrepo.archive.core.provider.PriorityFileProvider
-import com.tencent.bkrepo.archive.core.TaskResult
 import com.tencent.bkrepo.archive.model.TCompressFile
 import com.tencent.bkrepo.archive.repository.CompressFileDao
 import com.tencent.bkrepo.archive.repository.CompressFileRepository
@@ -17,11 +17,11 @@ import com.tencent.bkrepo.archive.utils.ArchiveUtils
 import com.tencent.bkrepo.common.artifact.api.toArtifactFile
 import com.tencent.bkrepo.common.artifact.stream.Range
 import com.tencent.bkrepo.common.bksync.transfer.exception.TooLowerReuseRateException
+import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
 import com.tencent.bkrepo.common.service.util.SpringContextUtils
 import com.tencent.bkrepo.common.storage.core.StorageService
 import com.tencent.bkrepo.common.storage.monitor.Throughput
 import com.tencent.bkrepo.common.storage.util.toPath
-import com.tencent.bkrepo.repository.api.FileReferenceClient
 import org.slf4j.LoggerFactory
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.context.annotation.Lazy
@@ -50,7 +50,7 @@ class BDZipManager(
 
     @Autowired
     @Lazy
-    private lateinit var fileReferenceClient: FileReferenceClient
+    private lateinit var fileReferenceService: FileReferenceService
 
     @Autowired
     @Lazy
@@ -190,7 +190,7 @@ class BDZipManager(
                 if (it !is TooLowerReuseRateException) {
                     logger.error("Failed to compress file [$sha256].", it)
                 }
-                fileReferenceClient.decrement(baseSha256, storageCredentialsKey)
+                fileReferenceService.decrement(baseSha256, storageCredentialsKey)
                 file.status = CompressStatus.COMPRESS_FAILED
                 file.lastModifiedDate = LocalDateTime.now()
                 compressFileRepository.save(file)

diff --git a/...ive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt b/...ive/biz-archive/src/main/kotlin/com/tencent/bkrepo/archive/service/CompressServiceImpl.kt
@@ -15,9 +15,9 @@ import com.tencent.bkrepo.archive.request.UncompressFileRequest
 import com.tencent.bkrepo.archive.request.UpdateCompressFileStatusRequest
 import com.tencent.bkrepo.archive.utils.ArchiveUtils
 import com.tencent.bkrepo.common.api.exception.ErrorCodeException
+import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
 import com.tencent.bkrepo.common.service.util.SpringContextUtils
 import com.tencent.bkrepo.common.storage.core.StorageService
-import com.tencent.bkrepo.repository.api.FileReferenceClient
 import org.slf4j.LoggerFactory
 import org.springframework.stereotype.Service
 import java.time.LocalDateTime
@@ -29,7 +29,7 @@ import java.time.LocalDateTime
 class CompressServiceImpl(
     private val compressFileRepository: CompressFileRepository,
     private val storageService: StorageService,
-    private val fileReferenceClient: FileReferenceClient,
+    private val fileReferenceService: FileReferenceService,
 ) : CompressService {
 
     override fun compress(request: CompressFileRequest) {
@@ -99,7 +99,7 @@ class CompressServiceImpl(
                 status = CompressStatus.CREATED,
             )
             compressFileRepository.save(compressFile)
-            fileReferenceClient.increment(baseSha256, storageCredentialsKey)
+            fileReferenceService.increment(baseSha256, storageCredentialsKey)
             SpringContextUtils.publishEvent(FileEntityEvent(sha256, compressFile))
             logger.info("Compress file [$sha256] on $storageCredentialsKey.")
         }
@@ -133,7 +133,7 @@ class CompressServiceImpl(
                     storageService.deleteCompressed(sha256, storageCredentials)
                 }
                 // 压缩失败的已经解除了base sha256的引用
-                fileReferenceClient.decrement(file.baseSha256, storageCredentialsKey)
+                fileReferenceService.decrement(file.baseSha256, storageCredentialsKey)
             }
             /*
             * 解压是小概率事件，所以这里链长度我们就不减少，这样带来的问题是，

diff --git a/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/BaseTest.kt b/src/backend/archive/biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/BaseTest.kt
@@ -14,7 +14,7 @@ import org.springframework.cloud.sleuth.otel.bridge.OtelTracer
 import org.springframework.context.annotation.ComponentScan
 import org.springframework.test.context.TestPropertySource
 
-@ComponentScan("com.tencent.bkrepo.archive")
+@ComponentScan(value = ["com.tencent.bkrepo.archive", "com.tencent.bkrepo.common.metadata"])
 @SpringBootConfiguration
 @EnableAutoConfiguration
 @TestPropertySource(locations = ["classpath:bootstrap-ut.properties"])

diff --git a/.../biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt b/.../biz-archive/src/test/kotlin/com/tencent/bkrepo/archive/core/compress/BDZipManagerTest.kt
@@ -10,10 +10,9 @@ import com.tencent.bkrepo.common.artifact.api.FileSystemArtifactFile
 import com.tencent.bkrepo.common.artifact.hash.sha256
 import com.tencent.bkrepo.common.artifact.stream.Range
 import com.tencent.bkrepo.common.bksync.file.BkSyncDeltaSource.Companion.toBkSyncDeltaSource
-import com.tencent.bkrepo.common.service.util.ResponseBuilder
+import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
 import com.tencent.bkrepo.common.storage.StorageAutoConfiguration
 import com.tencent.bkrepo.common.storage.core.StorageService
-import com.tencent.bkrepo.repository.api.FileReferenceClient
 import com.tencent.bkrepo.repository.api.RepositoryClient
 import com.tencent.bkrepo.repository.api.StorageCredentialsClient
 import org.junit.jupiter.api.AfterEach
@@ -44,7 +43,7 @@ class BDZipManagerTest @Autowired constructor(
 ) : BaseTest() {
 
     @MockBean
-    lateinit var fileReferenceClient: FileReferenceClient
+    lateinit var fileReferenceService: FileReferenceService
 
     @MockBean
     lateinit var storageCredentialsClient: StorageCredentialsClient
@@ -84,10 +83,10 @@ class BDZipManagerTest @Autowired constructor(
     @Test
     fun compressFailedTest() {
         var decrement = 0
-        Mockito.`when`(fileReferenceClient.decrement(ArgumentMatchers.anyString(), ArgumentMatchers.isNull())).then {
+        Mockito.`when`(fileReferenceService.decrement(ArgumentMatchers.anyString(), ArgumentMatchers.isNull())).then {
             println("decrement file reference")
             decrement++
-            ResponseBuilder.success(true)
+            true
         }
         val data1 = Random.nextBytes(Random.nextInt(1024, 1 shl 20))
         val data2 = data1.copyOfRange(Random.nextInt(1, 10), data1.size)

diff --git a/...fact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt b/...fact-service/src/main/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManager.kt
@@ -35,12 +35,12 @@ import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream
 import com.tencent.bkrepo.common.artifact.stream.EmptyInputStream
 import com.tencent.bkrepo.common.artifact.stream.Range
 import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils.resolveRange
+import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
 import com.tencent.bkrepo.common.security.util.SecurityUtils
 import com.tencent.bkrepo.common.service.util.HttpContextHolder.getRequestOrNull
 import com.tencent.bkrepo.common.storage.core.StorageService
 import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
 import com.tencent.bkrepo.common.storage.innercos.http.HttpMethod
-import com.tencent.bkrepo.repository.api.FileReferenceClient
 import com.tencent.bkrepo.repository.api.NodeClient
 import com.tencent.bkrepo.repository.pojo.node.NodeDetail
 import com.tencent.bkrepo.repository.pojo.node.NodeInfo
@@ -66,7 +66,7 @@ import org.slf4j.LoggerFactory
 class StorageManager(
     private val storageService: StorageService,
     private val nodeClient: NodeClient,
-    private val fileReferenceClient: FileReferenceClient,
+    private val fileReferenceService: FileReferenceService,
     private val nodeResourceFactory: NodeResourceFactory,
     private val pluginManager: PluginManager,
 ) {
@@ -88,7 +88,7 @@ class StorageManager(
                 try {
                     // 当createNode调用超时，实际node和引用创建成功时不会做任何改变
                     // 当文件创建成功，但是node创建失败时，则创建一个计数为0的fileReference用于清理任务清理垃圾文件
-                    fileReferenceClient.increment(request.sha256!!, storageCredentials?.key, 0L)
+                    fileReferenceService.increment(request.sha256!!, storageCredentials?.key, 0L)
                 } catch (exception: Exception) {
                     // 创建引用失败后会通过定时任务StorageReconcileJob清理垃圾文件
                     logger.error("Failed to create ref for new created file[${request.sha256}]", exception)

diff --git a/...-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManagerTest.kt b/...-service/src/test/kotlin/com/tencent/bkrepo/common/artifact/manager/StorageManagerTest.kt
@@ -8,8 +8,8 @@ import com.tencent.bkrepo.common.artifact.util.Constant.UT_PROJECT_ID
 import com.tencent.bkrepo.common.artifact.util.Constant.UT_REPO_NAME
 import com.tencent.bkrepo.common.artifact.util.Constant.UT_SHA256
 import com.tencent.bkrepo.common.artifact.util.Constant.UT_USER
+import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
 import com.tencent.bkrepo.common.storage.core.StorageService
-import com.tencent.bkrepo.repository.api.FileReferenceClient
 import com.tencent.bkrepo.repository.api.NodeClient
 import com.tencent.bkrepo.repository.pojo.node.NodeDetail
 import com.tencent.bkrepo.repository.pojo.node.NodeInfo
@@ -50,7 +50,7 @@ class StorageManagerTest @Autowired constructor(
     private val storageManager: StorageManager,
 ) {
     @MockBean
-    private lateinit var fileReferenceClient: FileReferenceClient
+    private lateinit var fileReferenceService: FileReferenceService
 
     @MockBean
     private lateinit var nodeClient: NodeClient
@@ -70,15 +70,15 @@ class StorageManagerTest @Autowired constructor(
             .thenReturn(1)
         whenever(nodeClient.createNode(any()))
             .thenReturn(Response(code = 0, data = buildNodeDetail(UT_SHA256)))
-        whenever(fileReferenceClient.increment(anyString(), anyOrNull(), any()))
-            .thenReturn(Response(code = 0, data = true))
+        whenever(fileReferenceService.increment(anyString(), anyOrNull(), any()))
+            .thenReturn(true)
     }
 
     @Test
     fun testStoreSuccess() {
         store()
         verify(nodeClient, times(1)).createNode(any())
-        verify(fileReferenceClient, times(0)).increment(anyString(), anyOrNull(), any())
+        verify(fileReferenceService, times(0)).increment(anyString(), anyOrNull(), any())
     }
 
     @Test
@@ -94,7 +94,7 @@ class StorageManagerTest @Autowired constructor(
         // store failed
         assertThrows<RuntimeException> { store() }
         verify(nodeClient, times(0)).createNode(any())
-        verify(fileReferenceClient, times(0)).increment(anyString(), anyOrNull(), any())
+        verify(fileReferenceService, times(0)).increment(anyString(), anyOrNull(), any())
 
         // reset mock
         field.set(storageManager, oldStorageService)
@@ -108,7 +108,7 @@ class StorageManagerTest @Autowired constructor(
         // store failed
         assertThrows<RuntimeException> { store() }
         verify(nodeClient, times(1)).createNode(any())
-        verify(fileReferenceClient, times(1)).increment(anyString(), anyOrNull(), any())
+        verify(fileReferenceService, times(1)).increment(anyString(), anyOrNull(), any())
     }
 
     private fun store(): String {

diff --git a/src/backend/common/common-metadata/metadata-api/build.gradle.kts b/src/backend/common/common-metadata/metadata-api/build.gradle.kts
@@ -28,3 +28,7 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
+
+dependencies {
+    api(project(":common:common-api"))
+}
diff --git a/...epo/repository/pojo/file/FileReference.kt → ...ommon/metadata/pojo/file/FileReference.kt b/...epo/repository/pojo/file/FileReference.kt → ...ommon/metadata/pojo/file/FileReference.kt
@@ -25,7 +25,7 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.bkrepo.repository.pojo.file
+package com.tencent.bkrepo.common.metadata.pojo.file
 
 import io.swagger.annotations.ApiModel
 import io.swagger.annotations.ApiModelProperty

diff --git a/...krepo/common/metadata/dao/BlockNodeDao.kt → ...on/metadata/dao/blocknode/BlockNodeDao.kt b/...krepo/common/metadata/dao/BlockNodeDao.kt → ...on/metadata/dao/blocknode/BlockNodeDao.kt
@@ -25,7 +25,7 @@
  *  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.bkrepo.common.metadata.dao
+package com.tencent.bkrepo.common.metadata.dao.blocknode
 
 import com.tencent.bkrepo.common.metadata.condition.SyncCondition
 import com.tencent.bkrepo.common.metadata.model.TBlockNode

diff --git a/...repo/common/metadata/dao/RBlockNodeDao.kt → ...n/metadata/dao/blocknode/RBlockNodeDao.kt b/...repo/common/metadata/dao/RBlockNodeDao.kt → ...n/metadata/dao/blocknode/RBlockNodeDao.kt
@@ -25,7 +25,7 @@
  *  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package com.tencent.bkrepo.common.metadata.dao
+package com.tencent.bkrepo.common.metadata.dao.blocknode
 
 import com.tencent.bkrepo.common.metadata.condition.ReactiveCondition
 import com.tencent.bkrepo.common.metadata.model.TBlockNode

diff --git a/...bkrepo/repository/dao/FileReferenceDao.kt → ...mon/metadata/dao/file/FileReferenceDao.kt b/...bkrepo/repository/dao/FileReferenceDao.kt → ...mon/metadata/dao/file/FileReferenceDao.kt
@@ -29,14 +29,17 @@
  * SOFTWARE.
  */
 
-package com.tencent.bkrepo.repository.dao
+package com.tencent.bkrepo.common.metadata.dao.file
 
+import com.tencent.bkrepo.common.metadata.condition.SyncCondition
 import com.tencent.bkrepo.common.mongo.dao.sharding.HashShardingMongoDao
-import com.tencent.bkrepo.repository.model.TFileReference
+import com.tencent.bkrepo.common.metadata.model.TFileReference
+import org.springframework.context.annotation.Conditional
 import org.springframework.stereotype.Repository
 
 /**
  * 文件摘要引用 Dao
  */
 @Repository
+@Conditional(SyncCondition::class)
 class FileReferenceDao : HashShardingMongoDao<TFileReference>()
diff --git a/...-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/file/RFileReferenceDao.kt b/...-service/src/main/kotlin/com/tencent/bkrepo/common/metadata/dao/file/RFileReferenceDao.kt
@@ -0,0 +1,45 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2020 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package com.tencent.bkrepo.common.metadata.dao.file
+
+import com.tencent.bkrepo.common.metadata.condition.ReactiveCondition
+import com.tencent.bkrepo.common.metadata.model.TFileReference
+import com.tencent.bkrepo.common.mongo.reactive.dao.ShardingMongoReactiveDao
+import org.springframework.context.annotation.Conditional
+import org.springframework.stereotype.Repository
+
+/**
+ * 文件摘要引用 Dao
+ */
+@Repository
+@Conditional(ReactiveCondition::class)
+class RFileReferenceDao : ShardingMongoReactiveDao<TFileReference>()
diff --git a/...bkrepo/repository/model/TFileReference.kt → ...o/common/metadata/model/TFileReference.kt b/...bkrepo/repository/model/TFileReference.kt → ...o/common/metadata/model/TFileReference.kt
@@ -29,7 +29,7 @@
  * SOFTWARE.
  */
 
-package com.tencent.bkrepo.repository.model
+package com.tencent.bkrepo.common.metadata.model
 
 import com.tencent.bkrepo.common.api.mongo.ShardingDocument
 import com.tencent.bkrepo.common.api.mongo.ShardingKey

diff --git a/...lin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt b/...lin/com/tencent/bkrepo/common/metadata/service/blocknode/impl/AbstractBlockNodeService.kt
@@ -30,9 +30,10 @@ package com.tencent.bkrepo.common.metadata.service.blocknode.impl
 import com.tencent.bkrepo.common.artifact.stream.Range
 import com.tencent.bkrepo.common.metadata.constant.FAKE_SHA256
 import com.tencent.bkrepo.common.metadata.constant.ID
-import com.tencent.bkrepo.common.metadata.dao.BlockNodeDao
+import com.tencent.bkrepo.common.metadata.dao.blocknode.BlockNodeDao
 import com.tencent.bkrepo.common.metadata.model.TBlockNode
 import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService
+import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
 import com.tencent.bkrepo.common.metadata.util.BlockNodeQueryHelper
 import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
 import com.tencent.bkrepo.common.storage.pojo.RegionResource
@@ -46,13 +47,14 @@ import org.springframework.data.mongodb.core.query.isEqualTo
 import java.time.LocalDateTime
 
 abstract class AbstractBlockNodeService(
-    private val blockNodeDao: BlockNodeDao
+    private val blockNodeDao: BlockNodeDao,
+    private val fileReferenceService: FileReferenceService
 ) : BlockNodeService {
 
     override fun createBlock(blockNode: TBlockNode, storageCredentials: StorageCredentials?): TBlockNode {
         with(blockNode) {
             val bn = blockNodeDao.save(blockNode)
-            incFileRef(bn.sha256, storageCredentials?.key)
+            fileReferenceService.increment(bn.sha256, storageCredentials?.key)
             logger.info("Create block node[$projectId/$repoName$nodeFullPath-$startPos] ,sha256[$sha256] success.")
             return bn
         }
@@ -131,8 +133,6 @@ abstract class AbstractBlockNodeService(
         }
     }
 
-    abstract fun incFileRef(sha256: String, credentialsKey: String?)
-
     abstract fun getNodeDetail(projectId: String, repoName: String, fullPath: String): NodeDetail
 
     companion object {