Skip to content

Commit

Permalink
feat: StorageCredential提取到common-metadata TencentBlueKing#2547
Browse files Browse the repository at this point in the history
* feat: StorageCredential提取到common-metadata TencentBlueKing#2547

* feat: StorageCredential提取到common-metadata TencentBlueKing#2547

* feat: StorageCredential提取到common-metadata TencentBlueKing#2547
  • Loading branch information
yaoxuwan committed Sep 20, 2024
1 parent 3779d4f commit 785f444
Show file tree
Hide file tree
Showing 132 changed files with 812 additions and 347 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import com.tencent.bkrepo.common.analysis.pojo.scanner.dependencycheck.result.De
import com.tencent.bkrepo.common.analysis.pojo.scanner.dependencycheck.result.DependencyScanExecutorResult
import com.tencent.bkrepo.common.analysis.pojo.scanner.dependencycheck.scanner.DependencyScanner
import com.tencent.bkrepo.common.analysis.pojo.scanner.utils.normalizedLevel
import com.tencent.bkrepo.common.storage.core.StorageProperties
import com.tencent.bkrepo.common.storage.config.StorageProperties
import com.tencent.bkrepo.analysis.executor.ScanExecutor
import com.tencent.bkrepo.analysis.executor.pojo.ScanExecutorTask
import org.slf4j.LoggerFactory
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,35 +4,35 @@ import com.google.common.cache.CacheBuilder
import com.google.common.cache.CacheLoader
import com.google.common.cache.LoadingCache
import com.tencent.bkrepo.archive.config.ArchiveProperties
import com.tencent.bkrepo.common.storage.core.StorageProperties
import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
import com.tencent.bkrepo.common.storage.config.StorageProperties
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.repository.api.RepositoryClient
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Component
import java.util.concurrent.ArrayBlockingQueue
import java.util.concurrent.BlockingQueue
import java.util.concurrent.ThreadFactory
import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.TimeUnit
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Component
import java.util.concurrent.BlockingQueue

@Component
class ArchiveUtils(
storageCredentialsClient: StorageCredentialsClient,
storageCredentialService: StorageCredentialService,
storageProperties: StorageProperties,
repositoryClient: RepositoryClient,
archiveProperties: ArchiveProperties,
) {

init {
Companion.storageCredentialsClient = storageCredentialsClient
Companion.storageCredentialService = storageCredentialService
Companion.repositoryClient = repositoryClient
Companion.archiveProperties = archiveProperties
defaultStorageCredentials = storageProperties.defaultStorageCredentials()
}

companion object {
private lateinit var storageCredentialsClient: StorageCredentialsClient
private lateinit var storageCredentialService: StorageCredentialService
private lateinit var defaultStorageCredentials: StorageCredentials
private lateinit var repositoryClient: RepositoryClient
private lateinit var archiveProperties: ArchiveProperties
Expand All @@ -44,7 +44,7 @@ class ArchiveUtils(

private fun loadStorageCredentials(key: String): StorageCredentials {
if (key.isEmpty()) return defaultStorageCredentials
return storageCredentialsClient.findByKey(key).data ?: defaultStorageCredentials
return storageCredentialService.findByKey(key) ?: defaultStorageCredentials
}

fun getStorageCredentials(key: String?): StorageCredentials {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import com.tencent.bkrepo.common.storage.StorageAutoConfiguration
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.repository.api.FileReferenceClient
import com.tencent.bkrepo.repository.api.RepositoryClient
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
import org.junit.jupiter.api.AfterAll
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.BeforeAll
Expand Down Expand Up @@ -38,9 +37,6 @@ class FileCoreProcessorTest @Autowired constructor(
@MockBean
lateinit var fileReferenceClient: FileReferenceClient

@MockBean
lateinit var storageCredentialsClient: StorageCredentialsClient

@MockBean
lateinit var repositoryClient: RepositoryClient

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ import com.tencent.bkrepo.common.metadata.service.file.FileReferenceService
import com.tencent.bkrepo.common.storage.StorageAutoConfiguration
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.repository.api.RepositoryClient
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
import org.junit.jupiter.api.AfterEach
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.BeforeEach
Expand Down Expand Up @@ -45,9 +44,6 @@ class BDZipManagerTest @Autowired constructor(
@MockBean
lateinit var fileReferenceService: FileReferenceService

@MockBean
lateinit var storageCredentialsClient: StorageCredentialsClient

@MockBean
lateinit var repositoryClient: RepositoryClient

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import com.tencent.bkrepo.common.storage.StorageAutoConfiguration
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.repository.api.FileReferenceClient
import com.tencent.bkrepo.repository.api.RepositoryClient
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
Expand All @@ -38,9 +37,6 @@ class CompressServiceTest @Autowired constructor(
@MockBean
lateinit var fileReferenceClient: FileReferenceClient

@MockBean
lateinit var storageCredentialsClient: StorageCredentialsClient

@MockBean
lateinit var repositoryClient: RepositoryClient

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,15 @@ import com.tencent.bkrepo.common.artifact.cache.service.PreloadListener
import com.tencent.bkrepo.common.artifact.cache.service.PreloadPlanExecutor
import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.storage.core.StorageProperties
import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
import com.tencent.bkrepo.common.storage.config.StorageProperties
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.common.storage.core.cache.CacheStorageService
import com.tencent.bkrepo.common.storage.core.locator.FileLocator
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.common.storage.monitor.Throughput
import com.tencent.bkrepo.common.storage.monitor.measureThroughput
import com.tencent.bkrepo.common.storage.util.existReal
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
import org.slf4j.LoggerFactory
import java.nio.file.Files
import java.nio.file.Paths
Expand All @@ -60,7 +60,7 @@ import java.util.concurrent.TimeUnit
class DefaultPreloadPlanExecutor(
private val preloadProperties: ArtifactPreloadProperties,
private val cacheStorageService: StorageService,
private val storageCredentialsClient: StorageCredentialsClient,
private val storageCredentialService: StorageCredentialService,
private val fileLocator: FileLocator,
private val storageProperties: StorageProperties,
) : PreloadPlanExecutor {
Expand All @@ -75,7 +75,7 @@ class DefaultPreloadPlanExecutor(
require(cacheStorageService is CacheStorageService)
updateExecutor()
val credentials = if (plan.credentialsKey != null) {
storageCredentialsClient.findByKey(plan.credentialsKey).data!!
storageCredentialService.findByKey(plan.credentialsKey)!!
} else {
storageProperties.defaultStorageCredentials()
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,15 @@ import com.tencent.bkrepo.common.artifact.api.ArtifactFile
import com.tencent.bkrepo.common.artifact.api.FileSystemArtifactFile
import com.tencent.bkrepo.common.artifact.cache.config.ArtifactPreloadConfiguration
import com.tencent.bkrepo.common.artifact.cache.config.ArtifactPreloadProperties
import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
import com.tencent.bkrepo.common.storage.StorageAutoConfiguration
import com.tencent.bkrepo.common.storage.core.StorageProperties
import com.tencent.bkrepo.common.storage.config.StorageProperties
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.common.storage.core.cache.CacheStorageService
import com.tencent.bkrepo.common.storage.core.locator.FileLocator
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.repository.api.NodeClient
import com.tencent.bkrepo.repository.api.RepositoryClient
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.TestInstance
import org.springframework.boot.autoconfigure.ImportAutoConfiguration
Expand Down Expand Up @@ -73,7 +73,7 @@ open class ArtifactPreloadBaseServiceTest(
protected lateinit var repositoryClient: RepositoryClient

@MockBean
protected lateinit var storageCredentialsClient: StorageCredentialsClient
protected lateinit var storageCredentialService: StorageCredentialService

protected fun createTempArtifactFile(size: Long): ArtifactFile {
val tempFile = File.createTempFile("preload-", ".tmp")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ import com.tencent.bkrepo.common.artifact.pojo.RepositoryCategory
import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
import com.tencent.bkrepo.common.artifact.pojo.configuration.local.LocalConfiguration
import com.tencent.bkrepo.common.mongo.dao.util.Pages
import com.tencent.bkrepo.common.storage.core.StorageProperties
import com.tencent.bkrepo.common.storage.config.StorageProperties
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.common.storage.core.locator.FileLocator
import com.tencent.bkrepo.common.storage.util.existReal
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,13 @@
package com.tencent.bkrepo.common.artifact.cache.service.impl

import com.tencent.bkrepo.common.api.constant.StringPool
import com.tencent.bkrepo.common.api.pojo.Response
import com.tencent.bkrepo.common.artifact.api.ArtifactFile
import com.tencent.bkrepo.common.artifact.cache.UT_PROJECT_ID
import com.tencent.bkrepo.common.artifact.cache.UT_REPO_NAME
import com.tencent.bkrepo.common.artifact.cache.UT_SHA256
import com.tencent.bkrepo.common.artifact.cache.config.ArtifactPreloadProperties
import com.tencent.bkrepo.common.artifact.cache.pojo.ArtifactPreloadPlan
import com.tencent.bkrepo.common.storage.core.StorageProperties
import com.tencent.bkrepo.common.storage.config.StorageProperties
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.common.storage.core.cache.CacheStorageService
import com.tencent.bkrepo.common.storage.core.locator.FileLocator
Expand Down Expand Up @@ -172,8 +171,8 @@ class DefaultPreloadPlanExecutorTest @Autowired constructor(

private fun resetMock() {
val defaultCredentials = storageProperties.defaultStorageCredentials() as FileSystemCredentials
whenever(storageCredentialsClient.findByKey(anyString())).thenReturn(
Response(0, null, defaultCredentials.copy(key = "test"))
whenever(storageCredentialService.findByKey(anyString())).thenReturn(
defaultCredentials.copy(key = "test")
)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@ import com.tencent.bkrepo.common.api.pojo.Response
import com.tencent.bkrepo.common.artifact.cns.CnsProperties
import com.tencent.bkrepo.common.artifact.cns.CnsService
import com.tencent.bkrepo.common.artifact.pojo.RepositoryType
import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
import com.tencent.bkrepo.common.security.constant.MS_AUTH_HEADER_SECURITY_TOKEN
import com.tencent.bkrepo.common.security.service.ServiceAuthManager
import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace
import com.tencent.bkrepo.common.service.util.HttpContextHolder
import com.tencent.bkrepo.common.storage.core.StorageProperties
import com.tencent.bkrepo.common.storage.config.StorageProperties
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
import com.tencent.bkrepo.repository.constant.SYSTEM_USER
import org.slf4j.LoggerFactory
import org.springframework.beans.factory.annotation.Value
Expand All @@ -62,7 +62,7 @@ import java.util.concurrent.TimeUnit
class CnsServiceImpl(
private val discoveryClient: DiscoveryClient,
private val storageService: StorageService,
private val storageCredentialsClient: StorageCredentialsClient,
private val storageCredentialService: StorageCredentialService,
private val storageProperties: StorageProperties,
private val cnsProperties: CnsProperties,
private val serviceAuthManager: ServiceAuthManager
Expand All @@ -80,7 +80,7 @@ class CnsServiceImpl(
private val restTemplate = RestTemplate()

override fun exist(key: String?, sha256: String): Boolean {
val storageCredentials = storageCredentialsClient.findByKey(key).data
val storageCredentials = storageCredentialService.findByKey(key)
?: storageProperties.defaultStorageCredentials()
return storageService.exist(sha256, storageCredentials)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,20 +37,20 @@ import com.tencent.bkrepo.common.artifact.manager.resource.NodeResource
import com.tencent.bkrepo.common.artifact.manager.resource.RemoteNodeResource
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.metadata.service.blocknode.BlockNodeService
import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
import com.tencent.bkrepo.common.service.cluster.ClusterInfo
import com.tencent.bkrepo.common.service.cluster.properties.ClusterProperties
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.fs.server.constant.FS_ATTR_KEY
import com.tencent.bkrepo.replication.api.ClusterNodeClient
import com.tencent.bkrepo.replication.exception.ReplicationMessageCode
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
import com.tencent.bkrepo.repository.pojo.node.NodeInfo

class NodeResourceFactoryImpl(
private val clusterProperties: ClusterProperties,
private val storageService: StorageService,
private val storageCredentialsClient: StorageCredentialsClient,
private val storageCredentialService: StorageCredentialService,
private val blockNodeService: BlockNodeService,
private val clusterNodeClient: ClusterNodeClient,
private val archiveClient: ArchiveClient,
Expand Down Expand Up @@ -88,7 +88,7 @@ class NodeResourceFactoryImpl(
range,
storageCredentials,
storageService,
storageCredentialsClient,
storageCredentialService,
archiveClient,
)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ import com.tencent.bkrepo.common.api.message.CommonMessageCode
import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHolder
import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream
import com.tencent.bkrepo.common.artifact.stream.Range
import com.tencent.bkrepo.common.metadata.service.repo.StorageCredentialService
import com.tencent.bkrepo.common.security.util.SecurityUtils
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.repository.api.StorageCredentialsClient
import com.tencent.bkrepo.repository.pojo.node.NodeInfo
import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail
import org.slf4j.LoggerFactory
Expand All @@ -51,7 +51,7 @@ class LocalNodeResource(
private val range: Range,
private val storageCredentials: StorageCredentials?,
private val storageService: StorageService,
private val storageCredentialsClient: StorageCredentialsClient,
private val storageCredentialService: StorageCredentialService,
private val archiveClient: ArchiveClient,
) : AbstractNodeResource() {

Expand Down Expand Up @@ -98,7 +98,7 @@ class LocalNodeResource(
node.copyFromCredentialsKey?.let {
val digest = node.sha256!!
logger.info("load data [$digest] from copy credentialsKey [$it]")
val fromCredentialsKey = storageCredentialsClient.findByKey(it).data
val fromCredentialsKey = storageCredentialService.findByKey(it)
return storageService.load(digest, range, fromCredentialsKey)
}
return null
Expand Down Expand Up @@ -150,7 +150,7 @@ class LocalNodeResource(
* */
private fun findStorageCredentialsByKey(credentialsKey: String?): StorageCredentials? {
credentialsKey ?: return null
return storageCredentialsClient.findByKey(credentialsKey).data
return storageCredentialService.findByKey(credentialsKey)
}

private fun restore(node: NodeInfo, storageCredentials: StorageCredentials?) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold
import com.tencent.bkrepo.common.artifact.resolve.response.ArtifactResource
import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream.Companion.METADATA_KEY_CACHE_ENABLED
import com.tencent.bkrepo.common.artifact.stream.FileArtifactInputStream
import com.tencent.bkrepo.common.storage.core.StorageProperties
import com.tencent.bkrepo.common.storage.config.StorageProperties
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail
import io.micrometer.core.instrument.Counter
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ package com.tencent.bkrepo.common.artifact.metrics
import com.tencent.bkrepo.common.artifact.metrics.export.ArtifactMetricsExporter
import com.tencent.bkrepo.common.metrics.push.custom.CustomMetricsExporter
import com.tencent.bkrepo.common.service.actuator.CommonTagProvider
import com.tencent.bkrepo.common.storage.core.StorageProperties
import com.tencent.bkrepo.common.storage.config.StorageProperties
import org.springframework.beans.factory.ObjectProvider
import org.springframework.boot.actuate.autoconfigure.metrics.export.influx.InfluxProperties
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import com.tencent.bkrepo.common.artifact.repository.context.ArtifactContextHold
import com.tencent.bkrepo.common.artifact.resolve.file.ArtifactDataReceiver
import com.tencent.bkrepo.common.artifact.stream.ArtifactInputStream
import com.tencent.bkrepo.common.artifact.stream.FileArtifactInputStream
import com.tencent.bkrepo.common.storage.core.StorageProperties
import com.tencent.bkrepo.common.storage.config.StorageProperties
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail
import io.micrometer.core.instrument.Tag
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ import com.tencent.bkrepo.common.artifact.repository.core.ArtifactRepository
import com.tencent.bkrepo.common.artifact.repository.proxy.ProxyRepository
import com.tencent.bkrepo.common.security.http.core.HttpAuthSecurity
import com.tencent.bkrepo.common.service.util.HttpContextHolder
import com.tencent.bkrepo.common.storage.core.config.RateLimitProperties
import com.tencent.bkrepo.common.storage.config.RateLimitProperties
import com.tencent.bkrepo.repository.pojo.node.NodeDetail
import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail
import org.springframework.beans.factory.ObjectProvider
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ import com.tencent.bkrepo.common.artifact.util.http.HttpRangeUtils
import com.tencent.bkrepo.common.security.manager.PermissionManager
import com.tencent.bkrepo.common.security.util.SecurityUtils
import com.tencent.bkrepo.common.service.util.HttpContextHolder
import com.tencent.bkrepo.common.storage.core.StorageProperties
import com.tencent.bkrepo.common.storage.config.StorageProperties
import com.tencent.bkrepo.common.storage.core.StorageService
import com.tencent.bkrepo.common.storage.credentials.InnerCosCredentials
import com.tencent.bkrepo.common.storage.credentials.StorageCredentials
Expand Down
Loading

0 comments on commit 785f444

Please sign in to comment.