Skip to content

Commit

Permalink
feat: 一次性任务支持分发仓库;支持自动创建远程项目仓库 TencentBlueKing#1105 (TencentBlueKing#…
Browse files Browse the repository at this point in the history
…1107)

* feat: 代码结构调整 TencentBlueKing#1105

* feat: 增加配置,当远程项目和仓库不存在时是否自动创建;一次性任务支持分发仓库 TencentBlueKing#1105

* feat: 解决代码冲突TencentBlueKing#1105

* feat: 代码优化TencentBlueKing#1105

* feat: 代码优化TencentBlueKing#1105

* feat: 代码优化TencentBlueKing#1105

* feat: 去掉无效参数TencentBlueKing#1105
  • Loading branch information
zacYL committed Sep 4, 2023
1 parent a07004c commit ce54206
Show file tree
Hide file tree
Showing 73 changed files with 575 additions and 368 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

package com.tencent.bkrepo.replication.pojo.remote.request

import com.tencent.bkrepo.replication.pojo.request.ReplicaObjectType
import com.tencent.bkrepo.replication.pojo.request.ReplicaType
import com.tencent.bkrepo.replication.pojo.task.objects.PackageConstraint
import com.tencent.bkrepo.replication.pojo.task.objects.PathConstraint
Expand Down Expand Up @@ -61,6 +62,8 @@ data class RemoteConfigCreateRequest(
val remoteProjectId: String? = null,
@ApiModelProperty("远程仓库")
val remoteRepoName: String? = null,
@ApiModelProperty("同步对象类型, 只支持clusterId存在的情况", required = true)
val replicaObjectType: ReplicaObjectType? = null,
@ApiModelProperty("包限制条件", required = false)
val packageConstraints: List<PackageConstraint>? = null,
@ApiModelProperty("路径限制条件", required = false)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

package com.tencent.bkrepo.replication.pojo.remote.request

import com.tencent.bkrepo.replication.pojo.request.ReplicaObjectType
import com.tencent.bkrepo.replication.pojo.request.ReplicaType
import com.tencent.bkrepo.replication.pojo.task.objects.PackageConstraint
import com.tencent.bkrepo.replication.pojo.task.objects.PathConstraint
Expand Down Expand Up @@ -59,6 +60,8 @@ data class RemoteConfigUpdateRequest(
val remoteProjectId: String? = null,
@ApiModelProperty("远程仓库")
val remoteRepoName: String? = null,
@ApiModelProperty("同步对象类型, 只支持clusterId存在的情况", required = true)
val replicaObjectType: ReplicaObjectType? = null,
@ApiModelProperty("包限制条件", required = false)
val packageConstraints: List<PackageConstraint>? = null,
@ApiModelProperty("路径限制条件", required = false)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

package com.tencent.bkrepo.replication.pojo.remote.request

import com.tencent.bkrepo.replication.pojo.request.ReplicaObjectType
import com.tencent.bkrepo.replication.pojo.request.ReplicaType
import com.tencent.bkrepo.replication.pojo.task.setting.ReplicaSetting
import io.swagger.annotations.ApiModel
Expand Down Expand Up @@ -57,6 +58,8 @@ data class RemoteRunOnceTaskCreateRequest(
val remoteProjectId: String? = null,
@ApiModelProperty("远程仓库")
val remoteRepoName: String? = null,
@ApiModelProperty("同步对象类型, 只支持clusterId存在的情况", required = true)
val replicaObjectType: ReplicaObjectType? = null,
@ApiModelProperty("包名", required = false)
val packageName: String? = null,
@ApiModelProperty("包版本", required = false)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,5 +58,9 @@ data class ReplicaSetting(
/**
* 是否校验文件存储一致性
*/
val storageConsistencyCheck: Boolean = false
val storageConsistencyCheck: Boolean = false,
/**
* 当远程项目或者仓库不存在时是否自动创建
*/
val automaticCreateRemoteRepo: Boolean = true
)
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import com.tencent.bkrepo.replication.pojo.request.ReplicaType
import com.tencent.bkrepo.replication.pojo.task.EdgeReplicaTaskRecord
import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskInfo
import com.tencent.bkrepo.replication.pojo.task.objects.ReplicaObjectInfo
import com.tencent.bkrepo.replication.replica.base.replicator.commitedge.EdgeReplicaContextHolder
import com.tencent.bkrepo.replication.replica.replicator.commitedge.EdgeReplicaContextHolder
import com.tencent.bkrepo.replication.service.EdgeReplicaTaskRecordService
import com.tencent.bkrepo.replication.service.ReplicaTaskService
import org.springframework.web.bind.annotation.RestController
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import com.tencent.bkrepo.common.security.util.SecurityUtils
import com.tencent.bkrepo.common.service.util.ResponseBuilder
import com.tencent.bkrepo.replication.api.ArtifactPushClient
import com.tencent.bkrepo.replication.pojo.remote.request.ArtifactPushRequest
import com.tencent.bkrepo.replication.replica.event.EventBasedReplicaJobExecutor
import com.tencent.bkrepo.replication.replica.type.event.EventBasedReplicaJobExecutor
import com.tencent.bkrepo.replication.service.ReplicaTaskService
import org.slf4j.LoggerFactory
import org.springframework.web.bind.annotation.RestController
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.impl.remote.exception
package com.tencent.bkrepo.replication.exception

/**
* 文件内容读取报错
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.impl.remote.exception
package com.tencent.bkrepo.replication.exception
/**
* 推送到远端集群异常
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.impl.remote.exception
package com.tencent.bkrepo.replication.exception

/**
* Artifact来源异常
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.impl.remote.exception
package com.tencent.bkrepo.replication.exception

/**
* 规则校验异常
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,12 @@ import com.tencent.bkrepo.replication.constant.RUN_ONCE_EXECUTOR_ACTIVE_COUNT
import com.tencent.bkrepo.replication.constant.RUN_ONCE_EXECUTOR_ACTIVE_COUNT_DESC
import com.tencent.bkrepo.replication.constant.RUN_ONCE_EXECUTOR_QUEUE_SIZE
import com.tencent.bkrepo.replication.constant.RUN_ONCE_EXECUTOR_QUEUE_SIZE_DESC
import com.tencent.bkrepo.replication.replica.base.executor.EdgePullThreadPoolExecutor
import com.tencent.bkrepo.replication.replica.base.executor.EventConsumerThreadPoolExecutor
import com.tencent.bkrepo.replication.replica.base.executor.ManualThreadPoolExecutor
import com.tencent.bkrepo.replication.replica.base.executor.OciThreadPoolExecutor
import com.tencent.bkrepo.replication.replica.base.executor.ReplicaThreadPoolExecutor
import com.tencent.bkrepo.replication.replica.base.executor.RunOnceThreadPoolExecutor
import com.tencent.bkrepo.replication.replica.executor.EdgePullThreadPoolExecutor
import com.tencent.bkrepo.replication.replica.executor.EventConsumerThreadPoolExecutor
import com.tencent.bkrepo.replication.replica.executor.ManualThreadPoolExecutor
import com.tencent.bkrepo.replication.replica.executor.OciThreadPoolExecutor
import com.tencent.bkrepo.replication.replica.executor.ReplicaThreadPoolExecutor
import com.tencent.bkrepo.replication.replica.executor.RunOnceThreadPoolExecutor
import io.micrometer.core.instrument.Gauge
import io.micrometer.core.instrument.MeterRegistry
import io.micrometer.core.instrument.binder.MeterBinder
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
* Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
* Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.context
package com.tencent.bkrepo.replication.replica.context

import com.tencent.bkrepo.common.api.pojo.Response
import okhttp3.OkHttpClient
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
* Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
Expand All @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.context
package com.tencent.bkrepo.replication.replica.context

import com.tencent.bkrepo.common.api.pojo.ClusterArchitecture
import com.tencent.bkrepo.common.api.pojo.ClusterNodeType
Expand All @@ -44,15 +44,15 @@ import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus
import com.tencent.bkrepo.replication.pojo.record.ReplicaRecordInfo
import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskDetail
import com.tencent.bkrepo.replication.pojo.task.objects.ReplicaObjectInfo
import com.tencent.bkrepo.replication.replica.base.OkHttpClientPool
import com.tencent.bkrepo.replication.replica.base.interceptor.RetryInterceptor
import com.tencent.bkrepo.replication.replica.base.interceptor.SignInterceptor
import com.tencent.bkrepo.replication.replica.base.replicator.ClusterReplicator
import com.tencent.bkrepo.replication.replica.base.replicator.EdgeNodeReplicator
import com.tencent.bkrepo.replication.replica.base.replicator.RemoteReplicator
import com.tencent.bkrepo.replication.replica.base.replicator.Replicator
import com.tencent.bkrepo.replication.replica.base.replicator.commitedge.CenterClusterReplicator
import com.tencent.bkrepo.replication.replica.base.replicator.commitedge.CenterRemoteReplicator
import com.tencent.bkrepo.replication.replica.replicator.Replicator
import com.tencent.bkrepo.replication.replica.replicator.commitedge.CenterClusterReplicator
import com.tencent.bkrepo.replication.replica.replicator.commitedge.CenterRemoteReplicator
import com.tencent.bkrepo.replication.replica.replicator.standalone.ClusterReplicator
import com.tencent.bkrepo.replication.replica.replicator.standalone.EdgeNodeReplicator
import com.tencent.bkrepo.replication.replica.replicator.standalone.RemoteReplicator
import com.tencent.bkrepo.replication.util.OkHttpClientPool
import com.tencent.bkrepo.repository.pojo.repo.RepositoryDetail
import okhttp3.OkHttpClient
import org.slf4j.LoggerFactory
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.context
package com.tencent.bkrepo.replication.replica.context

import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus
import com.tencent.bkrepo.replication.pojo.record.ReplicaProgress
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
* Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
Expand All @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.executor
package com.tencent.bkrepo.replication.replica.executor

import com.tencent.bkrepo.common.artifact.event.base.ArtifactEvent
import com.tencent.bkrepo.common.service.otel.util.AsyncUtils.trace
Expand All @@ -36,8 +36,8 @@ import com.tencent.bkrepo.replication.pojo.record.ExecutionResult
import com.tencent.bkrepo.replication.pojo.record.ExecutionStatus
import com.tencent.bkrepo.replication.pojo.record.ReplicaRecordInfo
import com.tencent.bkrepo.replication.pojo.task.ReplicaTaskDetail
import com.tencent.bkrepo.replication.replica.base.ReplicaService
import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext
import com.tencent.bkrepo.replication.replica.type.ReplicaService
import com.tencent.bkrepo.replication.replica.context.ReplicaContext
import com.tencent.bkrepo.replication.service.ClusterNodeService
import org.slf4j.LoggerFactory
import java.util.concurrent.Callable
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.executor
package com.tencent.bkrepo.replication.replica.executor

import com.google.common.util.concurrent.ThreadFactoryBuilder
import java.util.concurrent.SynchronousQueue
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.executor
package com.tencent.bkrepo.replication.replica.executor

import com.google.common.util.concurrent.ThreadFactoryBuilder
import java.util.concurrent.ArrayBlockingQueue
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.executor
package com.tencent.bkrepo.replication.replica.executor

import com.google.common.util.concurrent.ThreadFactoryBuilder
import java.util.concurrent.ArrayBlockingQueue
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.executor
package com.tencent.bkrepo.replication.replica.executor

import com.google.common.util.concurrent.ThreadFactoryBuilder
import java.util.concurrent.ArrayBlockingQueue
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
* Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
Expand All @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.executor
package com.tencent.bkrepo.replication.replica.executor

import com.google.common.util.concurrent.ThreadFactoryBuilder
import java.util.concurrent.ArrayBlockingQueue
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.executor
package com.tencent.bkrepo.replication.replica.executor

import com.google.common.util.concurrent.ThreadFactoryBuilder
import java.util.concurrent.ArrayBlockingQueue
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/*
* Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
*
* Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
* Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
*
* BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
*
Expand All @@ -25,9 +25,9 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.replicator
package com.tencent.bkrepo.replication.replica.replicator

import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext
import com.tencent.bkrepo.replication.replica.context.ReplicaContext
import com.tencent.bkrepo.repository.pojo.node.NodeInfo
import com.tencent.bkrepo.repository.pojo.packages.PackageSummary
import com.tencent.bkrepo.repository.pojo.packages.PackageVersion
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.handler
package com.tencent.bkrepo.replication.replica.replicator.base

import com.tencent.bkrepo.common.api.constant.HttpHeaders
import com.tencent.bkrepo.common.api.constant.MediaTypes
Expand All @@ -43,8 +43,9 @@ import com.tencent.bkrepo.replication.pojo.blob.RequestTag
import com.tencent.bkrepo.replication.pojo.remote.DefaultHandlerResult
import com.tencent.bkrepo.replication.pojo.remote.RequestProperty
import com.tencent.bkrepo.replication.pojo.request.ReplicaType
import com.tencent.bkrepo.replication.replica.base.context.FilePushContext
import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext
import com.tencent.bkrepo.replication.replica.context.FilePushContext
import com.tencent.bkrepo.replication.replica.context.ReplicaContext
import com.tencent.bkrepo.replication.util.DefaultHandler
import com.tencent.bkrepo.replication.util.StreamRequestBody
import okhttp3.Headers
import okhttp3.MediaType.Companion.toMediaTypeOrNull
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.handler
package com.tencent.bkrepo.replication.replica.replicator.base.internal

import com.tencent.bkrepo.common.api.constant.HttpStatus
import com.tencent.bkrepo.common.api.constant.StringPool
Expand All @@ -40,12 +40,13 @@ import com.tencent.bkrepo.replication.constant.SHA256
import com.tencent.bkrepo.replication.constant.SIZE
import com.tencent.bkrepo.replication.constant.STORAGE_KEY
import com.tencent.bkrepo.replication.enums.WayOfPushArtifact
import com.tencent.bkrepo.replication.exception.ArtifactPushException
import com.tencent.bkrepo.replication.fdtp.FdtpAFTClientFactory
import com.tencent.bkrepo.replication.fdtp.FdtpServerProperties
import com.tencent.bkrepo.replication.manager.LocalDataManager
import com.tencent.bkrepo.replication.pojo.blob.RequestTag
import com.tencent.bkrepo.replication.replica.base.context.FilePushContext
import com.tencent.bkrepo.replication.replica.base.impl.remote.exception.ArtifactPushException
import com.tencent.bkrepo.replication.replica.context.FilePushContext
import com.tencent.bkrepo.replication.replica.replicator.base.ArtifactReplicationHandler
import com.tencent.bkrepo.replication.replica.base.process.ProgressListener
import com.tencent.bkrepo.replication.util.StreamRequestBody
import io.netty.channel.ChannelProgressiveFuture
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,17 @@
* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package com.tencent.bkrepo.replication.replica.base.handler
package com.tencent.bkrepo.replication.replica.replicator.base.remote

import com.tencent.bkrepo.common.api.constant.HttpStatus
import com.tencent.bkrepo.common.artifact.util.http.UrlFormatter
import com.tencent.bkrepo.replication.config.ReplicationProperties
import com.tencent.bkrepo.replication.constant.OCI_BLOBS_UPLOAD_FIRST_STEP_URL
import com.tencent.bkrepo.replication.manager.LocalDataManager
import com.tencent.bkrepo.replication.pojo.remote.DefaultHandlerResult
import com.tencent.bkrepo.replication.replica.base.context.FilePushContext
import com.tencent.bkrepo.replication.replica.base.context.ReplicaContext
import com.tencent.bkrepo.replication.replica.context.FilePushContext
import com.tencent.bkrepo.replication.replica.context.ReplicaContext
import com.tencent.bkrepo.replication.replica.replicator.base.ArtifactReplicationHandler
import org.slf4j.LoggerFactory
import org.springframework.stereotype.Component
import java.net.URL
Expand Down
Loading

0 comments on commit ce54206

Please sign in to comment.