Merge pull request TencentBlueKing#10903 from yongyiduan/issue_10254_1

feat：第三方构建机 Job 间复用构建环境支持 Code 配置 TencentBlueKing#10254
royalhuang · Sep 13, 2024 · 6a7594d · 6a7594d
2 parents 7dab682 + b2358df
commit 6a7594d
Show file tree

Hide file tree

Showing 31 changed files with 354 additions and 88 deletions.
diff --git a/...main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionProjectService.kt b/...main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionProjectService.kt
@@ -211,7 +211,7 @@ class RbacPermissionProjectService(
             groupCode = BkAuthGroup.MANAGER.value
         )!!.relationId.toInt()
 
-        val remotedevManager = projectInfo.properties?.remotedevManager?.split(",")
+        val remotedevManager = projectInfo.properties?.remotedevManager?.split(";")
         val members = projectGroupAndUserList.flatMap { it.userIdList }.distinct()
 
         val owners = projectGroupAndUserList

diff --git a/...ommon-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelStage.kt b/...ommon-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelStage.kt
@@ -215,6 +215,7 @@ class ModelStage @Autowired(required = false) constructor(
                     reviewers = ModelCommon.parseReceivers(it.reviewers).toList()
                 )
             }.toMutableList()
+            check.notifyGroup = stageCheck.reviews.notifyGroups?.toMutableList()
         }
         if (stageCheck.gates?.isNotEmpty() == true) {
             check.ruleIds = createRules(

diff --git a/...peline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ContainerTransfer.kt b/...peline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ContainerTransfer.kt
@@ -177,6 +177,7 @@ class ContainerTransfer @Autowired(required = false) constructor(
         steps: List<PreStep>?
     ): PreJob {
         return PreJob(
+            enable = job.containerEnabled().nullIfDefault(true),
             name = job.name,
             runsOn = RunsOn(
                 selfHosted = null,
@@ -220,6 +221,7 @@ class ContainerTransfer @Autowired(required = false) constructor(
         steps: List<PreStep>?
     ): PreJob {
         return PreJob(
+            enable = job.containerEnabled().nullIfDefault(true),
             name = job.name,
             runsOn = dispatchTransfer.makeRunsOn(job)?.fix(
                 jobId = job.jobId.toString(),
@@ -285,6 +287,11 @@ class ContainerTransfer @Autowired(required = false) constructor(
                     arrayOf(I18nUtil.getCodeLanMessage(TRANSFER_ERROR_CHECK_ENV_ID_FAILED) + ": $poolName")
                 )
             }
+
+            JobRunsOnPoolType.AGENT_REUSE_JOB.name -> {
+                nodeName = null
+                poolName = null
+            }
         }
 
         /*修正docker配额数据*/
@@ -368,6 +375,7 @@ class ContainerTransfer @Autowired(required = false) constructor(
 
         return if (finalStage) {
             JobControlOption(
+                enable = jobEnable,
                 timeout = timeout,
                 timeoutVar = timeoutVar,
                 runCondition = when (job.ifField) {
@@ -379,7 +387,9 @@ class ContainerTransfer @Autowired(required = false) constructor(
                 dependOnType = DependOnType.NAME,
                 dependOnName = dependOnName,
                 prepareTimeout = job.runsOn.queueTimeoutMinutes ?: VariableDefault.DEFAULT_JOB_PREPARE_TIMEOUT,
-                continueWhenFailed = job.continueOnError ?: DEFAULT_CONTINUE_WHEN_FAILED
+                continueWhenFailed = job.continueOnError ?: DEFAULT_CONTINUE_WHEN_FAILED,
+                singleNodeConcurrency = job.runsOn.singleNodeConcurrency,
+                allNodeConcurrency = job.runsOn.allNodeConcurrency
             )
         } else {
             val runCondition = kotlin.run {
@@ -398,7 +408,9 @@ class ContainerTransfer @Autowired(required = false) constructor(
                 dependOnType = DependOnType.NAME,
                 dependOnName = dependOnName,
                 prepareTimeout = job.runsOn.queueTimeoutMinutes ?: VariableDefault.DEFAULT_JOB_PREPARE_TIMEOUT,
-                continueWhenFailed = job.continueOnError ?: DEFAULT_CONTINUE_WHEN_FAILED
+                continueWhenFailed = job.continueOnError ?: DEFAULT_CONTINUE_WHEN_FAILED,
+                singleNodeConcurrency = job.runsOn.singleNodeConcurrency,
+                allNodeConcurrency = job.runsOn.allNodeConcurrency
             )
         }
     }

diff --git a/...ipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/DispatchTransfer.kt b/...ipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/DispatchTransfer.kt
@@ -104,6 +104,10 @@ class DispatchTransfer @Autowired(required = false) constructor(
                 else -> null
             }
         }
+        if (dispatchType is ThirdPartyAgentEnvDispatchType) {
+            runsOn.singleNodeConcurrency = job.jobControlOption?.singleNodeConcurrency
+            runsOn.allNodeConcurrency = job.jobControlOption?.allNodeConcurrency
+        }
         runsOn.needs = job.buildEnv?.ifEmpty { null }
         runsOn.queueTimeoutMinutes = job.jobControlOption?.prepareTimeout?.nullIfDefault(DEFAULT_JOB_PREPARE_TIMEOUT)
         return runsOn
@@ -161,6 +165,7 @@ class DispatchTransfer @Autowired(required = false) constructor(
                         workspace = workspace,
                         agentName = nodeName,
                         dockerInfo = getDockerInfo(job, buildTemplateAcrossInfo),
+                        lockResourceWith = lockResourceWith,
                         envProjectId = envProjectId
                     )
                 }

diff --git a/...n-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt b/...n-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt
@@ -170,12 +170,17 @@ class ModelTransfer @Autowired constructor(
         // 添加finally
         val finallyJobs = yamlInput.yaml.formatFinallyStage()
         if (finallyJobs.isNotEmpty()) {
+            yamlInput.aspectWrapper.setYamlStage4Yaml(
+                aspectType = PipelineTransferAspectWrapper.AspectType.BEFORE
+            )
             stageList.add(
                 modelStage.yaml2FinallyStage(
                     stageIndex = stageIndex,
                     finallyJobs = finallyJobs,
                     yamlInput = yamlInput
-                )
+                ).also {
+                    yamlInput.aspectWrapper.setModelStage4Model(it, PipelineTransferAspectWrapper.AspectType.AFTER)
+                }
             )
         }
         checkExtends(yamlInput.yaml.templateFilter().extends, model)

diff --git a/...in/kotlin/com/tencent/devops/process/yaml/transfer/aspect/PipelineTransferAspectLoader.kt b/...in/kotlin/com/tencent/devops/process/yaml/transfer/aspect/PipelineTransferAspectLoader.kt
@@ -1,7 +1,12 @@
 package com.tencent.devops.process.yaml.transfer.aspect
 
+import com.tencent.devops.common.api.exception.ErrorCodeException
+import com.tencent.devops.common.pipeline.container.VMBuildContainer
 import com.tencent.devops.common.pipeline.pojo.transfer.Resources
 import com.tencent.devops.common.pipeline.pojo.transfer.ResourcesPools
+import com.tencent.devops.common.pipeline.type.agent.AgentType
+import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentDispatch
+import com.tencent.devops.process.constant.ProcessMessageCode
 import com.tencent.devops.process.yaml.v3.models.PreTemplateScriptBuildYamlV3Parser
 import com.tencent.devops.process.yaml.v3.models.job.RunsOn
 import java.util.LinkedList
@@ -26,9 +31,85 @@ object PipelineTransferAspectLoader {
         return instance
     }
 
+    /*
+    * feat：第三方构建机 Job 间复用构建环境支持 Code 配置 #10254
+    * 支持检查值的有效性
+    * */
+    fun checkLockResourceJob(
+        aspects: LinkedList<IPipelineTransferAspect> = LinkedList()
+    ) {
+        val jobsCheck = mutableListOf<String/*job_id*/>()
+        val jobsNotCheck = mutableMapOf<String/*job_id*/, String/*child_Job_id*/>()
+        aspects.add(
+            object : IPipelineTransferAspectJob {
+                override fun after(jp: PipelineTransferJoinPoint) {
+                    val job = jp.modelJob()
+                    if (job != null && job is VMBuildContainer && job.dispatchType is ThirdPartyAgentDispatch) {
+                        when ((job.dispatchType as ThirdPartyAgentDispatch).agentType) {
+                            AgentType.REUSE_JOB_ID -> {
+                                jobsNotCheck[job.jobId!!] = (job.dispatchType as ThirdPartyAgentDispatch).value
+                            }
+
+                            else -> jobsCheck.add(job.jobId!!)
+                        }
+                    }
+                }
+            })
+        aspects.add(
+            object : IPipelineTransferAspectStage {
+                override fun after(jp: PipelineTransferJoinPoint) {
+                    // 回环检测
+                    if (jobsNotCheck.isNotEmpty()) {
+                        hasCycle(jobsNotCheck, jobsCheck)
+                    }
+                }
+
+                private fun hasCycle(map: Map<String, String>, checked: List<String>): Boolean {
+                    val visited = mutableSetOf<String>()
+                    val stack = mutableSetOf<String>()
+
+                    fun dfs(jobId: String): Boolean {
+                        if (stack.contains(jobId)) {
+                            return true
+                        }
+                        if (visited.contains(jobId)) {
+                            return false
+                        }
+
+                        visited.add(jobId)
+                        stack.add(jobId)
+
+                        val childJobId = map[jobId]
+                        if (childJobId != null && dfs(childJobId)) {
+                            return true
+                        }
+                        if (childJobId == null && jobId !in checked) {
+                            return true
+                        }
+
+                        stack.remove(jobId)
+                        return false
+                    }
+
+                    map.forEach { (k, v) ->
+                        if (!visited.contains(k) && dfs(k)) {
+                            throw ErrorCodeException(
+                                errorCode = ProcessMessageCode.ERROR_AGENT_REUSE_MUTEX_DEP_NULL_NODE,
+                                params = arrayOf(k, v)
+                            )
+                        }
+                    }
+                    jobsCheck.addAll(map.keys)
+                    jobsNotCheck.clear()
+                    return false
+                }
+            }
+        )
+    }
+
     fun sharedEnvTransfer(
         aspects: LinkedList<IPipelineTransferAspect> = LinkedList()
-    ): LinkedList<IPipelineTransferAspect> {
+    ) {
         val pools = mutableListOf<ResourcesPools>()
         aspects.add(
             object : IPipelineTransferAspectJob {
@@ -78,7 +159,6 @@ object PipelineTransferAspectLoader {
                 }
             }
         )
-        return aspects
     }
 
     fun initByDefaultTriggerOn(

diff --git a/...pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/PreStageReviews.kt b/...pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/PreStageReviews.kt
@@ -28,12 +28,15 @@
 package com.tencent.devops.process.yaml.v2.check
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonProperty
 
 @JsonIgnoreProperties(ignoreUnknown = true)
 data class PreStageReviews(
     val flows: List<PreFlow>?,
     val variables: Map<String, ReviewVariable>?,
-    val description: String?
+    val description: String?,
+    @JsonProperty("chat-id")
+    val notifyGroups: List<String>?
 )
 
 @JsonIgnoreProperties(ignoreUnknown = true)

diff --git a/...on-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/StageReviews.kt b/...on-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/check/StageReviews.kt
@@ -28,12 +28,15 @@
 package com.tencent.devops.process.yaml.v2.check
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonProperty
 
 @JsonIgnoreProperties(ignoreUnknown = true)
 data class StageReviews(
     val flows: List<Flow>?,
     val variables: Map<String, ReviewVariable>?,
-    val description: String?
+    val description: String?,
+    @JsonProperty("chat-id")
+    val notifyGroups: List<String>?
 )
 
 @JsonIgnoreProperties(ignoreUnknown = true)

diff --git a/...-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt b/...-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/utils/ScriptYmlUtils.kt
@@ -517,7 +517,8 @@ object ScriptYmlUtils {
                         )
                     },
                     variables = preCheck.reviews.variables,
-                    description = preCheck.reviews.description
+                    description = preCheck.reviews.description,
+                    notifyGroups = preCheck.reviews.notifyGroups
                 )
             } else {
                 null

diff --git a/...in/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlV3Parser.kt b/...in/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlV3Parser.kt
@@ -96,6 +96,9 @@ data class PreTemplateScriptBuildYamlV3Parser(
     @JsonIgnore
     lateinit var preYaml: PreScriptBuildYamlV3Parser
 
+    private val formatStages = lazy { ScriptYmlUtils.formatStage(preYaml, transferData) }
+    private val formatFinallyStage = lazy { ScriptYmlUtils.preJobs2Jobs(preYaml.finally, transferData) }
+
     @JsonIgnore
     val transferData: YamlTransferData = YamlTransferData()
 
@@ -137,12 +140,12 @@ data class PreTemplateScriptBuildYamlV3Parser(
 
     override fun formatStages(): List<Stage> {
         checkInitialized()
-        return ScriptYmlUtils.formatStage(preYaml, transferData)
+        return formatStages.value
     }
 
     override fun formatFinallyStage(): List<Job> {
         checkInitialized()
-        return ScriptYmlUtils.preJobs2Jobs(preYaml.finally, transferData)
+        return formatFinallyStage.value
     }
 
     override fun formatResources(): Resources? {

diff --git a/...mon-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/image/Pool.kt b/...mon-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/image/Pool.kt
@@ -49,6 +49,7 @@ data class Pool(
     val workspace: String? = null,
     val buildType: BuildType? = BuildType.DEVCLOUD,
     val dockerInfo: ThirdPartyAgentDockerInfo? = null,
+    val lockResourceWith: String? = null,
     val image: PoolImage? = null
 )