diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt index 7bc6dd624e7..84dd2c98865 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt @@ -185,6 +185,7 @@ object CommonMessageCode { const val SVN_TOKEN_FAIL = "2100135" // SVN Token 不正确 const val SVN_TOKEN_EMPTY = "2100136" // SVN Token 为空, 请检查代码库的凭证类型 + const val ERROR_VARIABLE_NOT_FOUND = "2100137" // SVN Token 为空, 请检查代码库的凭证类型 const val BK_CONTAINER_TIMED_OUT = "bkContainerTimedOut" // 创建容器超时 const val BK_CREATION_FAILED_EXCEPTION_INFORMATION = "bkCreationFailedExceptionInformation" // 创建失败，异常信息 diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/exception/VariableNotFoundException.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/exception/VariableNotFoundException.kt new file mode 100644 index 00000000000..2cb2cd196a9 --- /dev/null +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/exception/VariableNotFoundException.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.api.exception + +import com.tencent.devops.common.api.constant.CommonMessageCode + +/** + * 变量不存在异常 + */ +class VariableNotFoundException( + val variableKey: String?, + errorCode: String = CommonMessageCode.ERROR_VARIABLE_NOT_FOUND +) : + ErrorCodeException( + errorCode = errorCode, + defaultMessage = "variable $variableKey not found", + params = variableKey?.let { arrayOf(it) } + ) diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/PipelineAsCodeSettings.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/PipelineAsCodeSettings.kt index 0ee3d4d9e45..a31760df568 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/PipelineAsCodeSettings.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/PipelineAsCodeSettings.kt @@ -32,5 +32,35 @@ import io.swagger.v3.oas.annotations.media.Schema @Schema(title = "设置-YAML流水线功能设置") data class PipelineAsCodeSettings( @get:Schema(title = "是否支持YAML流水线功能", required = true) - val enable: Boolean = false -) + val enable: Boolean = false, + @get:Schema(title = "项目级流水线语法风格", required = false) + var projectDialect: String? = null, + @get:Schema(title = "是否继承项目流水线语言风格", required = false) + val inheritedDialect: Boolean? = true, + @get:Schema(title = "流水线语言风格", required = false) + var pipelineDialect: String? = null +) { + companion object { + fun initDialect(inheritedDialect: Boolean?, pipelineDialect: String?): PipelineAsCodeSettings { + return PipelineAsCodeSettings( + inheritedDialect = inheritedDialect ?: true, + // 如果继承项目方言配置,置空pipelineDialect字段,防止数据库存储多余数据 + pipelineDialect = if (inheritedDialect == false) { + pipelineDialect + } else { + null + } + ) + } + } + + /** + * 入库时,重置方言字段值 + */ + fun resetDialect() { + projectDialect = null + if (inheritedDialect != false) { + pipelineDialect = null + } + } +} diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionException.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionException.kt index 37d42ccbd2c..48188a1aedb 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionException.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionException.kt @@ -82,12 +82,6 @@ class FunctionFormatException(override val message: String?) : ExpressionExcepti } } -class ContextNotFoundException(override val message: String?) : ExpressionException() { - companion object { - fun contextNameNotFound(arg0: String) = ContextNotFoundException( - "Expression context $arg0 not found." - ) - } -} +class ContextNotFoundException(override var message: String? = null) : ExpressionException() class ContextJsonFormatException(override val message: String?) : ExpressionException() diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionParser.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionParser.kt index 7e473d70363..dd71969721e 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionParser.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionParser.kt @@ -27,9 +27,11 @@ package com.tencent.devops.common.expression +import com.tencent.devops.common.expression.ExpressionParser.legalizeExpression import com.tencent.devops.common.expression.context.ContextValueNode import com.tencent.devops.common.expression.context.DictionaryContextData import com.tencent.devops.common.expression.context.PipelineContextData +import com.tencent.devops.common.expression.expression.EvaluationOptions import com.tencent.devops.common.expression.expression.ExpressionConstants import com.tencent.devops.common.expression.expression.IExpressionNode import com.tencent.devops.common.expression.expression.IFunctionInfo @@ -74,8 +76,9 @@ object ExpressionParser { nameValue: List, fetchValue: Boolean ): Any? { + val options = EvaluationOptions(false) val result = createTree(expression.legalizeExpression(), null, nameValue, null)!! - .evaluate(null, context, null, null) + .evaluate(null, context, options, null) if (!fetchValue) { return result } @@ -90,10 +93,32 @@ object ExpressionParser { val context = ExecutionContext(DictionaryContextData()) val nameValue = mutableListOf() fillContextByMap(contextMap, context, nameValue) + val options = EvaluationOptions(false) + try { + return doEvaluateByMap( + expression = expression, + context = context, + nameValue = nameValue, + options = options, + fetchValue = fetchValue + ) + } catch (e: Throwable) { + if (options.contextNotNull() && e is ContextNotFoundException) { + throw ContextNotFoundException("Expression context ${options.contextNotNull.errKey()} not found.") + } + throw e + } + } + private fun doEvaluateByMap( + expression: String, + context: ExecutionContext, + nameValue: MutableList, + options: EvaluationOptions, + fetchValue: Boolean + ): Any? { val result = createTree(expression.legalizeExpression(), null, nameValue, null)!! - .evaluate(null, context, null, null) - + .evaluate(null, context, options, null) if (!fetchValue) { return result } @@ -104,8 +129,9 @@ object ExpressionParser { /** * 将流水线变量转换为表达式上下文类型，存在如下情况 * 1、a = str, 直接使用 string 类型的上下文保存即可 - * 2、a.b.c = str, 将 a.b.c 升格为上下文中的嵌套 map 保存 既 a {b: {c: str}} - * 3、a.b.c = str 且 a.b = {"c": "str"}, 需要校验 a.b 所保存的 json 与 a.b.c 结构和数据是否相同后再升格 + * 2、a.b.c = str, 将 a.b.c 转换为上下文中的嵌套 map 保存 既 a {b: {c: str}} + * 3、a.b.c = str 且 a.b = {"c": "str"}, 将 a.b 保存为兼容用户的数据类型，在不涉及引擎计算纯输出的情况下使用 a.b 的原数据， + * 在涉及引擎计算时，则使用 a.b.c 转换后的 map，同 2 中所述 */ fun fillContextByMap( contextMap: Map, diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/AbsDictionaryContextData.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/AbsDictionaryContextData.kt index 0de7a0d8bf7..c88b3fd03ae 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/AbsDictionaryContextData.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/AbsDictionaryContextData.kt @@ -1,6 +1,7 @@ package com.tencent.devops.common.expression.context import com.fasterxml.jackson.databind.JsonNode +import com.tencent.devops.common.expression.expression.sdk.CollectionPipelineResult import com.tencent.devops.common.expression.expression.sdk.IReadOnlyObject import com.tencent.devops.common.expression.utils.ExpressionJsonUtil import java.util.TreeMap @@ -8,11 +9,7 @@ import java.util.TreeMap /** * dict 的抽象类，总结公共方法 */ -abstract class AbsDictionaryContextData : - PipelineContextData(PipelineContextDataType.DICTIONARY), - Iterable>, - IReadOnlyObject { - +abstract class AbsDictionaryContextData : PipelineContextData(PipelineContextDataType.DICTIONARY), IReadOnlyObject { protected open var mIndexLookup: TreeMap? = null protected open var mList: MutableList = mutableListOf() @@ -24,14 +21,6 @@ abstract class AbsDictionaryContextData : return emptyList() } - override fun tryGetValue(key: String): Pair { - if (mList.isNotEmpty() && indexLookup.containsKey(key)) { - return Pair(mList[indexLookup[key]!!].value, true) - } - - return Pair(null, false) - } - protected val indexLookup: MutableMap get() { if (mIndexLookup == null) { @@ -51,6 +40,29 @@ abstract class AbsDictionaryContextData : return mList } + override operator fun get(key: String): PipelineContextData? { + val index = indexLookup[key] ?: return null + return list[index].value + } + + override fun getRes(key: String): CollectionPipelineResult { + return if (containsKey(key)) { + CollectionPipelineResult(list.getOrNull(indexLookup[key]!!)?.value) + } else { + CollectionPipelineResult.noKey() + } + } + + override fun toJson(): JsonNode { + val json = ExpressionJsonUtil.createObjectNode() + if (mList.isNotEmpty()) { + mList.forEach { + json.set(it.key, it.value?.toJson()) + } + } + return json + } + operator fun set(k: String, value: PipelineContextData?) { // Existing val index = indexLookup[k] @@ -64,22 +76,6 @@ abstract class AbsDictionaryContextData : } } - open operator fun get(k: String): PipelineContextData? { - // Existing - val index = indexLookup[k] ?: return null - return list[index].value - } - - open operator fun IReadOnlyObject.get(key: String): Any? { - val index = indexLookup[key] ?: return null - return list[index].value - } - - operator fun Pair.get(key: Int): Pair { - val pair = mList[key] - return Pair(pair.key, pair.value) - } - fun add(pairs: Iterable>) { pairs.forEach { pair -> add(pair.first, pair.second) @@ -94,18 +90,8 @@ abstract class AbsDictionaryContextData : list.add(DictionaryContextDataPair(key, value)) } - override fun iterator(): Iterator> { - return mList.map { pair -> Pair(pair.key, pair.value); }.iterator() - } - - override fun toJson(): JsonNode { - val json = ExpressionJsonUtil.createObjectNode() - if (mList.isNotEmpty()) { - mList.forEach { - json.set(it.key, it.value?.toJson()) - } - } - return json + fun containsKey(key: String): Boolean { + return mList.isNotEmpty() && indexLookup.containsKey(key) } protected data class DictionaryContextDataPair( diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/ArrayContextData.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/ArrayContextData.kt index edd43646ef0..7dbb0e41b73 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/ArrayContextData.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/ArrayContextData.kt @@ -28,6 +28,7 @@ package com.tencent.devops.common.expression.context import com.fasterxml.jackson.databind.JsonNode +import com.tencent.devops.common.expression.expression.sdk.CollectionPipelineResult import com.tencent.devops.common.expression.expression.sdk.IReadOnlyArray import com.tencent.devops.common.expression.utils.ExpressionJsonUtil @@ -40,11 +41,14 @@ class ArrayContextData : PipelineContextData(PipelineContextDataType.ARRAY), IRe override val count: Int get() = mItems.count() - fun add(item: PipelineContextData?) { - mItems.add(item) - } + override operator fun get(index: Int): PipelineContextData? = mItems[index] - override fun get(index: Int): Any? = mItems[index] + override fun getRes(index: Int): CollectionPipelineResult { + if (index >= 0 && index <= mItems.lastIndex) { + return CollectionPipelineResult(mItems[index]) + } + return CollectionPipelineResult.noKey() + } override fun clone(): PipelineContextData { val result = ArrayContextData() @@ -71,9 +75,17 @@ class ArrayContextData : PipelineContextData(PipelineContextDataType.ARRAY), IRe val list = mutableListOf() if (mItems.isNotEmpty()) { mItems.forEach { + if (it is DictionaryContextDataWithVal) { + list.add(it.fetchValueNative()) + return@forEach + } list.add(it?.fetchValue() ?: return@forEach) } } return list } + + fun add(item: PipelineContextData?) { + mItems.add(item) + } } diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/ContextValueNode.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/ContextValueNode.kt index 0843ef0d73e..8fe80fde0c8 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/ContextValueNode.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/ContextValueNode.kt @@ -27,6 +27,7 @@ package com.tencent.devops.common.expression.context +import com.tencent.devops.common.expression.ContextNotFoundException import com.tencent.devops.common.expression.ExecutionContext import com.tencent.devops.common.expression.expression.sdk.EvaluationContext import com.tencent.devops.common.expression.expression.sdk.NamedValue @@ -34,7 +35,12 @@ import com.tencent.devops.common.expression.expression.sdk.ResultMemory class ContextValueNode : NamedValue() { override fun evaluateCore(context: EvaluationContext): Pair { - return Pair(null, (context.state as ExecutionContext).expressionValues[name]) + val value = (context.state as ExecutionContext).expressionValues.getRes(name) + if (context.options.contextNotNull() && value.noKey()) { + context.options.contextNotNull.trace(name) + throw ContextNotFoundException() + } + return Pair(null, value.value) } override fun createNode(): NamedValue { @@ -43,7 +49,7 @@ class ContextValueNode : NamedValue() { override fun subNameValueEvaluateCore(context: EvaluationContext): Pair { val values = (context.state as ExecutionContext).expressionValues - return if (values[name] != null) { + return if (values.containsKey(name)) { Pair(values[name], true) } else { Pair(name, false) diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextData.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextData.kt index ff13dfc165c..61716b4007f 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextData.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextData.kt @@ -46,10 +46,14 @@ open class DictionaryContextData : AbsDictionaryContextData() { return result } - override fun fetchValue(): Map { + override fun fetchValue(): Any { val map = mutableMapOf() if (mList.isNotEmpty()) { mList.forEach { + if (it.value is DictionaryContextDataWithVal) { + map[it.key] = it.value.fetchValueNative() + return@forEach + } map[it.key] = it.value?.fetchValue() ?: "" } } diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextDataWithVal.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextDataWithVal.kt index 15bc0e3ba58..b47c08e9645 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextDataWithVal.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextDataWithVal.kt @@ -9,7 +9,7 @@ import java.util.TreeMap */ class DictionaryContextDataWithVal( private val oriValue: String -) : AbsDictionaryContextData() { +) : DictionaryContextData() { override var mIndexLookup: TreeMap? = null override var mList: MutableList = mutableListOf() @@ -26,6 +26,9 @@ class DictionaryContextDataWithVal( return result } + /** + * 兼容用户数据，输出的fetchValue + */ override fun fetchValue(): Any { return try { ExpressionJsonUtil.getObjectMapper().readTree(oriValue) @@ -33,4 +36,9 @@ class DictionaryContextDataWithVal( return oriValue } } + + /** + * 引擎中计算使用的fetchValue + */ + fun fetchValueNative() = super.fetchValue() } \ No newline at end of file diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/RuntimeDictionaryContextData.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/RuntimeDictionaryContextData.kt index f8c6793dae5..3cabb836d5e 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/RuntimeDictionaryContextData.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/RuntimeDictionaryContextData.kt @@ -28,7 +28,7 @@ package com.tencent.devops.common.expression.context import com.tencent.devops.common.expression.ContextDataRuntimeException -import com.tencent.devops.common.expression.expression.sdk.IReadOnlyObject +import com.tencent.devops.common.expression.expression.sdk.CollectionPipelineResult import java.lang.Exception import java.util.TreeMap @@ -43,40 +43,24 @@ interface RuntimeNamedValue { * @throws ContextDataRuntimeException */ @Suppress("TooManyFunctions", "ReturnCount") -class RuntimeDictionaryContextData(private val runtimeNamedValue: RuntimeNamedValue) : - DictionaryContextData() { - +class RuntimeDictionaryContextData(private val runtimeNamedValue: RuntimeNamedValue) : DictionaryContextData() { override var mIndexLookup: TreeMap? = null override var mList: MutableList = mutableListOf() - private fun requestAndSaveValue(key: String): PipelineContextData? { - return try { - val value = runtimeNamedValue.getValue(key) - if (value != null) { - set(key, value) - } - value - } catch (ignore: Exception) { - throw ContextDataRuntimeException("RuntimeDictionaryContextData request key:$key 's value error") - } + override operator fun get(key: String): PipelineContextData? { + return getRes(key).value } - override fun tryGetValue(key: String): Pair { - if (mList.isNotEmpty() && indexLookup.containsKey(key)) { - return Pair(mList[indexLookup[key]!!].value, true) + override fun getRes(key: String): CollectionPipelineResult { + if (containsKey(key)) { + return CollectionPipelineResult(mList.getOrNull(indexLookup[key]!!)?.value) } - // 对象中没有则去请求一次 - val value = requestAndSaveValue(key) ?: return Pair(null, false) - - return Pair(value, true) - } - override operator fun get(k: String): PipelineContextData? { - return tryGetValue(k).first - } + // 对象中没有则去请求一次 + // 暂时全部按照无KEY处理，上线后看看情况 + val value = requestAndSaveValue(key) ?: return CollectionPipelineResult.noKey() - override operator fun IReadOnlyObject.get(key: String): Any? { - return tryGetValue(key).first + return CollectionPipelineResult(value) } override fun clone(): PipelineContextData { @@ -91,4 +75,16 @@ class RuntimeDictionaryContextData(private val runtimeNamedValue: RuntimeNamedVa return result } + + private fun requestAndSaveValue(key: String): PipelineContextData? { + return try { + val value = runtimeNamedValue.getValue(key) + if (value != null) { + set(key, value) + } + value + } catch (ignore: Exception) { + throw ContextDataRuntimeException("RuntimeDictionaryContextData request key:$key 's value error") + } + } } diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/EvaluationOptions.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/EvaluationOptions.kt index 77591db5e4d..00d787b7adc 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/EvaluationOptions.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/EvaluationOptions.kt @@ -27,13 +27,47 @@ package com.tencent.devops.common.expression.expression -class EvaluationOptions() { +/** + * @param contextNotNull 上下文计算时需要不存在的变量抛出异常而不是返回空 + * @param maxMemory 暂未使用 + */ +data class EvaluationOptions( + val contextNotNull: ExceptionInsteadOfNullOption, + var maxMemory: Int = 0 +) { + fun contextNotNull(): Boolean { + return contextNotNull.enable + } - constructor(copy: EvaluationOptions?) : this() { - if (copy != null) { - maxMemory = copy.maxMemory + constructor(contextNotNull: Boolean) : this( + if (contextNotNull) { + ExceptionInsteadOfNullOption.enable() + } else { + ExceptionInsteadOfNullOption.disabled() } + ) +} + +/** + * @param enable 是否开启 + * @param exceptionTraceMsg 存放为空的变量的索引链路，方便排查 + */ +data class ExceptionInsteadOfNullOption( + val enable: Boolean, + val exceptionTraceMsg: MutableList? +) { + fun trace(name: String) { + if (!enable) { + return + } + // 字符串的格式化会带 '' + exceptionTraceMsg?.add(name.removeSurrounding("'")) } - var maxMemory: Int = 0 -} + fun errKey() = exceptionTraceMsg?.joinToString(".") + + companion object { + fun disabled(): ExceptionInsteadOfNullOption = ExceptionInsteadOfNullOption(false, null) + fun enable(): ExceptionInsteadOfNullOption = ExceptionInsteadOfNullOption(true, mutableListOf()) + } +} \ No newline at end of file diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/IExpressionNode.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/IExpressionNode.kt index f7e118642f4..d1d82c5bdb4 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/IExpressionNode.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/IExpressionNode.kt @@ -34,7 +34,7 @@ interface IExpressionNode { fun evaluate( trace: ITraceWriter?, state: Any?, - options: EvaluationOptions?, + options: EvaluationOptions, expressionOutput: ExpressionOutput? ): EvaluationResult @@ -53,7 +53,7 @@ interface IExpressionNode { fun subNameValueEvaluate( trace: ITraceWriter?, state: Any?, - options: EvaluationOptions?, + options: EvaluationOptions, subInfo: SubNameValueEvaluateInfo, expressionOutput: ExpressionOutput? ): SubNameValueEvaluateResult diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/CollectionResult.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/CollectionResult.kt new file mode 100644 index 00000000000..8863609ba49 --- /dev/null +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/CollectionResult.kt @@ -0,0 +1,63 @@ +package com.tencent.devops.common.expression.expression.sdk + +import com.tencent.devops.common.expression.ContextNotFoundException +import com.tencent.devops.common.expression.context.PipelineContextData + +/** + * 从集合类型上下文取出值的封装对象 + * 如 IReadOnlyArray，IReadOnlyObject + */ +open class CollectionResult( + open val type: CollectionResultType, + open val value: Any? +) { + constructor(value: Any?) : this( + type = if (value == null) { + CollectionResultType.NO_VALUE + } else { + CollectionResultType.VALUE + }, + value = value + ) + + fun noKey() = type == CollectionResultType.NO_KEY + + fun throwIfNoKey() { + if (noKey()) { + throw ContextNotFoundException() + } + } + + companion object { + fun noKey() = CollectionPipelineResult(CollectionResultType.NO_KEY, null) + } +} + +enum class CollectionResultType { + // 集合中不存在取值的KEY + NO_KEY, + + // 集合中存在KEY但是取值为空 + NO_VALUE, + + // 存在KEY且取值不为空 + VALUE +} + +data class CollectionPipelineResult( + override val type: CollectionResultType, + override val value: PipelineContextData? +) : CollectionResult(type, value) { + constructor(value: PipelineContextData?) : this( + type = if (value == null) { + CollectionResultType.NO_VALUE + } else { + CollectionResultType.VALUE + }, + value = value + ) + + companion object { + fun noKey() = CollectionPipelineResult(CollectionResultType.NO_KEY, null) + } +} \ No newline at end of file diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/EvaluationContext.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/EvaluationContext.kt index 5921c4f3887..a7df27314e4 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/EvaluationContext.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/EvaluationContext.kt @@ -35,11 +35,10 @@ import com.tencent.devops.common.expression.expression.ITraceWriter class EvaluationContext( val trace: ITraceWriter?, val state: Any?, - ops: EvaluationOptions?, + val options: EvaluationOptions, val node: ExpressionNode?, val expressionOutput: ExpressionOutput? ) { - val options: EvaluationOptions = EvaluationOptions(ops) val memory: EvaluationMemory private val mTraceResults = mutableMapOf() private val mTraceMemory: MemoryCounter diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/ExpressionNode.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/ExpressionNode.kt index eaa54dbcc77..663ed5287b3 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/ExpressionNode.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/ExpressionNode.kt @@ -69,10 +69,14 @@ abstract class ExpressionNode : IExpressionNode { protected abstract val traceFullyRealized: Boolean + // 用来展示打印这个节点 + protected open val formatValue: String? = null + open fun format() = formatValue ?: name + override fun evaluate( trace: ITraceWriter?, state: Any?, - options: EvaluationOptions?, + options: EvaluationOptions, expressionOutput: ExpressionOutput? ): EvaluationResult { if (container != null) { @@ -140,7 +144,7 @@ abstract class ExpressionNode : IExpressionNode { override fun subNameValueEvaluate( trace: ITraceWriter?, state: Any?, - options: EvaluationOptions?, + options: EvaluationOptions, subInfo: SubNameValueEvaluateInfo, expressionOutput: ExpressionOutput? ): SubNameValueEvaluateResult { diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/IReadOnlyArray.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/IReadOnlyArray.kt index 4ec03b5960a..3e563188d7f 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/IReadOnlyArray.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/IReadOnlyArray.kt @@ -30,5 +30,14 @@ package com.tencent.devops.common.expression.expression.sdk interface IReadOnlyArray : Iterable { val count: Int + /** + * 使用kotlin原生List的get方法 + * 使用时需要注意场景，小心IndexOutOfBoundsException + */ operator fun get(index: Int): Any? + + /** + * 封装get，令返回结果更清晰 + */ + fun getRes(index: Int): CollectionResult } diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/IReadOnlyObject.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/IReadOnlyObject.kt index 625a61d1eb0..0ae24065792 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/IReadOnlyObject.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/IReadOnlyObject.kt @@ -30,5 +30,14 @@ package com.tencent.devops.common.expression.expression.sdk interface IReadOnlyObject { val values: Iterable - fun tryGetValue(key: String): Pair + /** + * 使用kotlin原生Map的get方法 + * 使用时需要注意场景 + */ + operator fun get(key: String): Any? + + /** + * 封装get，令返回结果更清晰 + */ + fun getRes(key: String): CollectionResult } diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/Literal.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/Literal.kt index 910250c3e2d..561b46d757f 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/Literal.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/Literal.kt @@ -44,6 +44,8 @@ class Literal(v: Any?) : ExpressionNode() { // 这样可以避免不必要地复制内存中的值。 override val traceFullyRealized = false + override fun format() = ExpressionUtility.formatValue(value, kind) + override fun convertToExpression() = ExpressionUtility.formatValue(value, kind) override fun convertToRealizedExpression(context: EvaluationContext) = ExpressionUtility.formatValue(value, kind) diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/operators/Index.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/operators/Index.kt index 3f1e1101c9b..d7b5094d244 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/operators/Index.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/sdk/operators/Index.kt @@ -27,9 +27,12 @@ package com.tencent.devops.common.expression.expression.sdk.operators +import com.tencent.devops.common.expression.ContextNotFoundException import com.tencent.devops.common.expression.ExecutionContext import com.tencent.devops.common.expression.context.ContextValueNode import com.tencent.devops.common.expression.expression.EvaluationResult +import com.tencent.devops.common.expression.expression.ExpressionConstants +import com.tencent.devops.common.expression.expression.sdk.CollectionResult import com.tencent.devops.common.expression.expression.sdk.Container import com.tencent.devops.common.expression.expression.sdk.EvaluationContext import com.tencent.devops.common.expression.expression.sdk.ExpressionNode @@ -47,9 +50,10 @@ class Index : Container() { override val traceFullyRealized = true + override val formatValue: String = ExpressionConstants.DEREFERENCE.toString() + override fun convertToExpression(): String { - // 验证我们是否可以简化表达式，我们宁愿返回 - // github.sha 然后 github['sha'] 所以我们检查这是否是一个简单的案例。 + // 验证我们是否可以简化表达式，我们宁愿返回 github.sha 然后 github['sha'] 所以我们检查这是否是一个简单的案例 return if (parameters[1] is Literal && (parameters[1] as Literal).value is String && ExpressionUtility.isLegalKeyword((parameters[1] as Literal).value as String) @@ -68,12 +72,15 @@ class Index : Container() { } return parameters[0].convertToRealizedExpression(context) + - "[${parameters[1].convertToRealizedExpression(context)}]" + "[${parameters[1].convertToRealizedExpression(context)}]" } override fun evaluateCore(context: EvaluationContext): Pair { val left = parameters[0].evaluate(context) - + // 如果有多个索引如 a.b.c 因为是递归计算，到 c的时候拿到的左参数一定是 a.b 中的 .，所以追踪左参数只追踪第一个 + if (parameters[0] !is Index) { + context.options.contextNotNull.trace(parameters[0].format()) + } // Not a collection val (collection, ok) = left.tryGetCollectionInterface() if (!ok) { @@ -136,6 +143,7 @@ class Index : Container() { return Pair(result, true) } + @Suppress("ComplexMethod") private fun handleFilteredArray( context: EvaluationContext, filteredArray: FilteredArray @@ -161,9 +169,13 @@ class Index : Container() { } // String else if (index.hasStringIndex) { - val (nestedObjectValue, res) = nestedCollection.tryGetValue(index.stringIndex!!) - if (res) { - result.add(nestedObjectValue) + val key = index.stringIndex!! + val nestedObjectValueRes = nestedCollection.getRes(key) + if (context.options.contextNotNull()) { + nestedObjectValueRes.throwIfNoKey() + } + if (!nestedObjectValueRes.noKey()) { + result.add(nestedObjectValueRes.value) counter.add(Int.SIZE_BYTES) } } @@ -209,9 +221,16 @@ class Index : Container() { } // String else { - val (result, ok) = obj.tryGetValue(index.stringIndex ?: return Pair(null, null)) - if (index.hasStringIndex && ok) { - return Pair(null, result) + val key = index.stringIndex + if (key == null && context.options.contextNotNull()) { + throw ContextNotFoundException() + } + val result = obj.getRes(key ?: return Pair(null, null)) + if (context.options.contextNotNull()) { + result.throwIfNoKey() + } + if (index.hasStringIndex && !result.noKey()) { + return Pair(null, result.value) } } @@ -241,6 +260,9 @@ class Index : Container() { else if (index.hasIntegerIndex && index.integerIndex < array.count) { return Pair(null, array[index.integerIndex]) } + if (context.options.contextNotNull()) { + throw ContextNotFoundException() + } return Pair(null, null) } @@ -260,19 +282,29 @@ class Index : Container() { return mList[index] } + override fun getRes(index: Int): CollectionResult { + if (index >= 0 && index <= mList.lastIndex) { + return CollectionResult(mList[index]) + } + return CollectionResult.noKey() + } + override fun iterator(): Iterator { return mList.iterator() } } private class IndexHelper(context: EvaluationContext, val parameter: ExpressionNode) { - private val mResult: EvaluationResult + val mResult: EvaluationResult private val mIntegerIndex: Lazy private val mStringIndex: Lazy init { mResult = parameter.evaluate(context) + // 追踪右参数 + context.options.contextNotNull.trace(parameter.format()) + mIntegerIndex = lazy { var doubleIndex = mResult.convertToNumber() if (doubleIndex.isNaN() || doubleIndex < 0.0) { diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/specialFuctions/hashFiles/HashFilesFunction.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/specialFuctions/hashFiles/HashFilesFunction.kt index 5dd4ed7b08c..6cd16664970 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/specialFuctions/hashFiles/HashFilesFunction.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/specialFuctions/hashFiles/HashFilesFunction.kt @@ -48,18 +48,12 @@ class HashFilesFunction : Function() { val contextValues = context.state as ExecutionContext var workspaceData = contextValues.expressionValues[ciWorkSpaceKey.split(".")[0]] workspaceData = if (workspaceData == null) { - if (contextValues.expressionValues[workSpaceKey] == null) { - throw ContextNotFoundException("$workSpaceKey/$ciWorkSpaceKey") - } else { - contextValues.expressionValues[workSpaceKey] - } + contextValues.expressionValues[workSpaceKey] + ?: throw ContextNotFoundException("$workSpaceKey/$ciWorkSpaceKey") } else { - if (workspaceData !is DictionaryContextData || workspaceData[ciWorkSpaceKey.split(".")[1]] == null) { - if (contextValues.expressionValues[workSpaceKey] == null) { - throw ContextNotFoundException("$workSpaceKey/$ciWorkSpaceKey") - } else { - contextValues.expressionValues[workSpaceKey] - } + if (workspaceData !is DictionaryContextData || !workspaceData.containsKey(ciWorkSpaceKey.split(".")[1])) { + contextValues.expressionValues[workSpaceKey] + ?: throw ContextNotFoundException("$workSpaceKey/$ciWorkSpaceKey") } else { workspaceData[ciWorkSpaceKey.split(".")[1]] } diff --git a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/context/RuntimeDictionaryContextDataTest.kt b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/context/RuntimeDictionaryContextDataTest.kt index 2d49d08d896..be44807b5c4 100644 --- a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/context/RuntimeDictionaryContextDataTest.kt +++ b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/context/RuntimeDictionaryContextDataTest.kt @@ -29,6 +29,7 @@ package com.tencent.devops.common.expression.context import com.tencent.devops.common.expression.ExecutionContext import com.tencent.devops.common.expression.ExpressionParser +import com.tencent.devops.common.expression.expression.EvaluationOptions import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeAll @@ -46,7 +47,7 @@ internal class RuntimeDictionaryContextDataTest { fun contextSingleBuildTest() { val context = RuntimeDictionaryContextData(RuntimeNamedValueImpl()) data.forEach { (k, v) -> - Assertions.assertEquals(v, context.tryGetValue(k).first) + Assertions.assertEquals(v, context[k]) } Assertions.assertEquals(data.map { it.value }, context.values) } @@ -59,12 +60,13 @@ internal class RuntimeDictionaryContextDataTest { "settings.access_token.access_token == '456' => true", "settings['appId'].secretKey == '101112' => true", "settings.token['username'] == " + - "fromJSON('{\"token\":\"212223\",\"username\":\"789\",\"password\":\"123\"}').username => true" + "fromJSON('{\"token\":\"212223\",\"username\":\"789\",\"password\":\"123\"}').username => true" ] ) fun contextExpressionParseTest(expAndExpect: String) { val (exp, expect) = expAndExpect.split(" => ") - val res = ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, null, null).value + val res = ExpressionParser.createTree(exp, null, nameValue, null)!! + .evaluate(null, ev, EvaluationOptions(false), null).value Assertions.assertEquals( if (expect == "true" || expect == "false") { expect.toBoolean() diff --git a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/EvaluationOptionsTest.kt b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/EvaluationOptionsTest.kt new file mode 100644 index 00000000000..069d07a4dbe --- /dev/null +++ b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/EvaluationOptionsTest.kt @@ -0,0 +1,102 @@ +package com.tencent.devops.common.expression.expression + +import com.tencent.devops.common.expression.ContextNotFoundException +import com.tencent.devops.common.expression.ExecutionContext +import com.tencent.devops.common.expression.ExpressionParser +import com.tencent.devops.common.expression.context.ArrayContextData +import com.tencent.devops.common.expression.context.ContextValueNode +import com.tencent.devops.common.expression.context.DictionaryContextData +import com.tencent.devops.common.expression.context.StringContextData +import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Nested +import org.junit.jupiter.api.assertThrows +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ValueSource +import org.slf4j.LoggerFactory + +@Suppress("ComplexMethod", "LongMethod", "MaxLineLength") +@DisplayName("测试EvaluationOptions配置的不同选项") +class EvaluationOptionsTest { + @DisplayName("exceptionInsteadOfNull相关场景测试") + @Nested + inner class ExceptionTest { + @DisplayName("配置了exceptionInsteadOfNull") + @ParameterizedTest + @ValueSource( + strings = [ + "string => string", + "obj.a.obj_obj1[2] == 1 => obj.a", + "arr[0].arr_obj_1_n2 => arr.0.arr_obj_1_n2", + "obj.obj_obj1.obj_obj1_n1.a => obj.obj_obj1.obj_obj1_n1.a" + ] + ) + fun exceptionInsteadOfNull(group: String) { + val (exp, exArg) = group.split(" => ") + val options = EvaluationOptions(true) + assertThrows { + ExpressionParser.createTree(exp, null, nameValue, null)!! + .evaluate(TestTraceWriter(), ev, options, null) + } + println(options.contextNotNull.exceptionTraceMsg) + Assertions.assertEquals( + ContextNotFoundException(exArg).message, + options.contextNotNull.exceptionTraceMsg?.joinToString(".") + ) + } + + @DisplayName("不配置exceptionInsteadOfNull") + @ParameterizedTest + @ValueSource( + strings = [ + "string == null", + "obj.a.obj_obj1[2] == null", + "arr[0].arr_obj_1_n2 == null", + "obj.obj_obj1.obj_obj1_n1.a == null" + ] + ) + fun noExceptionInsteadOfNull(exp: String) { + val options = EvaluationOptions(false) + val result = ExpressionParser.createTree(exp, null, nameValue, null)!! + .evaluate(TestTraceWriter(), ev, options, null) + Assertions.assertTrue(result.equalsTrue) + } + + private val nameValue = mutableListOf().apply { + add(NamedValueInfo("string", ContextValueNode())) + add(NamedValueInfo("obj", ContextValueNode())) + add(NamedValueInfo("arr", ContextValueNode())) + } + private val ev = ExecutionContext(DictionaryContextData().apply { + add("obj", DictionaryContextData().apply { + add("obj_arr1", ArrayContextData().apply { + add(null) + }) + add("obj_obj1", DictionaryContextData().apply { + add("obj_obj1_n1", DictionaryContextData().apply { + }) + }) + }) + add("arr", ArrayContextData().apply { + add(DictionaryContextData().apply { + add("arr_obj_1_n1", StringContextData("arr_obj_1_n1_v")) + }) + }) + }) + } +} + +class TestTraceWriter : ITraceWriter { + override fun info(message: String?) { + logger.info(message ?: return) + } + + override fun verbose(message: String?) { + logger.debug(message ?: return) + } + + companion object { + private val logger = LoggerFactory.getLogger(TestTraceWriter::class.java) + } +} \ No newline at end of file diff --git a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/ExpressionParserTest.kt b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/ExpressionParserTest.kt index 17ad9d05165..8eb866a45bb 100644 --- a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/ExpressionParserTest.kt +++ b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/ExpressionParserTest.kt @@ -117,11 +117,6 @@ class ExpressionParserTest { @DisplayName("测试流水线变量中对象的转换") @Test fun variablesObjectConvert() { -// val variablesWithError = mapOf( -// "matrix.power" to "{url=cn.com, project=p-xxx}", -// "matrix.power.url" to "cn.com", -// "matrix.power.project" to "p-xxx" -// ) val variables = mapOf( "matrix.power" to "{ \"url\" : \"cn.com\", \"project\": \"p-xxx\" }", "matrix.power.url" to "cn.com", @@ -249,9 +244,50 @@ class ExpressionParserTest { } Assertions.assertEquals(v, ExpressionParser.evaluateByMap(k, variables, true)) } -// assertThrows { -// ExpressionParser.evaluateByMap("matrix.power.url=='cn.com'", variablesWithError, true) -// } + } + + @DisplayName("测试流水线变量中对象的转换2") + @Test + fun variablesObjectConvert2() { + val variables = mapOf( + "matrix.power" to "{ \"url\" : \"cn.com\", \"project\": \"p-xxx\" }", + "matrix.power.url" to "c1.com", + "matrix.power.project" to "p-xxx", + "jsonStr" to "{ \"url\" : \"cn.com\", \"project\": \"p-1xx\" }" + ) + val jsonKeys = setOf( + "matrix.power" + ) + val mapKeys = setOf( + "matrix" + ) + val expAndExpect = mapOf( + "matrix.power" to "{ \"url\" : \"cn.com\", \"project\": \"p-xxx\" }", + "matrix.power.url" to "c1.com", + "matrix.power.project" to "p-xxx", + "matrix" to mapOf("power" to mapOf("url" to "c1.com", "project" to "p-xxx")), + "fromJSON(toJSON(matrix.power)).url" to "c1.com", + "fromJSON(jsonStr).project" to "p-1xx" + ) + expAndExpect.forEach { (exp, expect) -> + if (exp in jsonKeys) { + Assertions.assertEquals( + JsonUtil.getObjectMapper().readTree(expect.toString()), + JsonUtil.getObjectMapper().readTree( + ExpressionParser.evaluateByMap(exp, variables, true).toString() + ) + ) + return@forEach + } + if (exp in mapKeys) { + Assertions.assertEquals( + JsonUtil.getObjectMapper().readTree(JsonUtil.toJson(expect)), + JsonUtil.getObjectMapper() + .readTree(JsonUtil.toJson(ExpressionParser.evaluateByMap(exp, variables, true)!!)) + ) + } + Assertions.assertEquals(expect, ExpressionParser.evaluateByMap(exp, variables, true)) + } } @DisplayName("测试解析文字") @@ -267,7 +303,8 @@ class ExpressionParserTest { ) literals.forEach { (exp, v) -> - val res = ExpressionParser.createTree(exp, null, null, null)!!.evaluate(null, null, null, null).value + val res = ExpressionParser.createTree(exp, null, null, null)!! + .evaluate(null, null, EvaluationOptions(false), null).value Assertions.assertEquals(v, res) } } @@ -495,7 +532,8 @@ class ExpressionParserTest { ) fun functionFromJsonTest(fromJson: String) { val (index, exp) = fromJson.split(" => ") - val res = ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, null, null).value + val res = ExpressionParser.createTree(exp, null, nameValue, null)!! + .evaluate(null, ev, EvaluationOptions(false), null).value when (index.toInt()) { 1 -> { Assertions.assertTrue(res is DictionaryContextData) @@ -536,7 +574,8 @@ class ExpressionParserTest { ) fun functionJoinTest(join: String) { val (index, exp) = join.split(" => ") - val res = ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, null, null).value + val res = ExpressionParser.createTree(exp, null, nameValue, null)!! + .evaluate(null, ev, EvaluationOptions(false), null).value when (index.toInt()) { 1 -> { Assertions.assertEquals("push|mr|tag", res) @@ -583,7 +622,13 @@ class ExpressionParserTest { val result = items[1] val subInfo = SubNameValueEvaluateInfo() val tree = ExpressionParser.createSubNameValueEvaluateTree(exp, null, parametersNameValue, null, subInfo)!! - var (res, isComplete, type) = tree.subNameValueEvaluate(null, parametersEv, null, subInfo, null) + var (res, isComplete, type) = tree.subNameValueEvaluate( + null, + parametersEv, + EvaluationOptions(false), + subInfo, + null + ) if (isComplete && (type == SubNameValueResultType.ARRAY || type == SubNameValueResultType.DICT)) { res = res.replace("\\\"", "\"") } @@ -598,7 +643,8 @@ class ExpressionParserTest { private fun valuesTest(param: String) { val (exp, result) = param.split(" => ") - val res = ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, null, null).value + val res = ExpressionParser.createTree(exp, null, nameValue, null)!! + .evaluate(null, ev, EvaluationOptions(false), null).value Assertions.assertEquals( when (result) { "true", "false" -> { diff --git a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/functions/FormatTest.kt b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/functions/FormatTest.kt index 8413abe9ab6..76f7f77a59d 100644 --- a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/functions/FormatTest.kt +++ b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/functions/FormatTest.kt @@ -37,6 +37,7 @@ import com.tencent.devops.common.expression.context.ContextValueNode import com.tencent.devops.common.expression.context.DictionaryContextData import com.tencent.devops.common.expression.context.NumberContextData import com.tencent.devops.common.expression.context.StringContextData +import com.tencent.devops.common.expression.expression.EvaluationOptions import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeAll @@ -56,28 +57,28 @@ internal class FormatTest { ExpressionParser.createTree( "format('{0}-{1}:{3}', variables.str, variables.doub, variables.arry)", null, nameValue, null - )!!.evaluate(null, ev, null, null) + )!!.evaluate(null, ev, EvaluationOptions(false), null) } Assertions.assertThrows(FunctionFormatException::class.java) { ExpressionParser.createTree( "format('{0}-{1}:3}', variables.str, variables.doub, variables.arry)", null, nameValue, null - )!!.evaluate(null, ev, null, null) + )!!.evaluate(null, ev, EvaluationOptions(false), null) } Assertions.assertThrows(FunctionFormatException::class.java) { ExpressionParser.createTree( "format('{0}-{1}:{3', variables.str, variables.doub, variables.arry)", null, nameValue, null - )!!.evaluate(null, ev, null, null) + )!!.evaluate(null, ev, EvaluationOptions(false), null) } Assertions.assertThrows(FunctionFormatException::class.java) { ExpressionParser.createTree( "format('{0:yyyyMMdd}', variables.str, variables.doub, variables.arry)", null, nameValue, null - )!!.evaluate(null, ev, null, null) + )!!.evaluate(null, ev, EvaluationOptions(false), null) } } @@ -92,7 +93,7 @@ internal class FormatTest { ) fun evaluateCoreTest(format: String) { val (exp, expect) = format.split(" => ") - val res = ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, null, null).value + val res = ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, EvaluationOptions(false), null).value Assertions.assertEquals(expect, res) } @@ -109,7 +110,7 @@ internal class FormatTest { val res = ExpressionParser .createSubNameValueEvaluateTree(exp, null, parametersNameValue, null, SubNameValueEvaluateInfo())!! - .subNameValueEvaluate(null, parametersEv, null, SubNameValueEvaluateInfo(), null).value + .subNameValueEvaluate(null, parametersEv, EvaluationOptions(false), SubNameValueEvaluateInfo(), null).value Assertions.assertEquals(expect, res) } diff --git a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/functions/StrToTimeTest.kt b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/functions/StrToTimeTest.kt index c027658e949..31a1ecc0c8d 100644 --- a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/functions/StrToTimeTest.kt +++ b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/functions/StrToTimeTest.kt @@ -37,6 +37,7 @@ import com.tencent.devops.common.expression.context.ContextValueNode import com.tencent.devops.common.expression.context.DictionaryContextData import com.tencent.devops.common.expression.context.NumberContextData import com.tencent.devops.common.expression.context.StringContextData +import com.tencent.devops.common.expression.expression.EvaluationOptions import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeAll @@ -56,7 +57,7 @@ internal class StrToTimeTest { ExpressionParser.createTree( "strToTime('2023-3-15')", null, nameValue, null - )!!.evaluate(null, ev, null, null) + )!!.evaluate(null, ev, EvaluationOptions(false), null) } } @@ -69,8 +70,8 @@ internal class StrToTimeTest { ) fun evaluateCoreTest(format: String) { val (exp, expect) = format.split(" => ") - val res1 = ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, null, null).value - val res2 = ExpressionParser.createTree(expect, null, nameValue, null)!!.evaluate(null, ev, null, null).value + val res1 = ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, EvaluationOptions(false), null).value + val res2 = ExpressionParser.createTree(expect, null, nameValue, null)!!.evaluate(null, ev, EvaluationOptions(false), null).value Assertions.assertEquals(res1, res2) } @@ -84,7 +85,7 @@ internal class StrToTimeTest { ) fun eqTest(format: String) { val (exp, expect) = format.split(" => ") - val res = ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, null, null).value + val res = ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, EvaluationOptions(false), null).value Assertions.assertEquals(expect, res.toString()) } @@ -100,7 +101,7 @@ internal class StrToTimeTest { val res = ExpressionParser .createSubNameValueEvaluateTree(exp, null, parametersNameValue, null, SubNameValueEvaluateInfo())!! - .subNameValueEvaluate(null, parametersEv, null, SubNameValueEvaluateInfo(), null).value + .subNameValueEvaluate(null, parametersEv, EvaluationOptions(false), SubNameValueEvaluateInfo(), null).value Assertions.assertEquals(expect, res) } diff --git a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/specialFuctions/hashFiles/HashFilesFunctionTest.kt b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/specialFuctions/hashFiles/HashFilesFunctionTest.kt index f65cb065926..1fd06d75d5c 100644 --- a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/specialFuctions/hashFiles/HashFilesFunctionTest.kt +++ b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/specialFuctions/hashFiles/HashFilesFunctionTest.kt @@ -36,6 +36,7 @@ import com.tencent.devops.common.expression.context.ContextValueNode import com.tencent.devops.common.expression.context.DictionaryContextData import com.tencent.devops.common.expression.context.NumberContextData import com.tencent.devops.common.expression.context.StringContextData +import com.tencent.devops.common.expression.expression.EvaluationOptions import com.tencent.devops.common.expression.expression.FunctionInfo import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo import org.junit.jupiter.api.Assertions @@ -62,11 +63,11 @@ internal class HashFilesFunctionTest { fun noRootPath() { val exp = "hashFiles('/data/a/a')" Assertions.assertThrows(RuntimeException::class.java) { - ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, null, null).value + ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, EvaluationOptions(false), null).value } val exp1 = "hashFiles('D: \\data\\.\\..')" Assertions.assertThrows(RuntimeException::class.java) { - ExpressionParser.createTree(exp1, null, nameValue, null)!!.evaluate(null, ev, null, null).value + ExpressionParser.createTree(exp1, null, nameValue, null)!!.evaluate(null, ev, EvaluationOptions(false), null).value } } @@ -75,7 +76,7 @@ internal class HashFilesFunctionTest { fun noIndexPath() { val exp = "hashFiles('/data/./..')" Assertions.assertThrows(RuntimeException::class.java) { - ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, null, null).value + ExpressionParser.createTree(exp, null, nameValue, null)!!.evaluate(null, ev, EvaluationOptions(false), null).value } } } @@ -103,7 +104,7 @@ internal class HashFilesFunctionTest { HashFilesFunction() ) ) - )!!.evaluate(null, ev, null, null).value + )!!.evaluate(null, ev, EvaluationOptions(false), null).value ) } @@ -129,7 +130,7 @@ internal class HashFilesFunctionTest { ) ), subInfo - )!!.subNameValueEvaluate(null, parametersEv, null, subInfo, null).value + )!!.subNameValueEvaluate(null, parametersEv, EvaluationOptions(false), subInfo, null).value ) } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt index 093fe644a3b..6e366899724 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt @@ -28,9 +28,11 @@ package com.tencent.devops.process.yaml.transfer import com.tencent.devops.common.api.constant.CommonMessageCode.YAML_NOT_VALID +import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.container.Stage +import com.tencent.devops.common.pipeline.dialect.PipelineDialectType import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting import com.tencent.devops.common.pipeline.pojo.setting.Subscription @@ -42,6 +44,7 @@ import com.tencent.devops.process.yaml.transfer.VariableDefault.nullIfDefault import com.tencent.devops.process.yaml.transfer.aspect.PipelineTransferAspectWrapper import com.tencent.devops.process.yaml.transfer.pojo.ModelTransferInput import com.tencent.devops.process.yaml.transfer.pojo.YamlTransferInput +import com.tencent.devops.process.yaml.v3.enums.SyntaxDialectType import com.tencent.devops.process.yaml.v3.models.Concurrency import com.tencent.devops.process.yaml.v3.models.Extends import com.tencent.devops.process.yaml.v3.models.GitNotices @@ -98,7 +101,7 @@ class ModelTransfer @Autowired constructor( maxQueueSize = yaml.concurrency?.queueLength ?: VariableDefault.DEFAULT_PIPELINE_SETTING_MAX_QUEUE_SIZE, maxConRunningQueueSize = yaml.concurrency?.maxParallel ?: PIPELINE_SETTING_MAX_CON_QUEUE_SIZE_MAX, labels = yaml2Labels(yamlInput), - pipelineAsCodeSettings = yamlInput.asCodeSettings, + pipelineAsCodeSettings = yamlSyntaxDialect2Setting(yaml.syntaxDialect), successSubscriptionList = yamlNotice2Setting( projectId = yamlInput.projectCode, notices = yaml.notices?.filter { it.checkNotifyForSuccess() } @@ -118,6 +121,22 @@ class ModelTransfer @Autowired constructor( } } + private fun yamlSyntaxDialect2Setting(syntaxDialectType: String?): PipelineAsCodeSettings? { + if (syntaxDialectType.isNullOrBlank()) return null + return when (syntaxDialectType) { + SyntaxDialectType.INHERIT.name -> PipelineAsCodeSettings(inheritedDialect = true) + SyntaxDialectType.CLASSIC.name -> PipelineAsCodeSettings( + inheritedDialect = false, + pipelineDialect = PipelineDialectType.CLASSIC.name + ) + SyntaxDialectType.CONSTRAINT.name -> PipelineAsCodeSettings( + inheritedDialect = false, + pipelineDialect = PipelineDialectType.CONSTRAINED.name + ) + else -> null + } + } + private fun prepareModelGroups(projectId: String, notice: Subscription): Subscription { if (notice.groups.isEmpty()) return notice val info = transferCache.getProjectGroupAndUsers(projectId)?.associateBy { it.displayName } ?: return notice @@ -207,7 +226,8 @@ class ModelTransfer @Autowired constructor( desc = modelInput.setting.desc.ifEmpty { null }, label = label, resources = modelInput.model.resources, - notices = makeNoticesV3(modelInput.setting) + notices = makeNoticesV3(modelInput.setting), + syntaxDialect = makeSyntaxDialect(modelInput.setting) ) else -> { throw PipelineTransferException( @@ -404,6 +424,16 @@ class ModelTransfer @Autowired constructor( } } + private fun makeSyntaxDialect(setting: PipelineSetting): String? { + val asCodeSettings = setting.pipelineAsCodeSettings ?: return null + return when { + asCodeSettings.inheritedDialect == true -> SyntaxDialectType.INHERIT.name + asCodeSettings.pipelineDialect == PipelineDialectType.CLASSIC.name -> SyntaxDialectType.CLASSIC.name + asCodeSettings.pipelineDialect == PipelineDialectType.CONSTRAINED.name -> SyntaxDialectType.CONSTRAINT.name + else -> null + } + } + @Suppress("NestedBlockDepth") private fun preparePipelineLabels( userId: String, diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/ParametersExpressionParse.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/ParametersExpressionParse.kt index bba8eade3cd..aaea3fdcabb 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/ParametersExpressionParse.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/ParametersExpressionParse.kt @@ -15,6 +15,7 @@ import com.tencent.devops.common.expression.context.ExpressionContextData import com.tencent.devops.common.expression.context.NumberContextData import com.tencent.devops.common.expression.context.PipelineContextData import com.tencent.devops.common.expression.context.StringContextData +import com.tencent.devops.common.expression.expression.EvaluationOptions import com.tencent.devops.common.expression.expression.FunctionInfo import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo import com.tencent.devops.common.expression.expression.specialFuctions.hashFiles.HashFilesFunction @@ -398,7 +399,7 @@ object ParametersExpressionParse { val (value, isComplete, type) = try { ExpressionParser.createSubNameValueEvaluateTree( expression, null, nameValues, functionList, subInfo - )?.subNameValueEvaluate(null, context, null, subInfo, null) + )?.subNameValueEvaluate(null, context, EvaluationOptions(false), subInfo, null) ?: throw YamlTemplateException("create evaluate tree is null") } catch (e: Throwable) { throw error(Constants.EXPRESSION_EVALUATE_ERROR.format(path, expression, e.message)) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/enums/SyntaxDialectType.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/enums/SyntaxDialectType.kt new file mode 100644 index 00000000000..47e63569be2 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/enums/SyntaxDialectType.kt @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.yaml.v3.enums + +/** + * 流水线方言类型 + */ +enum class SyntaxDialectType { + INHERIT, + CLASSIC, + CONSTRAINT +} diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreScriptBuildYamlParser.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreScriptBuildYamlParser.kt index fb547685247..403ad5a6647 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreScriptBuildYamlParser.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreScriptBuildYamlParser.kt @@ -57,6 +57,7 @@ interface PreScriptBuildYamlIParser : YamlVersionParser { val disablePipeline: Boolean? val recommendedVersion: RecommendedVersion? val customBuildNum: String? + val syntaxDialect: String? } /** @@ -83,7 +84,8 @@ data class PreScriptBuildYamlParser( override val concurrency: Concurrency? = null, override val disablePipeline: Boolean? = null, override val recommendedVersion: RecommendedVersion? = null, - override val customBuildNum: String? = null + override val customBuildNum: String? = null, + override val syntaxDialect: String? ) : PreScriptBuildYamlIParser { override fun yamlVersion() = YamlVersion.V2_0 } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreScriptBuildYamlV3Parser.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreScriptBuildYamlV3Parser.kt index 7c4933e59cb..c8777006258 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreScriptBuildYamlV3Parser.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreScriptBuildYamlV3Parser.kt @@ -61,7 +61,8 @@ data class PreScriptBuildYamlV3Parser( override val concurrency: Concurrency? = null, override val disablePipeline: Boolean? = null, override val recommendedVersion: RecommendedVersion? = null, - override val customBuildNum: String? = null + override val customBuildNum: String? = null, + override val syntaxDialect: String? ) : PreScriptBuildYamlIParser { override fun yamlVersion() = YamlVersion.V3_0 } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlParser.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlParser.kt index 35dc3c90cd0..87a06af8357 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlParser.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlParser.kt @@ -63,6 +63,7 @@ interface IPreTemplateScriptBuildYamlParser : YamlVersionParser { var disablePipeline: Boolean? var recommendedVersion: RecommendedVersion? var customBuildNum: String? + var syntaxDialect: String? fun replaceTemplate(f: (param: ITemplateFilter) -> PreScriptBuildYamlIParser) @@ -132,7 +133,9 @@ data class PreTemplateScriptBuildYamlParser( @JsonProperty("recommended-version") override var recommendedVersion: RecommendedVersion? = null, @JsonProperty("custom-build-num") - override var customBuildNum: String? = null + override var customBuildNum: String? = null, + @JsonProperty("syntax-dialect") + override var syntaxDialect: String? ) : IPreTemplateScriptBuildYamlParser, ITemplateFilter { init { @@ -149,7 +152,8 @@ data class PreTemplateScriptBuildYamlParser( triggerOn = triggerOn, resources = resources, notices = notices, - concurrency = concurrency + concurrency = concurrency, + syntaxDialect = syntaxDialect ) } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlV3Parser.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlV3Parser.kt index 3085cdc7ad5..4a94b46520f 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlV3Parser.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlV3Parser.kt @@ -68,7 +68,9 @@ data class PreTemplateScriptBuildYamlV3Parser( @JsonProperty("recommended-version") override var recommendedVersion: RecommendedVersion? = null, @JsonProperty("custom-build-num") - override var customBuildNum: String? = null + override var customBuildNum: String? = null, + @JsonProperty("syntax-dialect") + override var syntaxDialect: String? = null ) : IPreTemplateScriptBuildYamlParser, ITemplateFilter { companion object { private val logger = LoggerFactory.getLogger(PreTemplateScriptBuildYamlV3Parser::class.java) @@ -89,7 +91,8 @@ data class PreTemplateScriptBuildYamlV3Parser( resources = resources, notices = notices, concurrency = concurrency, - disablePipeline = disablePipeline + disablePipeline = disablePipeline, + syntaxDialect = syntaxDialect ) } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/parsers/template/ParametersExpressionParse.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/parsers/template/ParametersExpressionParse.kt index 22c64bee830..3aa2e9508a6 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/parsers/template/ParametersExpressionParse.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/parsers/template/ParametersExpressionParse.kt @@ -15,6 +15,7 @@ import com.tencent.devops.common.expression.context.ExpressionContextData import com.tencent.devops.common.expression.context.NumberContextData import com.tencent.devops.common.expression.context.PipelineContextData import com.tencent.devops.common.expression.context.StringContextData +import com.tencent.devops.common.expression.expression.EvaluationOptions import com.tencent.devops.common.expression.expression.FunctionInfo import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo import com.tencent.devops.common.expression.expression.specialFuctions.hashFiles.HashFilesFunction @@ -399,7 +400,7 @@ object ParametersExpressionParse { val (value, isComplete, type) = try { ExpressionParser.createSubNameValueEvaluateTree( expression, null, nameValues, functionList, subInfo - )?.subNameValueEvaluate(null, context, null, subInfo, null) + )?.subNameValueEvaluate(null, context, EvaluationOptions(false), subInfo, null) ?: throw YamlTemplateException("create evaluate tree is null") } catch (e: Throwable) { throw error(Constants.EXPRESSION_EVALUATE_ERROR.format(path, expression, e.message)) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/java/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtil.java b/src/backend/ci/core/common/common-pipeline/src/main/java/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtil.java new file mode 100644 index 00000000000..8f68673917b --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/java/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtil.java @@ -0,0 +1,159 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline; + +import com.tencent.devops.common.api.util.JsonSchemaUtil; +import com.tencent.devops.common.api.util.JsonUtil; +import com.tencent.devops.common.api.util.ReflectUtil; +import com.tencent.devops.common.pipeline.utils.ExprReplacementUtil; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ExprReplaceEnvVarUtil { + + public static Object replaceEnvVar(Object obj, Map envMap) { + return replaceEnvVar(obj, + new ExprReplacementOptions(envMap, false, null, null, null) + ); + } + + /** + * 把对象字段值中的表达式替换成环境变量 + * @param obj 需要把占位符替换环境变量的对象(对象如果是集合对象，注意要选择支持增加、删除等操作的集合类型，不要选择类似SingletonMap这种) + * @param envMap 环境变量Map + * @return 变量替换后的对象 + */ + @SuppressWarnings("all") + public static Object replaceEnvVar(Object obj, ExprReplacementOptions options) { + Map envMap = options.getContextMap(); + if (obj instanceof Map) { + // 递归替换map对象中的变量 + Set> entrySet = ((Map) obj).entrySet(); + for (Map.Entry entry : entrySet) { + Object value = entry.getValue(); + if (!isNormalReplaceEnvVar(value)) { + entry.setValue(replaceEnvVar(value, options)); + } else { + entry.setValue(handleNormalEnvVar(value, options)); + } + } + } else if (obj instanceof List) { + // 递归替换list对象中的变量 + List dataList = (List) obj; + for (int i = 0; i < dataList.size(); i++) { + Object value = dataList.get(i); + if (!isNormalReplaceEnvVar(value)) { + dataList.set(i, replaceEnvVar(value, options)); + } else { + dataList.set(i, handleNormalEnvVar(value, options)); + } + } + } else if (obj instanceof Set) { + // 递归替换set对象中的变量 + Set objSet = (Set) obj; + Set replaceObjSet = new HashSet(objSet); + Iterator it = replaceObjSet.iterator(); + while (it.hasNext()) { + Object value = it.next(); + objSet.remove(value); + if (!isNormalReplaceEnvVar(value)) { + objSet.add(replaceEnvVar(value, options)); + } else { + objSet.add(handleNormalEnvVar(value, options)); + } + } + } else if (isNormalReplaceEnvVar(obj)) { + // 替换基本类型对象或字符串对象中的变量 + obj = handleNormalEnvVar(obj, options); + } else { + try { + // 把对象转换成map后进行递归替换变量 + Map dataMap = JsonUtil.INSTANCE.toMap(obj); + replaceEnvVar(dataMap, options); + obj = JsonUtil.INSTANCE.to(JsonUtil.INSTANCE.toJson(dataMap, true), obj.getClass()); + } catch (Throwable e) { + // 转换不了map的对象则进行直接替换 + obj = ExprReplacementUtil.INSTANCE.parseExpression( + JsonUtil.INSTANCE.toJson(obj, true), options + ); + } + } + return obj; + } + + private static Object handleNormalEnvVar(Object obj, ExprReplacementOptions options) { + // 只有字符串参数才需要进行变量替换，其它基本类型参数无需进行变量替换 + if (obj instanceof String) { + String objStr = ((String) obj).trim(); + if (objStr.startsWith("{") && objStr.endsWith("}") && JsonSchemaUtil.INSTANCE.validateJson(objStr)) { + try { + Object dataObj = JsonUtil.INSTANCE.to((String) obj, Map.class); + // string能正常转换成map，则说明是json串，那么把dataObj进行递归替换变量后再转成json串 + dataObj = replaceEnvVar(dataObj, options); + obj = JsonUtil.INSTANCE.toJson(dataObj, true); + } catch (Throwable e) { + // 转换不了map的字符串对象则直接替换 + obj = ExprReplacementUtil.INSTANCE.parseExpression( + JsonUtil.INSTANCE.toJson(obj, true), options + ); + } + } else if (objStr.startsWith("[") && objStr.endsWith("]") && JsonSchemaUtil.INSTANCE.validateJson(objStr)) { + try { + Object dataObj = JsonUtil.INSTANCE.to((String) obj, List.class); + // string能正常转成list，说明是json串，把dataObj进行递归替换变量后再转成json串 + dataObj = replaceEnvVar(dataObj, options); + obj = JsonUtil.INSTANCE.toJson(dataObj, true); + } catch (Throwable e1) { + // 转换不了list的字符串对象则直接替换 + obj = ExprReplacementUtil.INSTANCE.parseExpression( + JsonUtil.INSTANCE.toJson(obj, true), options + ); + } + } else { + // 转换不了map或者list的字符串对象则直接替换 + obj = ExprReplacementUtil.INSTANCE.parseExpression( + JsonUtil.INSTANCE.toJson(obj, true), options + ); + } + } + return obj; + } + + /** + * 判断对象是否是普通替换对象 + * @param obj 需要把占位符替换环境变量的对象(对象如果是集合对象，注意要选择支持增加、删除等操作的集合类型，不要选择类似SingletonMap这种) + * @return 是否是普通替换对象 + */ + private static Boolean isNormalReplaceEnvVar(Object obj) { + return obj == null || ReflectUtil.INSTANCE.isNativeType(obj) || obj instanceof String; + } +} diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt index 161f22831f2..3624042aa80 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParser.kt @@ -30,20 +30,14 @@ package com.tencent.devops.common.pipeline import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.ObjectReplaceEnvVarUtil import com.tencent.devops.common.expression.ExecutionContext -import com.tencent.devops.common.expression.ExpressionParseException -import com.tencent.devops.common.expression.ExpressionParser -import com.tencent.devops.common.expression.context.ContextValueNode -import com.tencent.devops.common.expression.context.DictionaryContextData -import com.tencent.devops.common.expression.context.PipelineContextData -import com.tencent.devops.common.expression.context.RuntimeDictionaryContextData import com.tencent.devops.common.expression.context.RuntimeNamedValue import com.tencent.devops.common.expression.expression.ExpressionOutput import com.tencent.devops.common.expression.expression.IFunctionInfo import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo -import org.apache.tools.ant.filters.StringInputStream +import com.tencent.devops.common.pipeline.dialect.IPipelineDialect +import com.tencent.devops.common.pipeline.utils.ExprReplacementUtil import org.slf4j.LoggerFactory -import java.io.BufferedReader -import java.io.InputStreamReader +import java.util.regex.Pattern @Suppress( "LoopWithTooManyJumpStatements", @@ -56,6 +50,7 @@ import java.io.InputStreamReader object EnvReplacementParser { private val logger = LoggerFactory.getLogger(EnvReplacementParser::class.java) + private val expressionPattern = Pattern.compile("\\$[{]{2}([^$^{}]+)[}]{2}") /** * 根据环境变量map进行object处理并保持原类型 @@ -67,259 +62,75 @@ object EnvReplacementParser { * @param output 表达式计算时输出 */ fun parse( - value: String?, + value: Any?, contextMap: Map, onlyExpression: Boolean? = false, contextPair: Pair>? = null, functions: Iterable? = null, output: ExpressionOutput? = null ): String { - if (value.isNullOrBlank()) return "" - return if (onlyExpression == true) { - try { - val (context, nameValues) = contextPair - ?: getCustomExecutionContextByMap(contextMap) - ?: return value - parseExpression( - value = value, - context = context, - nameValues = nameValues, - functions = functions, - output = output - ) - } catch (ignore: Throwable) { - logger.warn("[$value]|EnvReplacementParser expression invalid: ", ignore) - value - } - } else { - ObjectReplaceEnvVarUtil.replaceEnvVar(value, contextMap).let { - JsonUtil.toJson(it, false) - } - } - } - - fun getCustomExecutionContextByMap( - variables: Map, - extendNamedValueMap: List? = null - ): Pair>? { - try { - val context = ExecutionContext(DictionaryContextData()) - val nameValue = mutableListOf() - extendNamedValueMap?.forEach { namedValue -> - nameValue.add(NamedValueInfo(namedValue.key, ContextValueNode())) - context.expressionValues.add( - namedValue.key, - RuntimeDictionaryContextData(namedValue) - ) - } - ExpressionParser.fillContextByMap(variables, context, nameValue) - return Pair(context, nameValue) - } catch (ignore: Throwable) { - logger.warn("EnvReplacementParser context invalid: $variables", ignore) - return null - } - } - - private fun parseExpression( - value: String, - nameValues: List, - context: ExecutionContext, - functions: Iterable? = null, - output: ExpressionOutput? = null - ): String { - val strReader = InputStreamReader(StringInputStream(value)) - val bufferReader = BufferedReader(strReader) - val newValue = StringBuilder() - try { - var line = bufferReader.readLine() - while (line != null) { - // 跳过空行和注释行 - val blocks = findExpressions(line) - if (line.isBlank() || blocks.isEmpty()) { - newValue.append(line).append("\n") - line = bufferReader.readLine() - continue - } - val onceResult = parseExpressionLine( - value = line, - blocks = blocks, - context = context, - nameValues = nameValues, - functions = functions, - output = output - ) - - val newLine = findExpressions(onceResult).let { - if (it.isEmpty()) { - onceResult - } else { - parseExpressionLine( - value = onceResult, - blocks = it, - context = context, - nameValues = nameValues, - functions = functions, - output = output - ) - } - } - newValue.append(newLine).append("\n") - line = bufferReader.readLine() - } - } finally { - strReader.close() - bufferReader.close() - } - return newValue.toString().removeSuffix("\n") + val options = ExprReplacementOptions( + contextMap = contextMap, + contextPair = contextPair, + functions = functions, + output = output + ) + return parse(value = value, onlyExpression = onlyExpression, options = options) } /** - * 解析表达式，根据 findExpressions 寻找的括号优先级进行解析 + * 根据环境变量map进行object处理并保持原类型 + * 根据方言的配置判断是否能够使用${}或者变量值是否存在 */ - private fun parseExpressionLine( - value: String, - blocks: List>, - nameValues: List, - context: ExecutionContext, + fun parse( + value: Any?, + contextMap: Map, + dialect: IPipelineDialect, + contextPair: Pair>? = null, functions: Iterable? = null, output: ExpressionOutput? = null ): String { - var chars = value.toList() - blocks.forEachIndexed nextBlockLevel@{ blockLevel, blocksInLevel -> - blocksInLevel.forEachIndexed nextBlock@{ blockI, block -> - // 表达式因为含有 ${{ }} 所以起始向后推3位，末尾往前推两位 - val expression = chars.joinToString("").substring(block.startIndex + 3, block.endIndex - 1) - if (expression.isBlank()) return@nextBlock - var result = try { - ExpressionParser.createTree(expression, null, nameValues, functions)!! - .evaluate(null, context, null, output).value.let { - if (it is PipelineContextData) it.fetchValue() else it - }?.let { - JsonUtil.toJson(it, false) - } ?: "" - } catch (ignore: ExpressionParseException) { - return@nextBlock - } - - if ((blockLevel + 1 < blocks.size) && - !( - (block.startIndex - 1 >= 0 && chars[block.startIndex - 1] == '.') || - (block.endIndex + 1 < chars.size && chars[block.endIndex + 1] == '.') - ) - ) { - result = "'$result'" - } - - val charList = result.toList() - - // 将替换后的表达式嵌入原本的line - val startSub = if (block.startIndex - 1 < 0) { - listOf() - } else { - chars.slice(0 until block.startIndex) - } - val endSub = if (block.endIndex + 1 >= chars.size) { - listOf() - } else { - chars.slice(block.endIndex + 1 until chars.size) - } - chars = startSub + charList + endSub - - // 将替换后的字符查传递给后边的括号位数 - val diffNum = charList.size - (block.endIndex - block.startIndex + 1) - blocks.forEachIndexed { i, bl -> - bl.forEachIndexed level@{ j, b -> - if (i <= blockLevel && j <= blockI) { - return@level - } - if (blocks[i][j].startIndex > block.endIndex) { - blocks[i][j].startIndex += diffNum - } - if (blocks[i][j].endIndex > block.endIndex) { - blocks[i][j].endIndex += diffNum - } - } - } - } - } - - return chars.joinToString("") + val options = ExprReplacementOptions( + contextMap = contextMap, + contextNotNull = !dialect.supportMissingVar(), + contextPair = contextPair, + functions = functions, + output = output + ) + return parse( + value = value, + onlyExpression = dialect.supportUseExpression(), + options = options + ) } - /** - * 寻找语句中包含 ${{}}的表达式的位置，返回成对的位置坐标，并根据优先级排序 - * 优先级算法目前暂定为 从里到外，从左到右 - * @param levelMax 返回的最大层数，从深到浅。默认为2层 - * 例如: 替换顺序如数字所示 ${{ 4 ${{ 2 ${{ 1 }} }} ${{ 3 }} }} - * @return [ 层数次序 [ 括号 ] ] [[1], [2, 3], [4]]]] - */ - private fun findExpressions(condition: String, levelMax: Int = 2): List> { - val stack = ArrayDeque() - var index = 0 - val chars = condition.toCharArray() - val levelMap = mutableMapOf>() - while (index < chars.size) { - if (index + 2 < chars.size && chars[index] == '$' && chars[index + 1] == '{' && chars[index + 2] == '{' - ) { - stack.addLast(index) - index += 3 - continue - } - - if (index + 1 < chars.size && chars[index] == '}' && chars[index + 1] == '}' - ) { - val start = stack.removeLastOrNull() - if (start != null) { - // 栈里剩下几个前括号，这个完整括号的优先级就是多少 - val level = stack.size + 1 - if (levelMap.containsKey(level)) { - levelMap[level]!!.add(ExpressionBlock(start, index + 1)) - } else { - levelMap[level] = mutableListOf(ExpressionBlock(start, index + 1)) - } - } - index += 2 - continue - } - - index++ - } - - if (levelMap.isEmpty()) { - return listOf() + fun parse( + value: Any?, + onlyExpression: Boolean?, + options: ExprReplacementOptions + ): String { + if (value == null) return "" + return if (onlyExpression == true) { + ExprReplaceEnvVarUtil.replaceEnvVar(value, options) + } else { + ObjectReplaceEnvVarUtil.replaceEnvVar(value, options.contextMap) + }.let { + JsonUtil.toJson(it, false) } + } - val result = mutableListOf>() - var max = 0 - var listIndex = 0 - run end@{ - levelMap.keys.sortedDescending().forEach result@{ level -> - val blocks = levelMap[level] ?: return@result - blocks.sortBy { it.startIndex } - blocks.forEach { block -> - if (result.size < listIndex + 1) { - result.add(mutableListOf(block)) - } else { - result[listIndex].add(block) - } - } - listIndex++ - max++ - if (max == levelMax) { - return@end - } - } - } - return result + fun getCustomExecutionContextByMap( + variables: Map, + extendNamedValueMap: List? = null + ): Pair>? { + return ExprReplacementUtil.getCustomExecutionContextByMap( + variables = variables, + extendNamedValueMap = extendNamedValueMap + ) } - /** - * 表达式括号项 ${{ }} - * @param startIndex 括号开始位置即 $ 位置 - * @param endIndex 括号结束位置即最后一个 } 位置 - */ - data class ExpressionBlock( - var startIndex: Int, - var endIndex: Int - ) + fun containsExpressions(value: String?): Boolean { + if (value == null) return false + return expressionPattern.matcher(value).find() + } } diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/ExprReplacementOptions.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/ExprReplacementOptions.kt new file mode 100644 index 00000000000..b3f0abd8989 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/ExprReplacementOptions.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline + +import com.tencent.devops.common.expression.ExecutionContext +import com.tencent.devops.common.expression.expression.ExpressionOutput +import com.tencent.devops.common.expression.expression.IFunctionInfo +import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo +import io.swagger.v3.oas.annotations.media.Schema + +/** + * 表达式替换上下文 + */ +@Schema(title = "表达式替换参数") +data class ExprReplacementOptions( + @get:Schema(title = "环境变量", required = true) + val contextMap: Map, + @get:Schema(title = "值是否能不存在", required = true) + val contextNotNull: Boolean = false, + @get:Schema(title = "表达式上下文", required = true) + val contextPair: Pair>? = null, + val functions: Iterable? = null, + val output: ExpressionOutput? = null +) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/PipelineVersionWithModel.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/PipelineVersionWithModel.kt index 88ce275873f..6717fea9cf5 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/PipelineVersionWithModel.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/PipelineVersionWithModel.kt @@ -51,5 +51,9 @@ data class PipelineVersionWithModel( @get:Schema(title = "是否支持YAML解析", required = true) val yamlSupported: Boolean, @get:Schema(title = "YAML解析异常信息") - val yamlInvalidMsg: String? + val yamlInvalidMsg: String?, + @get:Schema(title = "更新操作人", required = true) + val updater: String?, + @get:Schema(title = "版本修改时间", required = true) + val updateTime: Long? ) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ClassicPipelineDialect.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ClassicPipelineDialect.kt new file mode 100644 index 00000000000..8b090955d07 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ClassicPipelineDialect.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline.dialect + +/** + * 传统模式流水线方言 + */ +class ClassicPipelineDialect : IPipelineDialect { + override fun getPipelineDialectType() = PipelineDialectType.CLASSIC.name + + override fun supportUseExpression() = false + + override fun supportUseSingleCurlyBracesVar() = true + + override fun supportLongVarValue() = true + + override fun supportChineseVarName() = true + + override fun supportMissingVar() = true +} diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ConstrainedPipelineDialect.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ConstrainedPipelineDialect.kt new file mode 100644 index 00000000000..a3efb42ae1c --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/ConstrainedPipelineDialect.kt @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline.dialect + +/** + * 约束模式语法风格 + */ +class ConstrainedPipelineDialect : IPipelineDialect { + override fun getPipelineDialectType() = PipelineDialectType.CONSTRAINED.name + + override fun supportUseExpression() = true + + override fun supportUseSingleCurlyBracesVar() = false + + override fun supportLongVarValue() = false + + override fun supportChineseVarName() = false + + override fun supportMissingVar() = false +} diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/IPipelineDialect.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/IPipelineDialect.kt new file mode 100644 index 00000000000..c04fbff1d0a --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/IPipelineDialect.kt @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline.dialect + +/** + * 流水线语法风格 + */ +interface IPipelineDialect { + + fun getPipelineDialectType(): String + + /** + * 支持插件变量使用表达式 + */ + fun supportUseExpression(): Boolean + /** + * 是否支持${}变量引用 + * + */ + fun supportUseSingleCurlyBracesVar(): Boolean + + /** + * 是否支持长变量 + */ + fun supportLongVarValue(): Boolean + + /** + * 是否支持中文变量名 + */ + fun supportChineseVarName(): Boolean + + /** + * 是否支持变量不存在 + */ + fun supportMissingVar(): Boolean +} diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/PipelineDialectType.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/PipelineDialectType.kt new file mode 100644 index 00000000000..7ab9c207f71 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/PipelineDialectType.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline.dialect + +/** + * 流水线语法风格 + * + */ +enum class PipelineDialectType(val dialect: IPipelineDialect) { + // 传统模式 + CLASSIC(ClassicPipelineDialect()), + + // 约束模式 + CONSTRAINED(ConstrainedPipelineDialect()); +} diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/PipelineDialectUtil.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/PipelineDialectUtil.kt new file mode 100644 index 00000000000..07b3c51f667 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/dialect/PipelineDialectUtil.kt @@ -0,0 +1,115 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline.dialect + +import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings +import com.tencent.devops.common.pipeline.dialect.PipelineDialectType.CLASSIC +import com.tencent.devops.common.pipeline.dialect.PipelineDialectType.CONSTRAINED +import com.tencent.devops.common.pipeline.enums.ChannelCode + +object PipelineDialectUtil { + fun getPipelineDialect(pipelineDialectType: String?): IPipelineDialect { + return pipelineDialectType?.let { + PipelineDialectType.valueOf(it).dialect + } ?: CLASSIC.dialect + } + + fun getPipelineDialect(asCodeSettings: PipelineAsCodeSettings?): IPipelineDialect { + if (asCodeSettings == null) return CLASSIC.dialect + return with(asCodeSettings) { + getPipelineDialect( + inheritedDialect = inheritedDialect, + projectDialect = projectDialect, + pipelineDialect = pipelineDialect + ) + } + } + + fun getPipelineDialect( + inheritedDialect: Boolean?, + projectDialect: String?, + pipelineDialect: String? + ): IPipelineDialect { + return getPipelineDialectType( + inheritedDialect = inheritedDialect, + projectDialect = projectDialect, + pipelineDialect = pipelineDialect + ).dialect + } + + fun getPipelineDialectType( + inheritedDialect: Boolean?, + projectDialect: String?, + pipelineDialect: String? + ): PipelineDialectType { + return when { + // inheritedDialect为空和true都继承项目配置 + inheritedDialect != false && projectDialect != null -> + PipelineDialectType.valueOf(projectDialect) + + inheritedDialect == false && pipelineDialect != null -> + PipelineDialectType.valueOf(pipelineDialect) + + else -> + CLASSIC + } + } + + fun getPipelineDialectType(asCodeSettings: PipelineAsCodeSettings?): PipelineDialectType { + if (asCodeSettings == null) return CLASSIC + return with(asCodeSettings) { + getPipelineDialectType( + inheritedDialect = inheritedDialect, + projectDialect = projectDialect, + pipelineDialect = pipelineDialect + ) + } + } + + fun getPipelineDialectType( + channelCode: ChannelCode, + asCodeSettings: PipelineAsCodeSettings? + ): PipelineDialectType { + return when { + asCodeSettings == null -> CLASSIC + // stream并且开启pac需要使用制约模式 + channelCode == ChannelCode.GIT && asCodeSettings.enable -> + CONSTRAINED + + else -> { + with(asCodeSettings) { + getPipelineDialectType( + inheritedDialect = inheritedDialect, + projectDialect = projectDialect, + pipelineDialect = pipelineDialect + ) + } + } + } + } +} diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/extend/ModelCheckPlugin.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/extend/ModelCheckPlugin.kt index a4da973a95b..7b66e9ffca4 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/extend/ModelCheckPlugin.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/extend/ModelCheckPlugin.kt @@ -30,6 +30,7 @@ package com.tencent.devops.common.pipeline.extend import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.container.Container +import com.tencent.devops.common.pipeline.dialect.IPipelineDialect import com.tencent.devops.common.pipeline.option.JobControlOption import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.pipeline.pojo.element.atom.BeforeDeleteParam @@ -45,6 +46,7 @@ interface ModelCheckPlugin { * 检查[model]编排的完整性，并返回[JobSize + ElementSize = MetaSize]所有元素数量 * @param userId 操作人 * @param oauthUser 当前流水线权限代持人 + * @param pipelineDialect 流水线方言,只有新增/编辑流水线或模版时才需要传入 * @throws RuntimeException 子类 将检查失败或异常的以[ErrorCodeException]类抛出 */ @Throws(ErrorCodeException::class) @@ -53,7 +55,8 @@ interface ModelCheckPlugin { projectId: String?, userId: String, isTemplate: Boolean = false, - oauthUser: String? = null + oauthUser: String? = null, + pipelineDialect: IPipelineDialect? = null ): Int /** diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/option/MatrixControlOption.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/option/MatrixControlOption.kt index c0deded3ff5..8a120250a09 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/option/MatrixControlOption.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/option/MatrixControlOption.kt @@ -72,7 +72,7 @@ data class MatrixControlOption( /** * 根据[strategyStr], [includeCaseStr], [excludeCaseStr]计算后得到的矩阵配置 */ - fun convertMatrixConfig(buildContext: Map, asCodeEnabled: Boolean? = false): MatrixConfig { + fun convertMatrixConfig(buildContext: Map): MatrixConfig { val matrixConfig = try { // 由于yaml和json结构不同，就不放在同一函数进行解析了 convertStrategyYaml(buildContext) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/StagePauseCheck.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/StagePauseCheck.kt index 54a00096db9..5238e314f49 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/StagePauseCheck.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/StagePauseCheck.kt @@ -30,6 +30,7 @@ package com.tencent.devops.common.pipeline.pojo import com.tencent.devops.common.api.util.UUIDUtil import com.tencent.devops.common.api.util.timestampmilli import com.tencent.devops.common.pipeline.EnvReplacementParser +import com.tencent.devops.common.pipeline.dialect.IPipelineDialect import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.ManualReviewAction import com.tencent.devops.common.pipeline.option.StageControlOption @@ -180,25 +181,44 @@ data class StagePauseCheck( /** * 进入审核流程前完成所有审核人变量替换 */ - fun parseReviewVariables(variables: Map, asCodeEnabled: Boolean?) { + fun parseReviewVariables(variables: Map, dialect: IPipelineDialect) { + val contextPair = EnvReplacementParser.getCustomExecutionContextByMap(variables) reviewGroups?.forEach { group -> if (group.status != null) return@forEach if (group.reviewers.isNotEmpty()) { val reviewers = group.reviewers.joinToString(",") - val realReviewers = EnvReplacementParser.parse(reviewers, variables, asCodeEnabled) - .split(",").toList() + val realReviewers = EnvReplacementParser.parse( + value = reviewers, + contextMap = variables, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ).split(",").toList() group.reviewers = realReviewers } if (group.groups.isNotEmpty()) { val groups = group.groups.joinToString(",") - val realGroups = EnvReplacementParser.parse(groups, variables, asCodeEnabled) - .split(",").toList() + val realGroups = EnvReplacementParser.parse( + value = groups, + contextMap = variables, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ).split(",").toList() group.groups = realGroups } } - reviewDesc = EnvReplacementParser.parse(reviewDesc, variables, asCodeEnabled) + reviewDesc = EnvReplacementParser.parse( + value = reviewDesc, + contextMap = variables, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ) notifyGroup = notifyGroup?.map { - EnvReplacementParser.parse(it, variables, asCodeEnabled) + EnvReplacementParser.parse( + value = it, + contextMap = variables, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ) }?.toMutableList() reviewParams?.forEach { it.parseValueWithType(variables) } } diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/TemplateInstanceCreateRequest.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/TemplateInstanceCreateRequest.kt index c9bea63c3e8..9d9e8948e1d 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/TemplateInstanceCreateRequest.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/TemplateInstanceCreateRequest.kt @@ -48,5 +48,9 @@ data class TemplateInstanceCreateRequest( @get:Schema(title = "是否为空模板", required = false) var emptyTemplate: Boolean? = false, @get:Schema(title = "静态流水线组", required = false) - var staticViews: List = emptyList() + var staticViews: List = emptyList(), + @get:Schema(title = "是否继承项目流水线语言风格", required = false) + var inheritedDialect: Boolean? = true, + @get:Schema(title = "流水线语言风格", required = false) + var pipelineDialect: String? = null ) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSetting.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSetting.kt index 0b20127f6a8..41c077ce668 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSetting.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSetting.kt @@ -101,7 +101,9 @@ data class PipelineSetting( pipelineId: String, pipelineName: String, maxPipelineResNum: Int? = null, - failSubscription: Subscription? = null + failSubscription: Subscription? = null, + inheritedDialectSetting: Boolean? = null, + pipelineDialectSetting: String? = null ): PipelineSetting { return PipelineSetting( projectId = projectId, @@ -117,7 +119,10 @@ data class PipelineSetting( failSubscription = null, successSubscriptionList = emptyList(), failSubscriptionList = failSubscription?.let { listOf(it) }, - pipelineAsCodeSettings = PipelineAsCodeSettings() + pipelineAsCodeSettings = PipelineAsCodeSettings.initDialect( + inheritedDialect = inheritedDialectSetting, + pipelineDialect = pipelineDialectSetting + ) ) } } diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/ExprReplacementUtil.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/ExprReplacementUtil.kt new file mode 100644 index 00000000000..6a923f71b0d --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/ExprReplacementUtil.kt @@ -0,0 +1,324 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline.utils + +import com.tencent.devops.common.api.exception.VariableNotFoundException +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.expression.ContextNotFoundException +import com.tencent.devops.common.expression.ExecutionContext +import com.tencent.devops.common.expression.ExpressionParseException +import com.tencent.devops.common.expression.ExpressionParser +import com.tencent.devops.common.expression.context.ContextValueNode +import com.tencent.devops.common.expression.context.DictionaryContextData +import com.tencent.devops.common.expression.context.PipelineContextData +import com.tencent.devops.common.expression.context.RuntimeDictionaryContextData +import com.tencent.devops.common.expression.context.RuntimeNamedValue +import com.tencent.devops.common.expression.expression.EvaluationOptions +import com.tencent.devops.common.expression.expression.ExpressionOutput +import com.tencent.devops.common.expression.expression.IFunctionInfo +import com.tencent.devops.common.expression.expression.ParseExceptionKind +import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo +import com.tencent.devops.common.pipeline.ExprReplacementOptions +import org.apache.tools.ant.filters.StringInputStream +import org.slf4j.LoggerFactory +import java.io.BufferedReader +import java.io.InputStreamReader + +@Suppress( + "LoopWithTooManyJumpStatements", + "ComplexCondition", + "ComplexMethod", + "NestedBlockDepth", + "ReturnCount", + "LongParameterList" +) +object ExprReplacementUtil { + private val logger = LoggerFactory.getLogger(ExprReplacementUtil::class.java) + + fun parseExpression(value: String, options: ExprReplacementOptions): String { + with(options) { + return try { + val (executeContext, nameValues) = contextPair + ?: getCustomExecutionContextByMap(contextMap) + ?: return value + parseExpression( + value = value, + context = executeContext, + nameValues = nameValues, + functions = functions, + output = output, + contextNotNull = contextNotNull + ) + } catch (ex: VariableNotFoundException) { + throw ex + } catch (ignore: Throwable) { + logger.warn("[$value]|EnvReplacementParser expression invalid: ", ignore) + value + } + } + } + + fun getCustomExecutionContextByMap( + variables: Map, + extendNamedValueMap: List? = null + ): Pair>? { + try { + val context = ExecutionContext(DictionaryContextData()) + val nameValue = mutableListOf() + extendNamedValueMap?.forEach { namedValue -> + nameValue.add(NamedValueInfo(namedValue.key, ContextValueNode())) + context.expressionValues.add( + namedValue.key, + RuntimeDictionaryContextData(namedValue) + ) + } + ExpressionParser.fillContextByMap(variables, context, nameValue) + return Pair(context, nameValue) + } catch (ignore: Throwable) { + logger.warn("EnvReplacementParser context invalid: $variables", ignore) + return null + } + } + + private fun parseExpression( + value: String, + nameValues: List, + context: ExecutionContext, + functions: Iterable? = null, + output: ExpressionOutput? = null, + contextNotNull: Boolean + ): String { + val strReader = InputStreamReader(StringInputStream(value)) + val bufferReader = BufferedReader(strReader) + val newValue = StringBuilder() + try { + var line = bufferReader.readLine() + while (line != null) { + // 跳过空行和注释行 + val blocks = findExpressions(line) + if (line.isBlank() || blocks.isEmpty()) { + newValue.append(line).append("\n") + line = bufferReader.readLine() + continue + } + val onceResult = parseExpressionLine( + value = line, + blocks = blocks, + context = context, + nameValues = nameValues, + functions = functions, + output = output, + contextNotNull = contextNotNull + ) + + val newLine = findExpressions(onceResult).let { + if (it.isEmpty()) { + onceResult + } else { + parseExpressionLine( + value = onceResult, + blocks = it, + context = context, + nameValues = nameValues, + functions = functions, + output = output, + contextNotNull = contextNotNull + ) + } + } + newValue.append(newLine).append("\n") + line = bufferReader.readLine() + } + } finally { + strReader.close() + bufferReader.close() + } + return newValue.toString().removeSuffix("\n") + } + + /** + * 解析表达式，根据 findExpressions 寻找的括号优先级进行解析 + */ + private fun parseExpressionLine( + value: String, + blocks: List>, + nameValues: List, + context: ExecutionContext, + functions: Iterable? = null, + output: ExpressionOutput? = null, + contextNotNull: Boolean + ): String { + var chars = value.toList() + blocks.forEachIndexed nextBlockLevel@{ blockLevel, blocksInLevel -> + blocksInLevel.forEachIndexed nextBlock@{ blockI, block -> + // 表达式因为含有 ${{ }} 所以起始向后推3位，末尾往前推两位 + val expression = chars.joinToString("").substring(block.startIndex + 3, block.endIndex - 1) + if (expression.isBlank()) return@nextBlock + val options = EvaluationOptions(contextNotNull) + var result = try { + ExpressionParser.createTree(expression, null, nameValues, functions)!! + .evaluate(null, context, options, output).value.let { + if (it is PipelineContextData) it.fetchValue() else it + }?.let { + JsonUtil.toJson(it, false) + } ?: "" + } catch (ignore: ContextNotFoundException) { + throw VariableNotFoundException( + variableKey = options.contextNotNull.errKey() + ) + } catch (ignore: ExpressionParseException) { + if (contextNotNull && ignore.kind == ParseExceptionKind.UnrecognizedNamedValue) { + throw VariableNotFoundException( + variableKey = ignore.expression + ) + } + return@nextBlock + } + + if ((blockLevel + 1 < blocks.size) && + !( + (block.startIndex - 1 >= 0 && chars[block.startIndex - 1] == '.') || + (block.endIndex + 1 < chars.size && chars[block.endIndex + 1] == '.') + ) + ) { + result = "'$result'" + } + + val charList = result.toList() + + // 将替换后的表达式嵌入原本的line + val startSub = if (block.startIndex - 1 < 0) { + listOf() + } else { + chars.slice(0 until block.startIndex) + } + val endSub = if (block.endIndex + 1 >= chars.size) { + listOf() + } else { + chars.slice(block.endIndex + 1 until chars.size) + } + chars = startSub + charList + endSub + + // 将替换后的字符查传递给后边的括号位数 + val diffNum = charList.size - (block.endIndex - block.startIndex + 1) + blocks.forEachIndexed { i, bl -> + bl.forEachIndexed level@{ j, b -> + if (i <= blockLevel && j <= blockI) { + return@level + } + if (blocks[i][j].startIndex > block.endIndex) { + blocks[i][j].startIndex += diffNum + } + if (blocks[i][j].endIndex > block.endIndex) { + blocks[i][j].endIndex += diffNum + } + } + } + } + } + + return chars.joinToString("") + } + + /** + * 寻找语句中包含 ${{}}的表达式的位置，返回成对的位置坐标，并根据优先级排序 + * 优先级算法目前暂定为 从里到外，从左到右 + * @param levelMax 返回的最大层数，从深到浅。默认为2层 + * 例如: 替换顺序如数字所示 ${{ 4 ${{ 2 ${{ 1 }} }} ${{ 3 }} }} + * @return [ 层数次序 [ 括号 ] ] [[1], [2, 3], [4]]]] + */ + private fun findExpressions(condition: String, levelMax: Int = 2): List> { + val stack = ArrayDeque() + var index = 0 + val chars = condition.toCharArray() + val levelMap = mutableMapOf>() + while (index < chars.size) { + if (index + 2 < chars.size && chars[index] == '$' && chars[index + 1] == '{' && chars[index + 2] == '{' + ) { + stack.addLast(index) + index += 3 + continue + } + + if (index + 1 < chars.size && chars[index] == '}' && chars[index + 1] == '}' + ) { + val start = stack.removeLastOrNull() + if (start != null) { + // 栈里剩下几个前括号，这个完整括号的优先级就是多少 + val level = stack.size + 1 + if (levelMap.containsKey(level)) { + levelMap[level]!!.add(ExpressionBlock(start, index + 1)) + } else { + levelMap[level] = mutableListOf(ExpressionBlock(start, index + 1)) + } + } + index += 2 + continue + } + + index++ + } + + if (levelMap.isEmpty()) { + return listOf() + } + + val result = mutableListOf>() + var max = 0 + var listIndex = 0 + run end@{ + levelMap.keys.sortedDescending().forEach result@{ level -> + val blocks = levelMap[level] ?: return@result + blocks.sortBy { it.startIndex } + blocks.forEach { block -> + if (result.size < listIndex + 1) { + result.add(mutableListOf(block)) + } else { + result[listIndex].add(block) + } + } + listIndex++ + max++ + if (max == levelMax) { + return@end + } + } + } + return result + } + + /** + * 表达式括号项 ${{ }} + * @param startIndex 括号开始位置即 $ 位置 + * @param endIndex 括号结束位置即最后一个 } 位置 + */ + data class ExpressionBlock( + var startIndex: Int, + var endIndex: Int + ) +} diff --git a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt index fda0762b9c4..d41c80b7b5b 100644 --- a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt +++ b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/EnvReplacementParserTest.kt @@ -346,10 +346,19 @@ internal class EnvReplacementParserTest { val command8 = "echo \${{ variables.hello }}" val command9 = "echo \${{ ci.workspace }}" + val command10 = mutableMapOf( + "params" to mutableListOf( + mutableMapOf( + "key" to "instance", + "value" to "\${{variables.instance}}" + ) + ) + ) val data = mapOf( "variables.abc" to "variables.value", - "variables.hello" to "hahahahaha" + "variables.hello" to "hahahahaha", + "variables.instance" to "{\"instances\":[{\"cluster\":\"ci-prod\",\"pod\":\"ci-123\"}]}" ) // 与EnvUtils的差异点：不支持传可空对象 // Assertions.assertEquals("", EnvReplacementParser.parse(null, data)) @@ -396,6 +405,13 @@ internal class EnvReplacementParserTest { onlyExpression = true ) ) + val command10Expected = """ + {"params":[{"key":"instance","value":"{\"instances\":[{\"cluster\":\"ci-prod\",\"pod\":\"ci-123\"}]}"}]} + """.trimIndent() + Assertions.assertEquals( + command10Expected, + EnvReplacementParser.parse(command10, data, true) + ) } @Test @@ -559,4 +575,52 @@ echo true""" println("template=$template\nreplaced=$buff\n") Assertions.assertEquals(expect, buff) } + + @Test + fun containsExpressions() { + val command = "{\"age\": \${{age}} , \"sex\": \"boy\", \"name\": \${{name}}}" + Assertions.assertTrue(EnvReplacementParser.containsExpressions(command)) + + val command1 = "hello \${{variables.abc}} world" + Assertions.assertTrue(EnvReplacementParser.containsExpressions(command1)) + + val command2 = "\${{variables.abc}}world" + Assertions.assertTrue(EnvReplacementParser.containsExpressions(command2)) + + val command3 = "hello\${{variables.abc}}" + Assertions.assertTrue(EnvReplacementParser.containsExpressions(command3)) + + val command4 = "hello\${{variables.abc" + Assertions.assertFalse(EnvReplacementParser.containsExpressions(command4)) + + val command5 = "hello\${{variables.abc}" + Assertions.assertFalse(EnvReplacementParser.containsExpressions(command5)) + + val command6 = "hello\${variables.abc}}" + Assertions.assertFalse(EnvReplacementParser.containsExpressions(command6)) + + val command7 = "hello\$variables.abc}}" + Assertions.assertFalse(EnvReplacementParser.containsExpressions(command7)) + + val command8 = "echo \${{ variables.hello }}" + Assertions.assertTrue(EnvReplacementParser.containsExpressions(command8)) + + val command9 = "echo \${{ ci.workspace }} || \${{variables.hello}}" + Assertions.assertTrue(EnvReplacementParser.containsExpressions(command9)) + + val command10 = "echo \${{ ci.xyz == 'zzzz' }}" + Assertions.assertTrue(EnvReplacementParser.containsExpressions(command10)) + + val command11 = "echo \${{ variables.xyz == 'zzzz' }}" + Assertions.assertTrue(EnvReplacementParser.containsExpressions(command11)) + + val command12 = "echo \${{ strToTime(variables.date) > strToTime('2023-03-16 12:06:21') }}" + Assertions.assertTrue(EnvReplacementParser.containsExpressions(command12)) + + val command13 = "echo \${{ strToTime(variables.date) > strToTime('2023-03-14 12:06:21') }}" + Assertions.assertTrue(EnvReplacementParser.containsExpressions(command13)) + + val command14 = "\${{ strToTime(\${{variables.date}}) > strToTime('2023-03-14 12:06:21') }}" + Assertions.assertTrue(EnvReplacementParser.containsExpressions(command14)) + } } diff --git a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtilTest.kt b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtilTest.kt new file mode 100644 index 00000000000..f6a11c54c0a --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/ExprReplaceEnvVarUtilTest.kt @@ -0,0 +1,319 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline + +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.api.util.JsonUtil.toJson +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +@Suppress("ALL", "UNCHECKED_CAST") +class ExprReplaceEnvVarUtilTest { + + private val envMap: MutableMap = HashMap() + + @BeforeEach + fun setup() { + envMap["normalStrEnvVar"] = "123" + envMap["specStrEnvVar"] = "D:\\tmp\\hha" + envMap["jsonStrEnvVar"] = "{\"abc\":\"123\"}" + } + + private val lineSeparator = System.getProperty("line.separator") + private val jsonExcept = "{$lineSeparator" + + " \"abc\" : \"变量替换测试_{\\\"abc\\\":\\\"123\\\"}\"$lineSeparator" + + "}" + + private val arrayJsonExcept = "[ \"变量替换测试_{\\\"abc\\\":\\\"123\\\"}\" ]" + + @Test + fun replaceList() { + val testBean = TestBean( + testBeanKey = "bean变量替换测试_\${{specStrEnvVar}}", + testBeanValue = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + ) + // 对list对象进行变量替换 + val originDataListObj = ArrayList() + originDataListObj.add("变量替换测试_\${{normalStrEnvVar}}") + originDataListObj.add("变量替换测试_\${{specStrEnvVar}}") + originDataListObj.add("变量替换测试_\${{jsonStrEnvVar}}") + originDataListObj.add("{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}") + originDataListObj.add("[\"变量替换测试_\${{jsonStrEnvVar}}\"]") + originDataListObj.add(testBean) + val dataMapObj: MutableMap = HashMap() + dataMapObj["dataMapKey"] = "变量替换测试_\${{specStrEnvVar}}" + dataMapObj["testBean"] = testBean + originDataListObj.add(dataMapObj) + val convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(originDataListObj, envMap) as List<*> + + assertEquals("变量替换测试_${envMap["normalStrEnvVar"]}", convertDataObj[0]) + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", convertDataObj[1]) + assertEquals("变量替换测试_${envMap["jsonStrEnvVar"]}", convertDataObj[2]) + assertEquals(jsonExcept, convertDataObj[3]) + assertEquals(arrayJsonExcept, convertDataObj[4]) + + val convertTestBean = convertDataObj[5] as TestBean + assertEquals("bean变量替换测试_${envMap["specStrEnvVar"]}", convertTestBean.testBeanKey) + assertEquals(jsonExcept, convertTestBean.testBeanValue) + } + + @Test + fun replaceIllegalJson() { + val objectJson = "{\"abc:\"变量替换测试_\${{normalStrEnvVar}}\"" + val convertDataObj1 = ExprReplaceEnvVarUtil.replaceEnvVar(objectJson, envMap) + println(convertDataObj1) + assertEquals("{\"abc:\"变量替换测试_${envMap["normalStrEnvVar"]}\"", convertDataObj1) + + val arrayJson = "[1, \"变量替换测试_\${{normalStrEnvVar}}\"" + val convertDataObj2 = ExprReplaceEnvVarUtil.replaceEnvVar(arrayJson, envMap) + println(convertDataObj2) + assertEquals("[1, \"变量替换测试_${envMap["normalStrEnvVar"]}\"", convertDataObj2) + } + + @Test + fun replaceSet() { + + val testBean = TestBean( + testBeanKey = "bean变量替换测试_\${{specStrEnvVar}}", + testBeanValue = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + ) + // 对set对象进行变量替换 + val originDataSetObj = HashSet() + originDataSetObj.add("1变量替换测试_\${{normalStrEnvVar}}") + originDataSetObj.add("2变量替换测试_\${{specStrEnvVar}}") + originDataSetObj.add("3变量替换测试_\${{jsonStrEnvVar}}") + originDataSetObj.add("{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}") + originDataSetObj.add("[\"变量替换测试_\${{jsonStrEnvVar}}\"]") + originDataSetObj.add(testBean) + + val setDataMapObj: MutableMap = HashMap() + setDataMapObj["dataMapKey"] = "变量替换测试_\${{specStrEnvVar}}" + setDataMapObj["testBean"] = testBean + originDataSetObj.add(setDataMapObj) + val convertDataObj = (ExprReplaceEnvVarUtil.replaceEnvVar(originDataSetObj, envMap) as Set<*>) + + convertDataObj.forEach { member -> + when { + member is Map<*, *> -> { + member.forEach { sm -> + when { + sm.key.toString() == "testBean" -> { + assertEquals( + "bean变量替换测试_${envMap["specStrEnvVar"]}", + (sm.value as TestBean).testBeanKey + ) + assertEquals(jsonExcept, (sm.value as TestBean).testBeanValue) + } + sm.key.toString() == "dataMapKey" -> { + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", sm.value) + } + else -> { + assertEquals(member.toString(), "setDataMapObj") + } + } + } + } + member is TestBean -> { + assertEquals("bean变量替换测试_${envMap["specStrEnvVar"]}", member.testBeanKey) + assertEquals(jsonExcept, member.testBeanValue) + } + member.toString().startsWith("1") -> { + assertEquals("1变量替换测试_${envMap["normalStrEnvVar"]}", member) + } + member.toString().startsWith("2") -> { + assertEquals("2变量替换测试_${envMap["specStrEnvVar"]}", member) + } + member.toString().startsWith("3") -> { + assertEquals("3变量替换测试_${envMap["jsonStrEnvVar"]}", member) + } + member.toString().startsWith("{") -> { + assertEquals(jsonExcept, member) + } + member.toString().startsWith("[") -> { + assertEquals(arrayJsonExcept, member) + } + else -> { + assertEquals(member.toString(), "convertDataObj") + } + } + } + } + + @Test + fun replaceMapWithTestBean() { + // 对map对象进行变量替换 + val originDataMapObj: MutableMap = HashMap() + originDataMapObj["normalStrEnvVarKey"] = "变量替换测试_\${{normalStrEnvVar}}" + originDataMapObj["specStrEnvVarKey"] = "变量替换测试_\${{specStrEnvVar}}" + originDataMapObj["jsonStrEnvVarKey1"] = "变量替换测试_\${{jsonStrEnvVar}}" + originDataMapObj["jsonStrEnvVarKey2"] = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + originDataMapObj["jsonStrEnvVarKey3"] = "\${{jsonStrEnvVar}}" + var originSubDataMapObj: MutableMap? = HashMap() + originSubDataMapObj!!["normalStrEnvVarKey"] = "变量替换测试_\${{normalStrEnvVar}}" + originSubDataMapObj["specStrEnvVarKey"] = "变量替换测试_\${{specStrEnvVar}}" + originSubDataMapObj["jsonStrEnvVarKey1"] = "变量替换测试_\${{jsonStrEnvVar}}" + originSubDataMapObj["jsonStrEnvVarKey2"] = "\${{jsonStrEnvVar}}" + + val testBean = TestBean( + testBeanKey = "变量替换测试_\${{specStrEnvVar}}", + testBeanValue = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + ) + originSubDataMapObj["testBean"] = testBean + originDataMapObj["originSubDataMapObj"] = originSubDataMapObj + + val cpb = ExprReplaceEnvVarUtil.replaceEnvVar(originDataMapObj, envMap) + val testBeanMap = ((cpb as Map)["originSubDataMapObj"] as Map)["testBean"] as TestBean + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", testBeanMap.testBeanKey) + assertEquals(jsonExcept, testBeanMap.testBeanValue) + // 判断map中jsonStrEnvVarKey3对应的值进行变量替换后能否正常转换为json串 + assertEquals(envMap["jsonStrEnvVar"], (cpb as Map)["jsonStrEnvVarKey3"]!!) + originSubDataMapObj = cpb["originSubDataMapObj"] as MutableMap? + // 判断嵌套的map中jsonStrEnvVarKey2对应的值进行变量替换后能否正常转换为json串 + assertEquals(envMap["jsonStrEnvVar"], originSubDataMapObj!!["jsonStrEnvVarKey2"]!!) + } + + @Test + fun replaceTestComplexBean() { + // 对普通的javaBean对象进行转换 + val testComplexBean = TestComplexBean() + testComplexBean.testBeanKey = "变量替换测试_\${{specStrEnvVar}}" + testComplexBean.testBeanValue = "[\"变量替换测试_\${{jsonStrEnvVar}}\"]" + + val dataList = ArrayList() + dataList.add("变量替换测试_\${{normalStrEnvVar}}") + dataList.add("变量替换测试_\${{specStrEnvVar}}") + dataList.add("变量替换测试_\${{jsonStrEnvVar}}") + dataList.add("{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}") + dataList.add("[\"变量替换测试_\${{jsonStrEnvVar}}\"]") + testComplexBean.dataList = dataList + + var dataMap: MutableMap = HashMap() + dataMap["normalStrEnvVarKey"] = " 变量替换测试_\${{normalStrEnvVar}} " + dataMap["specStrEnvVarKey"] = "变量替换测试_\${{specStrEnvVar}}" + dataMap["jsonStrEnvVarKey1"] = "变量替换测试_\${{jsonStrEnvVar}}" + dataMap["jsonStrEnvVarKey2"] = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + dataMap["jsonStrEnvVarKey3"] = "[\"变量替换测试_\${{jsonStrEnvVar}}\"]" + val subDataMap: MutableMap = HashMap() + subDataMap["normalStrEnvVarKey"] = "变量替换测试_\${{normalStrEnvVar}}" + subDataMap["specStrEnvVarKey"] = "变量替换测试_\${{specStrEnvVar}}" + subDataMap["jsonStrEnvVarKey1"] = "变量替换测试_\${{jsonStrEnvVar}}" + subDataMap["jsonStrEnvVarKey2"] = "{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}" + + val testBean = TestBean( + testBeanKey = "bean变量替换测试_\${{specStrEnvVar}}", + testBeanValue = "{\"abc\":\"bean变量替换测试_\${{jsonStrEnvVar}}\"}" + ) + subDataMap["testBean"] = testBean + dataMap["subDataMap"] = subDataMap + testComplexBean.dataMap = dataMap + + val dataSet = HashSet() + dataSet.add("变量替换测试_\${{normalStrEnvVar}}") + dataSet.add("变量替换测试_\${{specStrEnvVar}}") + dataSet.add("变量替换测试_\${{jsonStrEnvVar}}") + dataSet.add("{\"abc\":\"变量替换测试_\${{jsonStrEnvVar}}\"}") + dataSet.add("[\"变量替换测试_\${{jsonStrEnvVar}}\"]") + testComplexBean.dataSet = dataSet + + // start to test + var convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(testComplexBean, envMap) + val convertBean = convertDataObj as TestComplexBean + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", convertBean.testBeanKey) + + assertEquals("变量替换测试_${envMap["normalStrEnvVar"]}", convertBean.dataList!![0]) + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", convertBean.dataList!![1]) + assertEquals("变量替换测试_${envMap["jsonStrEnvVar"]}", convertBean.dataList!![2]) + assertEquals(jsonExcept, convertBean.dataList!![3]) + assertEquals("[ \"变量替换测试_{\\\"abc\\\":\\\"123\\\"}\" ]", convertBean.dataList!![4]) + + assertEquals(" 变量替换测试_${envMap["normalStrEnvVar"]} ", convertBean.dataMap!!["normalStrEnvVarKey"]) + assertEquals("变量替换测试_${envMap["specStrEnvVar"]}", convertBean.dataMap!!["specStrEnvVarKey"]) + assertEquals("变量替换测试_${envMap["jsonStrEnvVar"]}", convertBean.dataMap!!["jsonStrEnvVarKey1"]) + assertEquals(jsonExcept, convertBean.dataMap!!["jsonStrEnvVarKey2"]) + assertEquals(arrayJsonExcept, convertBean.dataMap!!["jsonStrEnvVarKey3"]) + + // 替换包含null的对象 + dataMap = HashMap() + dataMap["key1"] = "变量" + dataMap["key2"] = arrayOf(null, "哈哈") + + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(dataMap, envMap) as Map<*, *> + assertEquals(dataMap["key1"], convertDataObj["key1"]) + assertEquals(toJson(dataMap["key2"]!!), convertDataObj["key2"]) + println("convertDataObj=$convertDataObj") + } + + @Test + fun replaceEnvVar() { + + // 对普通字符串进行普通字符串变量替换 + var originDataObj: Any = "变量替换测试_\${{normalStrEnvVar}}" + var convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap) + assertEquals("变量替换测试_123", toJson(convertDataObj)) + + // 对普通字符串进行带特殊字符字符串变量替换 + originDataObj = "变量替换测试_\${{specStrEnvVar}}" + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap) + assertEquals("变量替换测试_D:\\tmp\\hha", toJson(convertDataObj)) + + // 对普通字符串进行json字符串变量替换 + originDataObj = "变量替换测试_\${{jsonStrEnvVar}}" + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap) + assertEquals("变量替换测试_{\"abc\":\"123\"}", toJson(convertDataObj)) + + // number类型变量替换 + originDataObj = "[1,2,3]" + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar(originDataObj, envMap) + println(toJson(convertDataObj)) + assertEquals(toJson(JsonUtil.to(originDataObj, List::class.java)), toJson(convertDataObj)) + + // 魔法数字符创测试 + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar("12E2", envMap) + assertEquals("12E2", toJson(convertDataObj)) + // 替换”[133]-[sid-${normalStrEnvVar}]-[sid-zhiliang-test1]“带多个[]的字符串 + convertDataObj = ExprReplaceEnvVarUtil.replaceEnvVar( + "[133]-[sid-\${{normalStrEnvVar}}]-[sid-zhiliang-test1]", + envMap + ) + assertEquals("[133]-[sid-123]-[sid-zhiliang-test1]", toJson(convertDataObj)) + } + + internal data class TestBean( + var testBeanKey: String? = null, + var testBeanValue: String? = null + ) + + internal data class TestComplexBean( + var testBeanKey: String? = null, + var testBeanValue: String? = null, + var dataList: List<*>? = null, + var dataMap: Map<*, *>? = null, + var dataSet: Set<*>? = null + ) +} diff --git a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/dialect/PipelineDialectUtilTest.kt b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/dialect/PipelineDialectUtilTest.kt new file mode 100644 index 00000000000..c3878111cea --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/dialect/PipelineDialectUtilTest.kt @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline.dialect + +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test + +class PipelineDialectUtilTest { + + @Test + fun testGetDialect() { + val actual = PipelineDialectUtil.getPipelineDialect(null, null, null) + Assertions.assertEquals(PipelineDialectType.CLASSIC.dialect, actual) + + val actual2 = PipelineDialectUtil.getPipelineDialect( + inheritedDialect = null, + projectDialect = PipelineDialectType.CLASSIC.name, + pipelineDialect = null + ) + Assertions.assertEquals(PipelineDialectType.CLASSIC.dialect, actual2) + + val actual3 = PipelineDialectUtil.getPipelineDialect( + inheritedDialect = true, + projectDialect = null, + pipelineDialect = null + ) + Assertions.assertEquals(PipelineDialectType.CLASSIC.dialect, actual3) + + val actual4 = PipelineDialectUtil.getPipelineDialect( + inheritedDialect = true, + projectDialect = null, + pipelineDialect = PipelineDialectType.CONSTRAINED.name + ) + Assertions.assertEquals(PipelineDialectType.CLASSIC.dialect, actual4) + + val actual5 = PipelineDialectUtil.getPipelineDialect( + inheritedDialect = false, + projectDialect = null, + pipelineDialect = PipelineDialectType.CONSTRAINED.name + ) + Assertions.assertEquals(PipelineDialectType.CONSTRAINED.dialect, actual5) + + val actual6 = PipelineDialectUtil.getPipelineDialect( + inheritedDialect = true, + projectDialect = PipelineDialectType.CLASSIC.name, + pipelineDialect = PipelineDialectType.CONSTRAINED.name + ) + Assertions.assertEquals(PipelineDialectType.CLASSIC.dialect, actual6) + + val actual7 = PipelineDialectUtil.getPipelineDialect( + inheritedDialect = false, + projectDialect = PipelineDialectType.CLASSIC.name, + pipelineDialect = PipelineDialectType.CONSTRAINED.name + ) + Assertions.assertEquals(PipelineDialectType.CONSTRAINED.dialect, actual7) + } +} diff --git a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/pojo/element/ElementTest.kt b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/pojo/element/ElementTest.kt index 4fde8155858..9aa331569ad 100644 --- a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/pojo/element/ElementTest.kt +++ b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/pojo/element/ElementTest.kt @@ -51,7 +51,7 @@ class ElementTest { @Test fun testElementJsonOrder() { val jsonFile = ElementTest::class.java.classLoader.getResource("windowsElement.json") - val expected = jsonFile!!.readText().trim('\n') + val expected = jsonFile!!.readText().trim() val wel = WindowsScriptElement( id = "e-326ce1c320204980a3d2a0f241bccd63", name = "batch script", diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt index 591e64cdbc8..a7ca239c2de 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt @@ -361,6 +361,7 @@ object ProcessMessageCode { const val ERROR_PIPELINE_CONDITION_EXPRESSION_TOO_LONG = "2101253" // 自定义条件表达式{0}的长度超过{1}位 const val ERROR_PIPELINE_BUILD_START_PARAM_NO_EMPTY = "2101254" // 构建启动参数如果必填，不能为空 const val ERROR_REPEATEDLY_START_VM = "2101255" // 重复启动构建机，当前构建机的状态为：{0} + const val ERROR_PIPELINE_VARIABLES_OUT_OF_LENGTH = "2101256" // 流水线启动参数{0}超出4000长度限制 const val BK_SUCCESSFULLY_DISTRIBUTED = "bkSuccessfullyDistributed" // 跨项目构件分发成功，共分发了{0}个文件 const val BK_SUCCESSFULLY_FAILED = "bkSuccessfullyFailed" // 跨项目构件分发失败， diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/setting/PipelineSettingVersion.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/setting/PipelineSettingVersion.kt index efcdadabd78..8f1dc7aa6ce 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/setting/PipelineSettingVersion.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/setting/PipelineSettingVersion.kt @@ -27,6 +27,7 @@ package com.tencent.devops.process.pojo.setting +import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting import com.tencent.devops.common.pipeline.pojo.setting.Subscription @@ -73,7 +74,9 @@ data class PipelineSettingVersion( @get:Schema(title = "并发时,是否相同group取消正在执行的流水线", required = false) var concurrencyCancelInProgress: Boolean?, @get:Schema(title = "并发构建数量限制", required = false) - var maxConRunningQueueSize: Int? = null // MULTIPLE类型时，并发构建数量限制 + var maxConRunningQueueSize: Int? = null, // MULTIPLE类型时，并发构建数量限制 + @get:Schema(title = "YAML流水线特殊配置", required = false) + var pipelineAsCodeSettings: PipelineAsCodeSettings? = null ) { companion object { @@ -93,7 +96,8 @@ data class PipelineSettingVersion( buildNumRule = setting.buildNumRule, concurrencyCancelInProgress = setting.concurrencyCancelInProgress, concurrencyGroup = setting.concurrencyGroup, - maxConRunningQueueSize = setting.maxConRunningQueueSize + maxConRunningQueueSize = setting.maxConRunningQueueSize, + pipelineAsCodeSettings = setting.pipelineAsCodeSettings ) } } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt index db7dd3584b0..e682c48f61f 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt @@ -124,6 +124,7 @@ const val PIPELINE_ATOM_VERSION = "BK_CI_ATOM_VERSION" // "流水线插件版本 const val PIPELINE_TASK_NAME = "BK_CI_TASK_NAME" // "流水线任务名称（步骤名称）" const val PIPELINE_STEP_ID = "BK_CI_STEP_ID" // "用户自定义ID（上下文标识）" const val PIPELINE_ATOM_TIMEOUT = "BK_CI_ATOM_TIMEOUT" // "流水线插件超时时间" +const val PIPELINE_DIALECT = "BK_CI_PIPELINE_DIALECT" // 流水线语法风格 /** * 自定义触发材料 */ diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingDao.kt index a27fe282702..cf09bf1e3fb 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingDao.kt @@ -92,7 +92,8 @@ class PipelineSettingDao { CLEAN_VARIABLES_WHEN_RETRY, SUCCESS_SUBSCRIPTION, FAILURE_SUBSCRIPTION, - VERSION + VERSION, + PIPELINE_AS_CODE_SETTINGS ).values( setting.projectId, setting.pipelineName, @@ -126,7 +127,8 @@ class PipelineSettingDao { setting.cleanVariablesWhenRetry, JsonUtil.toJson(successSubscriptionList, false), JsonUtil.toJson(failSubscriptionList, false), - setting.version + setting.version, + setting.pipelineAsCodeSettings?.let { JsonUtil.toJson(it, false) } ).onDuplicateKeyUpdate() .set(NAME, setting.pipelineName) .set(DESC, setting.desc) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingVersionDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingVersionDao.kt index 0991b83670a..c72ad643261 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingVersionDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingVersionDao.kt @@ -29,6 +29,7 @@ package com.tencent.devops.process.dao import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.module.kotlin.readValue +import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings import com.tencent.devops.common.api.util.DateTimeUtil import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType @@ -114,6 +115,9 @@ class PipelineSettingVersionDao { .set(SUCCESS_SUBSCRIPTION, JsonUtil.toJson(successSubscriptionList, false)) .set(FAILURE_SUBSCRIPTION, JsonUtil.toJson(failSubscriptionList, false)) .set(MAX_CON_RUNNING_QUEUE_SIZE, setting.maxConRunningQueueSize ?: -1) + .set(PIPELINE_AS_CODE_SETTINGS, setting.pipelineAsCodeSettings?.let { self -> + JsonUtil.toJson(self, false) + }) .execute() } } @@ -223,7 +227,10 @@ class PipelineSettingVersionDao { buildNumRule = t.buildNumRule, concurrencyCancelInProgress = t.concurrencyCancelInProgress, concurrencyGroup = t.concurrencyGroup, - maxConRunningQueueSize = t.maxConRunningQueueSize + maxConRunningQueueSize = t.maxConRunningQueueSize, + pipelineAsCodeSettings = t.pipelineAsCodeSettings?.let { self -> + JsonUtil.to(self, PipelineAsCodeSettings::class.java) + } ) } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt index 90112cc805b..c99c41322ae 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt @@ -37,6 +37,7 @@ import com.tencent.devops.common.pipeline.container.NormalContainer import com.tencent.devops.common.pipeline.container.Stage import com.tencent.devops.common.pipeline.container.TriggerContainer import com.tencent.devops.common.pipeline.container.VMBuildContainer +import com.tencent.devops.common.pipeline.dialect.IPipelineDialect import com.tencent.devops.common.pipeline.enums.JobRunCondition import com.tencent.devops.common.pipeline.enums.StageRunCondition import com.tencent.devops.common.pipeline.extend.ModelCheckPlugin @@ -90,7 +91,8 @@ open class DefaultModelCheckPlugin constructor( projectId: String?, userId: String, isTemplate: Boolean, - oauthUser: String? + oauthUser: String?, + pipelineDialect: IPipelineDialect? ): Int { var metaSize = 0 // 检查流水线名称 @@ -122,7 +124,10 @@ open class DefaultModelCheckPlugin constructor( val trigger = stages.getOrNull(0) ?: throw ErrorCodeException(errorCode = ProcessMessageCode.ERROR_PIPELINE_MODEL_NEED_JOB) // 检查触发容器 - val paramsMap = checkTriggerContainer(trigger) + val paramsMap = checkTriggerContainer( + trigger = trigger, + supportChineseVarName = pipelineDialect?.supportChineseVarName() + ) val contextMap = PipelineVarUtil.fillVariableMap(paramsMap.mapValues { it.value.defaultValue.toString() }) val elementCnt = mutableMapOf() val containerCnt = mutableMapOf() @@ -474,7 +479,10 @@ open class DefaultModelCheckPlugin constructor( } } - open fun checkTriggerContainer(trigger: Stage): Map { + open fun checkTriggerContainer( + trigger: Stage, + supportChineseVarName: Boolean? + ): Map { if (trigger.containers.size != 1) { logger.warn("The trigger stage contain more than one container (${trigger.containers.size})") throw ErrorCodeException( @@ -484,7 +492,11 @@ open class DefaultModelCheckPlugin constructor( val triggerContainer = (trigger.containers.getOrNull(0) ?: throw ErrorCodeException( errorCode = ProcessMessageCode.ERROR_PIPELINE_MODEL_NEED_JOB )) as TriggerContainer - return PipelineUtils.checkPipelineParams(triggerContainer.params) + return if (supportChineseVarName != false) { + triggerContainer.params.associateBy { it.id } + } else { + PipelineUtils.checkPipelineParams(triggerContainer.params) + } } companion object { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt index 5968fda12d2..6309eff2f9b 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt @@ -48,6 +48,7 @@ import com.tencent.devops.common.pipeline.container.NormalContainer import com.tencent.devops.common.pipeline.container.Stage import com.tencent.devops.common.pipeline.container.TriggerContainer import com.tencent.devops.common.pipeline.container.VMBuildContainer +import com.tencent.devops.common.pipeline.dialect.IPipelineDialect import com.tencent.devops.common.pipeline.enums.BranchVersionAction import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.enums.PipelineInstanceTypeEnum @@ -233,7 +234,10 @@ class PipelineRepositoryService constructor( versionStatus: VersionStatus? = VersionStatus.RELEASED, branchName: String? = null, description: String? = null, - yamlInfo: PipelineYamlVo? = null + yamlInfo: PipelineYamlVo? = null, + inheritedDialectSetting: Boolean? = null, + pipelineDialectSetting: String? = null, + pipelineDialect: IPipelineDialect? = null ): DeployPipelineResult { // 生成流水线ID,新流水线以p-开头，以区分以前旧数据 @@ -247,7 +251,8 @@ class PipelineRepositoryService constructor( create = create, versionStatus = versionStatus, channelCode = channelCode, - yamlInfo = yamlInfo + yamlInfo = yamlInfo, + pipelineDialect = pipelineDialect ) val triggerContainer = model.getTriggerContainer() val buildNo = triggerContainer.buildNo?.apply { @@ -313,7 +318,9 @@ class PipelineRepositoryService constructor( versionStatus = versionStatus, branchName = branchName, description = description, - baseVersion = baseVersion + baseVersion = baseVersion, + inheritedDialectSetting = inheritedDialectSetting, + pipelineDialectSetting = pipelineDialectSetting ) } operationLogService.addOperationLog( @@ -342,13 +349,15 @@ class PipelineRepositoryService constructor( create: Boolean = true, versionStatus: VersionStatus? = VersionStatus.RELEASED, channelCode: ChannelCode, - yamlInfo: PipelineYamlVo? = null + yamlInfo: PipelineYamlVo? = null, + pipelineDialect: IPipelineDialect? = null ): List { val metaSize = modelCheckPlugin.checkModelIntegrity( model = model, projectId = projectId, userId = userId, - oauthUser = getPipelineOauthUser(projectId, pipelineId) + oauthUser = getPipelineOauthUser(projectId, pipelineId), + pipelineDialect = pipelineDialect ) // 去重id val distinctIdSet = HashSet(metaSize, 1F /* loadFactor */) @@ -637,7 +646,9 @@ class PipelineRepositoryService constructor( templateId: String? = null, versionStatus: VersionStatus? = VersionStatus.RELEASED, branchName: String?, - description: String? + description: String?, + inheritedDialectSetting: Boolean? = true, + pipelineDialectSetting: String? = null ): DeployPipelineResult { // #8161 如果只有一个草稿版本的创建操作，流水线状态也为仅有草稿 val modelVersion = 1 @@ -703,7 +714,13 @@ class PipelineRepositoryService constructor( content = NotifyTemplateUtils.getCommonShutdownFailureContent() ).takeIf { failType.isNotEmpty() } PipelineSetting.defaultSetting( - projectId, pipelineId, model.name, maxPipelineResNum, failSubscription + projectId = projectId, + pipelineId = pipelineId, + pipelineName = model.name, + maxPipelineResNum = maxPipelineResNum, + failSubscription = failSubscription, + inheritedDialectSetting = inheritedDialectSetting, + pipelineDialectSetting = pipelineDialectSetting ) } @@ -743,7 +760,10 @@ class PipelineRepositoryService constructor( } setting.labels = labels } - setting.pipelineAsCodeSettings = PipelineAsCodeSettings() + setting.pipelineAsCodeSettings = PipelineAsCodeSettings.initDialect( + inheritedDialect = inheritedDialectSetting, + pipelineDialect = pipelineDialectSetting + ) newSetting = setting } // 如果不需要覆盖模板内容，则直接保存传值或默认值 @@ -1677,11 +1697,13 @@ class PipelineRepositoryService constructor( fun getSettingByPipelineVersion( projectId: String, pipelineId: String, - pipelineVersion: Int + pipelineVersion: Int? ): PipelineSetting? { - val resource = pipelineResourceVersionDao.getPipelineVersionSimple( - dslContext, projectId, pipelineId, pipelineVersion - ) + val resource = pipelineVersion?.let { + pipelineResourceVersionDao.getPipelineVersionSimple( + dslContext, projectId, pipelineId, pipelineVersion + ) + } return resource?.settingVersion?.let { pipelineSettingVersionService.getPipelineSetting( projectId = projectId, @@ -1739,6 +1761,7 @@ class PipelineRepositoryService constructor( isTemplate: Boolean ): PipelineName { var oldName: String = setting.pipelineName + setting.pipelineAsCodeSettings?.resetDialect() (context ?: dslContext).transaction { t -> val transactionContext = DSL.using(t) val old = pipelineSettingDao.getSetting( diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt index fabf2abe690..635c688e8ab 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt @@ -44,9 +44,13 @@ import com.tencent.devops.common.event.enums.ActionType import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildStatusBroadCastEvent import com.tencent.devops.common.log.utils.BuildLogPrinter import com.tencent.devops.common.pipeline.EnvReplacementParser +import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.NameAndValue +import com.tencent.devops.common.pipeline.container.Container import com.tencent.devops.common.pipeline.container.NormalContainer +import com.tencent.devops.common.pipeline.container.Stage import com.tencent.devops.common.pipeline.container.VMBuildContainer +import com.tencent.devops.common.pipeline.dialect.PipelineDialectUtil import com.tencent.devops.common.pipeline.enums.BuildFormPropertyType import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.BuildTaskStatus @@ -71,6 +75,7 @@ import com.tencent.devops.process.engine.control.BuildingHeartBeatUtils import com.tencent.devops.process.engine.control.ControlUtils import com.tencent.devops.process.engine.control.lock.ContainerIdLock import com.tencent.devops.process.engine.pojo.BuildInfo +import com.tencent.devops.process.engine.pojo.PipelineBuildContainer import com.tencent.devops.process.engine.pojo.PipelineBuildTask import com.tencent.devops.process.engine.pojo.UpdateTaskInfo import com.tencent.devops.process.engine.pojo.builds.CompleteTask @@ -98,6 +103,7 @@ import com.tencent.devops.process.service.PipelineAsCodeService import com.tencent.devops.process.service.PipelineContextService import com.tencent.devops.process.util.TaskUtils import com.tencent.devops.process.utils.PIPELINE_BUILD_REMARK +import com.tencent.devops.process.utils.PIPELINE_DIALECT import com.tencent.devops.process.utils.PIPELINE_ELEMENT_ID import com.tencent.devops.process.utils.PIPELINE_VMSEQ_ID import com.tencent.devops.process.utils.PipelineVarUtil @@ -193,8 +199,9 @@ class EngineVMBuildService @Autowired(required = false) constructor( val variables = buildVariableService.getAllVariable(projectId, buildInfo.pipelineId, buildId) val variablesWithType = buildVariableService.getAllVariableWithType(projectId, buildId).toMutableList() val model = containerBuildDetailService.getBuildModel(projectId, buildId) + // TODO 没有升级的worker还需要用到这个变量，下一版删除 val asCodeSettings = pipelineAsCodeService.getPipelineAsCodeSettings( - projectId, buildInfo.pipelineId, buildId, buildInfo + projectId = projectId, pipelineId = buildInfo.pipelineId ) Preconditions.checkNotNull(model, NotFoundException("Build Model ($buildId) is not exist")) @@ -224,72 +231,20 @@ class EngineVMBuildService @Autowired(required = false) constructor( params = arrayOf(c.startVMStatus ?: "") ) ) - val containerAppResource = client.get(ServiceContainerAppResource::class) - // #4518 填充构建机环境变量、构建上下文、获取超时时间 - val (containerEnv, context, timeoutMills) = when (c) { - is VMBuildContainer -> { - val envList = mutableListOf() - val tm = transMinuteTimeoutToMills(container.controlOption.jobControlOption.timeout) - val contextMap = variables.plus( - pipelineContextService.buildContext( - projectId = projectId, pipelineId = pipelineId, buildId = buildId, - stageId = s.id!!, containerId = c.id!!, taskId = null, - variables = variables, model = model, executeCount = buildInfo.executeCount - ) - ).toMutableMap() - fillContainerContext(contextMap, c.customEnv, c.matrixContext, asCodeSettings?.enable) - val asCodeEnabled = asCodeSettings?.enable == true - val contextPair = if (asCodeEnabled) { - EnvReplacementParser.getCustomExecutionContextByMap(contextMap) - } else null - c.buildEnv?.forEach { env -> - containerAppResource.getBuildEnv( - name = env.key, - version = EnvReplacementParser.parse( - value = env.value, - contextMap = contextMap, - onlyExpression = asCodeEnabled, - contextPair = contextPair - ), - os = c.baseOS.name.lowercase() - ).data?.let { self -> envList.add(self) } - } - - // 设置Job环境变量customEnv到variablesWithType和variables中 - // TODO 此处应收敛到variablesWithType或variables的其中一个 - val customBuildParameters = mutableListOf() - c.customEnv?.forEach { nameAndValue -> - val value = EnvReplacementParser.parse( - value = nameAndValue.value, - contextMap = contextMap, - onlyExpression = asCodeEnabled, - contextPair = contextPair - ) - val key = nameAndValue.key ?: return@forEach - contextMap[key] = value - customBuildParameters.add( - BuildParameters( - key = key, - value = value, - valueType = BuildFormPropertyType.STRING, - readOnly = true - ) - ) - } - variablesWithType.addAll(customBuildParameters) - Triple(envList, contextMap, tm) - } - - is NormalContainer -> { - val tm = transMinuteTimeoutToMills(container.controlOption.jobControlOption.timeout) - val contextMap = pipelineContextService.getAllBuildContext(variables).toMutableMap() - fillContainerContext(contextMap, null, c.matrixContext, asCodeSettings?.enable) - Triple(mutableListOf(), contextMap, tm) - } - - else -> throw OperationException("vmName($vmName) is an illegal container type: $c") - } + val (containerEnv, context, timeoutMills) = getContainerContext( + container = c, + buildContainer = container, + variables = variables, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + stage = s, + model = model, + buildInfo = buildInfo, + variablesWithType = variablesWithType, + vmName = vmName + ) buildingHeartBeatUtils.addHeartBeat(buildId, vmSeqId, System.currentTimeMillis()) // # 2365 将心跳监听事件 构建机主动上报成功状态时才触发 buildingHeartBeatUtils.dispatchHeartbeatEvent(buildInfo = buildInfo, containerId = vmSeqId) @@ -325,6 +280,129 @@ class EngineVMBuildService @Autowired(required = false) constructor( throw NotFoundException("Fail to find the vm build container: j($vmSeqId) vmName($vmName)") } + private fun getContainerContext( + container: Container, + buildContainer: PipelineBuildContainer, + variables: Map, + projectId: String, + pipelineId: String, + buildId: String, + stage: Stage, + model: Model?, + buildInfo: BuildInfo, + variablesWithType: MutableList, + vmName: String + ): Triple, MutableMap, Long> { + return when (container) { + is VMBuildContainer -> { + getVMBuildContainerContext( + container = container, + buildContainer = buildContainer, + variables = variables, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + stage = stage, + model = model, + buildInfo = buildInfo, + variablesWithType = variablesWithType + ) + } + + is NormalContainer -> { + val tm = transMinuteTimeoutToMills(buildContainer.controlOption.jobControlOption.timeout) + val contextMap = pipelineContextService.getAllBuildContext(variables).toMutableMap() + fillContainerContext(contextMap, null, container.matrixContext) + Triple(mutableListOf(), contextMap, tm) + } + + else -> throw OperationException("vmName($vmName) is an illegal container type: $container") + } + } + + private fun getVMBuildContainerContext( + container: VMBuildContainer, + buildContainer: PipelineBuildContainer, + variables: Map, + projectId: String, + pipelineId: String, + buildId: String, + stage: Stage, + model: Model?, + buildInfo: BuildInfo, + variablesWithType: MutableList + ): Triple, MutableMap, Long> { + val containerAppResource = client.get(ServiceContainerAppResource::class) + val envList = mutableListOf() + val tm = transMinuteTimeoutToMills(buildContainer.controlOption.jobControlOption.timeout) + val contextMap = variables.plus( + pipelineContextService.buildContext( + projectId = projectId, pipelineId = pipelineId, buildId = buildId, + stageId = stage.id!!, containerId = container.id!!, taskId = null, + variables = variables, model = model, executeCount = buildInfo.executeCount + ) + ).toMutableMap() + val dialect = PipelineDialectUtil.getPipelineDialect(variables[PIPELINE_DIALECT]) + fillContainerContext(contextMap, container.customEnv, container.matrixContext) + val contextPair by lazy { + EnvReplacementParser.getCustomExecutionContextByMap(contextMap) + } + container.buildEnv?.forEach { env -> + containerAppResource.getBuildEnv( + name = env.key, + version = EnvReplacementParser.parse( + value = env.value, + contextMap = contextMap, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ), + os = container.baseOS.name.lowercase() + ).data?.let { self -> envList.add(self) } + } + + // 设置Job环境变量customEnv到variablesWithType和variables中 + // TODO 此处应收敛到variablesWithType或variables的其中一个 + val customBuildParameters = mutableListOf() + // 兼容历史数据 + container.customBuildEnv?.forEach { (k, v) -> + val value = EnvReplacementParser.parse( + value = v, + contextMap = contextMap, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ) + contextMap[k] = value + customBuildParameters.add( + BuildParameters( + key = k, + value = value, + valueType = BuildFormPropertyType.STRING, + readOnly = true + ) + ) + } + container.customEnv?.forEach { nameAndValue -> + val key = nameAndValue.key ?: return@forEach + val value = EnvReplacementParser.parse( + value = nameAndValue.value, + contextMap = contextMap, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ) + contextMap[key] = value + customBuildParameters.add( + BuildParameters( + key = key, + value = value, + valueType = BuildFormPropertyType.STRING, + readOnly = true + ) + ) + } + variablesWithType.addAll(customBuildParameters) + return Triple(envList, contextMap, tm) + } + /** * 对[customBuildEnv]的占位符进行替换， * 再追加env.前缀的构建机容器的上下文[context]， @@ -333,14 +411,22 @@ class EngineVMBuildService @Autowired(required = false) constructor( private fun fillContainerContext( context: MutableMap, customBuildEnv: List?, - matrixContext: Map?, - asCodeEnabled: Boolean? + matrixContext: Map? ) { + val contextPair by lazy { + EnvReplacementParser.getCustomExecutionContextByMap(context) + } + val dialect = PipelineDialectUtil.getPipelineDialect(context[PIPELINE_DIALECT]) customBuildEnv?.let { context.putAll( - customBuildEnv.map { - "$ENV_CONTEXT_KEY_PREFIX${it.key}" to EnvReplacementParser.parse(it.value, context, asCodeEnabled) - }.toMap() + customBuildEnv.associate { + "$ENV_CONTEXT_KEY_PREFIX${it.key}" to EnvReplacementParser.parse( + value = it.value, + contextMap = context, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ) + } ) } @@ -518,12 +604,7 @@ class EngineVMBuildService @Autowired(required = false) constructor( val task = allTasks.firstOrNull() ?: return BuildTask(buildId, vmSeqId, BuildTaskStatus.WAIT, buildInfo.executeCount) - return claim( - task = task, buildId = buildId, userId = task.starter, vmSeqId = vmSeqId, - asCodeEnabled = pipelineAsCodeService.asCodeEnabled( - task.projectId, task.pipelineId, buildId, buildInfo - ) == true - ) + return claim(task = task, buildId = buildId, userId = task.starter, vmSeqId = vmSeqId) } finally { containerIdLock.unlock() } @@ -533,8 +614,7 @@ class EngineVMBuildService @Autowired(required = false) constructor( task: PipelineBuildTask, buildId: String, userId: String, - vmSeqId: String, - asCodeEnabled: Boolean + vmSeqId: String ): BuildTask { LOG.info("ENGINE|$buildId|BC_ING|${task.projectId}|j($vmSeqId)|[${task.taskId}-${task.taskName}]") return when { @@ -642,6 +722,7 @@ class EngineVMBuildService @Autowired(required = false) constructor( expiredInSecond = transMinuteTimeoutToSec(task.additionalOptions?.timeout?.toInt()) ) } + val dialect = PipelineDialectUtil.getPipelineDialect(buildVariable[PIPELINE_DIALECT]) BuildTask( buildId = buildId, vmSeqId = vmSeqId, @@ -653,12 +734,14 @@ class EngineVMBuildService @Autowired(required = false) constructor( executeCount = task.executeCount, type = task.taskType, params = task.taskParams.map { - // 在pipeline as code模式下，此处直接保持原文传给worker - val obj = if (asCodeEnabled) { + // 表达式在worker端替换 + val obj = if (!dialect.supportUseExpression()) { + ObjectReplaceEnvVarUtil.replaceEnvVar( + it.value, buildVariable + ) + } else { it.value - } else ObjectReplaceEnvVarUtil.replaceEnvVar( - it.value, buildVariable - ) + } it.key to JsonUtil.toJson(obj, formatted = false) }.filter { !it.first.startsWith("@type") diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/utils/PipelineUtils.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/utils/PipelineUtils.kt index 176a3d38add..23925a1a9e0 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/utils/PipelineUtils.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/utils/PipelineUtils.kt @@ -68,7 +68,8 @@ object PipelineUtils { logger.warn("Pipeline's start params[${param.id}] is illegal") throw OperationException( message = I18nUtil.getCodeLanMessage( - ProcessMessageCode.ERROR_PIPELINE_PARAMS_NAME_ERROR + ProcessMessageCode.ERROR_PIPELINE_PARAMS_NAME_ERROR, + params = arrayOf(param.id) ) ) } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/BuildVariableService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/BuildVariableService.kt index 5685895238d..0106da13cca 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/BuildVariableService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/BuildVariableService.kt @@ -29,12 +29,14 @@ package com.tencent.devops.process.service import com.tencent.devops.common.api.util.TemplateFastReplaceUtils import com.tencent.devops.common.api.util.Watcher +import com.tencent.devops.common.pipeline.dialect.PipelineDialectUtil import com.tencent.devops.common.pipeline.enums.BuildFormPropertyType import com.tencent.devops.common.pipeline.pojo.BuildParameters import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.service.utils.LogUtils import com.tencent.devops.process.engine.control.lock.PipelineBuildVarLock import com.tencent.devops.process.engine.dao.PipelineBuildVarDao +import com.tencent.devops.process.utils.PIPELINE_DIALECT import com.tencent.devops.process.utils.PIPELINE_RETRY_COUNT import com.tencent.devops.process.utils.PipelineVarUtil import org.apache.commons.lang3.math.NumberUtils @@ -102,7 +104,8 @@ class BuildVariableService @Autowired constructor( buildId = buildId, keys = keys ) - return if (pipelineAsCodeService.asCodeEnabled(projectId, pipelineId, buildId, null) == true) { + val dialect = PipelineDialectUtil.getPipelineDialect(dataMap[PIPELINE_DIALECT]) + return if (dialect.supportUseExpression()) { dataMap } else { PipelineVarUtil.mixOldVarAndNewVar(dataMap) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/PipelineAsCodeService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/PipelineAsCodeService.kt index 0d123f51335..3c0987dd6a3 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/PipelineAsCodeService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/PipelineAsCodeService.kt @@ -27,10 +27,11 @@ package com.tencent.devops.process.service -import com.tencent.devops.process.dao.PipelineSettingDao import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings -import com.tencent.devops.process.engine.dao.PipelineBuildDao -import com.tencent.devops.process.engine.pojo.BuildInfo +import com.tencent.devops.common.pipeline.dialect.IPipelineDialect +import com.tencent.devops.common.pipeline.dialect.PipelineDialectType +import com.tencent.devops.common.pipeline.dialect.PipelineDialectUtil +import com.tencent.devops.process.dao.PipelineSettingDao import org.jooq.DSLContext import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -39,31 +40,95 @@ import org.springframework.stereotype.Service class PipelineAsCodeService @Autowired constructor( private val dslContext: DSLContext, private val pipelineSettingDao: PipelineSettingDao, - private val pipelineBuildDao: PipelineBuildDao + private val projectCacheService: ProjectCacheService ) { fun asCodeEnabled( projectId: String, - pipelineId: String, - buildId: String, - buildInfo: BuildInfo? + pipelineId: String ): Boolean? { - return getPipelineAsCodeSettings(projectId, pipelineId, buildId, buildInfo)?.enable + return getPipelineAsCodeSettings(projectId, pipelineId)?.enable } + fun getPipelineAsCodeSettings(projectId: String, pipelineId: String): PipelineAsCodeSettings? { + val settings = pipelineSettingDao.getPipelineAsCodeSettings( + dslContext = dslContext, projectId = projectId, pipelineId = pipelineId + ) + return getPipelineAsCodeSettings(projectId = projectId, asCodeSettings = settings) + } + + /** + * 前端或者构建机请求时,构造PipelineAsCodeSettings,保证dialect一定有值 + * + * 1. 如果asCodeSettings为空,则使用项目方言 + * 2. 如果继承项目方言,则查询项目方言 + */ fun getPipelineAsCodeSettings( projectId: String, - pipelineId: String, - buildId: String, - buildInfo: BuildInfo? + asCodeSettings: PipelineAsCodeSettings? ): PipelineAsCodeSettings? { - val settings = pipelineSettingDao.getPipelineAsCodeSettings( - dslContext = dslContext, projectId = projectId, pipelineId = pipelineId + return when { + asCodeSettings == null -> { + val projectDialect = + projectCacheService.getProjectDialect(projectId) ?: PipelineDialectType.CLASSIC.name + PipelineAsCodeSettings(inheritedDialect = true, projectDialect = projectDialect) + } + asCodeSettings.inheritedDialect != false -> { + val projectDialect = + projectCacheService.getProjectDialect(projectId) ?: PipelineDialectType.CLASSIC.name + asCodeSettings.copy(projectDialect = projectDialect) + } + else -> + asCodeSettings + } + } + + /** + * 获取项目级方言 + */ + fun getProjectDialect(projectId: String): IPipelineDialect { + val projectDialect = + projectCacheService.getProjectDialect(projectId) ?: PipelineDialectType.CLASSIC.name + return PipelineDialectType.valueOf(projectDialect).dialect + } + + fun getPipelineDialect(projectId: String, pipelineId: String): IPipelineDialect { + val asCodeSettings = getPipelineAsCodeSettings(projectId = projectId, pipelineId = pipelineId) + return getPipelineDialect(projectId = projectId, asCodeSettings = asCodeSettings) + } + + /** + * 获取流水线方言,根据流水线设置 + */ + fun getPipelineDialect(projectId: String, asCodeSettings: PipelineAsCodeSettings?): IPipelineDialect { + return PipelineDialectUtil.getPipelineDialect( + getPipelineAsCodeSettings( + projectId = projectId, + asCodeSettings = asCodeSettings + ) ) -// val info = buildInfo ?: pipelineBuildDao.getBuildInfo( -// dslContext, projectId, pipelineId, buildId -// ) -// return settings?.copy(enable = info?.yamlVersion == YamlVersion.V3_0.tag) - return settings + } + + /** + * 获取流水线方言,根据流水线设置或者方言设置 + */ + fun getPipelineDialect( + projectId: String, + asCodeSettings: PipelineAsCodeSettings?, + inheritedDialectSetting: Boolean?, + pipelineDialectSetting: String? + ): IPipelineDialect { + val projectDialect = projectCacheService.getProjectDialect(projectId = projectId) + return if (asCodeSettings != null) { + PipelineDialectUtil.getPipelineDialect( + asCodeSettings.copy(projectDialect = projectDialect) + ) + } else { + PipelineDialectUtil.getPipelineDialect( + inheritedDialect = inheritedDialectSetting, + projectDialect = projectDialect, + pipelineDialect = pipelineDialectSetting + ) + } } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/ProjectCacheService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/ProjectCacheService.kt index af82c466377..a52e95608c6 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/ProjectCacheService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/ProjectCacheService.kt @@ -68,6 +68,10 @@ class ProjectCacheService @Autowired constructor(private val client: Client) { } } + fun getProjectDialect(projectId: String): String? { + return getProject(projectId = projectId)?.properties?.pipelineDialect + } + private fun getProjectInner(projectId: String): ProjectVO { return client.get(ServiceProjectResource::class).get(projectId).data ?: throw NotFoundException("Fail to find the project info of project($projectId)") diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt index 55b84711fe5..cffab2ac0c1 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt @@ -36,6 +36,8 @@ import com.tencent.devops.common.audit.ActionAuditContent import com.tencent.devops.common.auth.api.ActionId import com.tencent.devops.common.auth.api.ResourceTypeId import com.tencent.devops.common.pipeline.Model +import com.tencent.devops.common.pipeline.dialect.IPipelineDialect +import com.tencent.devops.common.pipeline.dialect.PipelineDialectUtil import com.tencent.devops.common.pipeline.enums.BuildFormPropertyType import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.enums.StartType @@ -54,6 +56,7 @@ import com.tencent.devops.process.engine.service.PipelineRepositoryService import com.tencent.devops.process.engine.service.PipelineRuntimeService import com.tencent.devops.process.pojo.BuildId import com.tencent.devops.process.pojo.app.StartBuildContext +import com.tencent.devops.process.service.PipelineAsCodeService import com.tencent.devops.process.service.ProjectCacheService import com.tencent.devops.process.util.BuildMsgUtils import com.tencent.devops.process.utils.BK_CI_AUTHORIZER @@ -64,6 +67,7 @@ import com.tencent.devops.process.utils.PIPELINE_BUILD_ID import com.tencent.devops.process.utils.PIPELINE_BUILD_MSG import com.tencent.devops.process.utils.PIPELINE_BUILD_URL import com.tencent.devops.process.utils.PIPELINE_CREATE_USER +import com.tencent.devops.process.utils.PIPELINE_DIALECT import com.tencent.devops.process.utils.PIPELINE_ID import com.tencent.devops.process.utils.PIPELINE_NAME import com.tencent.devops.process.utils.PIPELINE_RETRY_BUILD_ID @@ -81,6 +85,7 @@ import com.tencent.devops.process.utils.PIPELINE_START_USER_ID import com.tencent.devops.process.utils.PIPELINE_START_USER_NAME import com.tencent.devops.process.utils.PIPELINE_START_WEBHOOK_USER_ID import com.tencent.devops.process.utils.PIPELINE_UPDATE_USER +import com.tencent.devops.process.utils.PIPELINE_VARIABLES_STRING_LENGTH_MAX import com.tencent.devops.process.utils.PIPELINE_VERSION import com.tencent.devops.process.utils.PROJECT_NAME import com.tencent.devops.process.utils.PROJECT_NAME_CHINESE @@ -99,7 +104,8 @@ class PipelineBuildService( private val projectCacheService: ProjectCacheService, private val pipelineUrlBean: PipelineUrlBean, private val simpleRateLimiter: SimpleRateLimiter, - private val buildIdGenerator: BuildIdGenerator + private val buildIdGenerator: BuildIdGenerator, + private val pipelineAsCodeService: PipelineAsCodeService ) { companion object { private val NO_LIMIT_CHANNEL = listOf(ChannelCode.CODECC) @@ -160,7 +166,12 @@ class PipelineBuildService( ) } ?: pipelineRepositoryService.getSetting(pipeline.projectId, pipeline.pipelineId) } else { - pipelineRepositoryService.getSetting(pipeline.projectId, pipeline.pipelineId) + // webhook、重试可以指定流水线版本 + pipelineRepositoryService.getSettingByPipelineVersion( + projectId = pipeline.projectId, + pipelineId = pipeline.pipelineId, + pipelineVersion = signPipelineVersion + ) } val bucketSize = setting!!.maxConRunningQueueSize val lockKey = "PipelineRateLimit:${pipeline.pipelineId}" @@ -177,6 +188,12 @@ class PipelineBuildService( ) } } + val asCodeSettings = pipelineAsCodeService.getPipelineAsCodeSettings( + projectId = pipeline.projectId, + asCodeSettings = setting.pipelineAsCodeSettings + ) + val pipelineDialectType = + PipelineDialectUtil.getPipelineDialectType(channelCode = channelCode, asCodeSettings = asCodeSettings) // 如果指定了版本号，则设置指定的版本号 pipeline.version = signPipelineVersion ?: pipeline.version @@ -212,7 +229,8 @@ class PipelineBuildService( ) } else { null - } + }, + pipelineDialectType = pipelineDialectType.name ) val context = StartBuildContext.init( @@ -232,6 +250,11 @@ class PipelineBuildService( versionName = versionName, yamlVersion = yamlVersion ) + // 校验流水线启动变量长度 + checkBuildParameterLength( + pipelineDialect = pipelineDialectType.dialect, + buildParameters = context.buildParameters + ) val interceptResult = pipelineInterceptorChain.filter( InterceptData( @@ -275,7 +298,8 @@ class PipelineBuildService( channelCode: ChannelCode, isMobile: Boolean, debug: Boolean? = false, - pipelineAuthorizer: String? = null + pipelineAuthorizer: String? = null, + pipelineDialectType: String ) { val userName = when (startType) { StartType.PIPELINE -> pipelineParamMap[PIPELINE_START_PIPELINE_USER_ID]?.value @@ -366,6 +390,7 @@ class PipelineBuildService( ), readOnly = true ) + pipelineParamMap[PIPELINE_DIALECT] = BuildParameters(PIPELINE_DIALECT, pipelineDialectType, readOnly = true) // 自定义触发源材料信息 startValues?.get(BK_CI_MATERIAL_ID)?.let { pipelineParamMap[BK_CI_MATERIAL_ID] = BuildParameters( @@ -405,4 +430,19 @@ class PipelineBuildService( } // return originStartParams } + + private fun checkBuildParameterLength( + pipelineDialect: IPipelineDialect, + buildParameters: List + ) { + val longVarNames = buildParameters.filter { + it.value.toString().length >= PIPELINE_VARIABLES_STRING_LENGTH_MAX + }.map { it.key } + if (longVarNames.isNotEmpty() && !pipelineDialect.supportLongVarValue()) { + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_VARIABLES_OUT_OF_LENGTH, + params = arrayOf(longVarNames.toString()) + ) + } + } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingVersionService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingVersionService.kt index bdc28ca5c69..18a7eefe58c 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingVersionService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingVersionService.kt @@ -38,6 +38,7 @@ import com.tencent.devops.process.dao.PipelineSettingDao import com.tencent.devops.process.dao.PipelineSettingVersionDao import com.tencent.devops.process.pojo.PipelineDetailInfo import com.tencent.devops.process.pojo.setting.PipelineSettingVersion +import com.tencent.devops.process.service.PipelineAsCodeService import com.tencent.devops.process.service.label.PipelineGroupService import com.tencent.devops.process.utils.PipelineVersionUtils import org.jooq.DSLContext @@ -51,7 +52,8 @@ class PipelineSettingVersionService @Autowired constructor( private val dslContext: DSLContext, private val pipelineGroupService: PipelineGroupService, private val pipelineSettingDao: PipelineSettingDao, - private val pipelineSettingVersionDao: PipelineSettingVersionDao + private val pipelineSettingVersionDao: PipelineSettingVersionDao, + private val pipelineAsCodeService: PipelineAsCodeService ) { /** @@ -133,6 +135,7 @@ class PipelineSettingVersionService @Autowired constructor( } else { null } + settingInfo.pipelineAsCodeSettings = ve.pipelineAsCodeSettings } // 来自前端的请求中，版本中的可能还不是正式生效的，如果和正式配置中有差异则重新获取名称 if (settingInfo.labels.isNotEmpty() && settingInfo.labels != labels && userId != null) { @@ -144,6 +147,10 @@ class PipelineSettingVersionService @Autowired constructor( } settingInfo.labelNames = labelNames } + settingInfo.pipelineAsCodeSettings = pipelineAsCodeService.getPipelineAsCodeSettings( + projectId = projectId, + settingInfo.pipelineAsCodeSettings + ) } return settingInfo diff --git a/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPluginTest.kt b/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPluginTest.kt index 06e4178fae4..ca38042d7c0 100644 --- a/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPluginTest.kt +++ b/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPluginTest.kt @@ -199,7 +199,7 @@ class DefaultModelCheckPluginTest : TestBase() { fun checkTriggerContainer() { // trigger val triggerStage = genStages(1, 1, 1) - checkPlugin.checkTriggerContainer(triggerStage[0]) + checkPlugin.checkTriggerContainer(triggerStage[0], false) } @Test diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt index 595867b53bf..3aedb903f5f 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/vm/DispatchVMStartupTaskAtom.kt @@ -42,6 +42,7 @@ import com.tencent.devops.common.pipeline.EnvReplacementParser import com.tencent.devops.common.pipeline.NameAndValue import com.tencent.devops.common.pipeline.container.Container import com.tencent.devops.common.pipeline.container.VMBuildContainer +import com.tencent.devops.common.pipeline.dialect.PipelineDialectUtil import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentEnvDispatchType import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentIDDispatchType @@ -65,8 +66,8 @@ import com.tencent.devops.process.engine.service.detail.ContainerBuildDetailServ import com.tencent.devops.process.engine.service.record.ContainerBuildRecordService import com.tencent.devops.process.pojo.mq.PipelineAgentShutdownEvent import com.tencent.devops.process.pojo.mq.PipelineAgentStartupEvent -import com.tencent.devops.process.service.PipelineAsCodeService import com.tencent.devops.process.service.PipelineContextService +import com.tencent.devops.process.utils.PIPELINE_DIALECT import com.tencent.devops.process.utils.BK_CI_AUTHORIZER import com.tencent.devops.store.api.container.ServiceContainerAppResource import org.slf4j.LoggerFactory @@ -92,7 +93,6 @@ class DispatchVMStartupTaskAtom @Autowired constructor( private val pipelineEventDispatcher: SampleEventDispatcher, private val buildLogPrinter: BuildLogPrinter, private val dispatchTypeBuilder: DispatchTypeBuilder, - private val pipelineAsCodeService: PipelineAsCodeService, private val pipelineContextService: PipelineContextService, private val pipelineTaskService: PipelineTaskService ) : IAtomTask { @@ -296,14 +296,11 @@ class DispatchVMStartupTaskAtom @Autowired constructor( ): Boolean { param.buildEnv?.let { buildEnv -> val asCode by lazy { - val asCodeSettings = pipelineAsCodeService.getPipelineAsCodeSettings( - task.projectId, task.pipelineId, task.buildId, null - ) - val asCodeEnabled = asCodeSettings?.enable == true - val contextPair = if (asCodeEnabled) { + val dialect = PipelineDialectUtil.getPipelineDialect(variables[PIPELINE_DIALECT]) + val contextPair = if (dialect.supportUseExpression()) { EnvReplacementParser.getCustomExecutionContextByMap(variables) } else null - Pair(asCodeEnabled, contextPair) + Pair(dialect, contextPair) } buildEnv.forEach { env -> if (!env.value.startsWith("$")) { @@ -312,7 +309,7 @@ class DispatchVMStartupTaskAtom @Autowired constructor( val version = EnvReplacementParser.parse( value = env.value, contextMap = variables, - onlyExpression = asCode.first, + onlyExpression = asCode.first.supportUseExpression(), contextPair = asCode.second ) val res = client.get(ServiceContainerAppResource::class).getBuildEnv( diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/ContainerControl.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/ContainerControl.kt index 6d8740c287d..281eb2cd48d 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/ContainerControl.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/ContainerControl.kt @@ -177,7 +177,7 @@ class ContainerControl @Autowired constructor( stageId = stageId, onlyMatrixGroup = true ) - val pipelineAsCodeEnabled = pipelineAsCodeService.asCodeEnabled(projectId, pipelineId, buildId, buildInfo) + val pipelineAsCodeEnabled = pipelineAsCodeService.asCodeEnabled(projectId, pipelineId) val context = ContainerContext( buildStatus = this.status, // 初始状态为容器状态，中间流转会切换状态，并最终赋值给该容器状态 diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/StageControl.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/StageControl.kt index e68e4456530..ee96cfd4070 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/StageControl.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/StageControl.kt @@ -140,7 +140,7 @@ class StageControl @Autowired constructor( containsMatrix = false ) val executeCount = buildVariableService.getBuildExecuteCount(projectId, pipelineId, buildId) - val pipelineAsCodeEnabled = pipelineAsCodeService.asCodeEnabled(projectId, pipelineId, buildId, buildInfo) + val pipelineAsCodeEnabled = pipelineAsCodeService.asCodeEnabled(projectId, pipelineId) // #10082 过滤Agent复用互斥的endJob信息 val mutexJobs = containers.filter { it.controlOption.agentReuseMutex?.endJob == true && diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/InitializeMatrixGroupStageCmd.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/InitializeMatrixGroupStageCmd.kt index 6e0b4dee045..4ca56c8c632 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/InitializeMatrixGroupStageCmd.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/InitializeMatrixGroupStageCmd.kt @@ -37,6 +37,7 @@ import com.tencent.devops.common.pipeline.EnvReplacementParser import com.tencent.devops.common.pipeline.NameAndValue import com.tencent.devops.common.pipeline.container.NormalContainer import com.tencent.devops.common.pipeline.container.VMBuildContainer +import com.tencent.devops.common.pipeline.dialect.PipelineDialectUtil import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.matrix.MatrixConfig import com.tencent.devops.common.pipeline.option.JobControlOption @@ -69,6 +70,7 @@ import com.tencent.devops.process.pojo.TemplateAcrossInfoType import com.tencent.devops.process.pojo.pipeline.record.BuildRecordContainer import com.tencent.devops.process.pojo.pipeline.record.BuildRecordTask import com.tencent.devops.process.service.PipelineBuildTemplateAcrossInfoService +import com.tencent.devops.process.utils.PIPELINE_DIALECT import com.tencent.devops.process.utils.PIPELINE_MATRIX_CON_RUNNING_SIZE_MAX import com.tencent.devops.process.utils.PIPELINE_MATRIX_MAX_CON_RUNNING_SIZE_DEFAULT import com.tencent.devops.process.utils.PIPELINE_STAGE_CONTAINERS_COUNT_MAX @@ -177,7 +179,6 @@ class InitializeMatrixGroupStageCmd( val event = commandContext.event val variables = commandContext.variables - val asCodeEnabled = commandContext.pipelineAsCodeEnabled ?: false val modelStage = containerBuildDetailService.getBuildModel( projectId = parentContainer.projectId, buildId = parentContainer.buildId @@ -197,6 +198,7 @@ class InitializeMatrixGroupStageCmd( containerId = parentContainer.containerId, executeCount = parentContainer.executeCount ) + val dialect = PipelineDialectUtil.getPipelineDialect(variables[PIPELINE_DIALECT]) // #4518 待生成的分裂后container表和task表记录 val buildContainerList = mutableListOf() val buildTaskList = mutableListOf() @@ -215,8 +217,8 @@ class InitializeMatrixGroupStageCmd( LOG.info( "ENGINE|${event.buildId}|${event.source}|INIT_MATRIX_CONTAINER|${event.stageId}|" + - "matrixGroupId=$matrixGroupId|containerHashId=${modelContainer.containerHashId}" + - "|context=$context|asCodeEnabled=$asCodeEnabled" + "matrixGroupId=$matrixGroupId|containerHashId=${modelContainer.containerHashId}" + + "|context=$context|dialect=${variables[PIPELINE_DIALECT]}" ) val matrixOption: MatrixControlOption @@ -238,7 +240,7 @@ class InitializeMatrixGroupStageCmd( dependOnContainerId2JobIds = null ) matrixOption = checkAndFetchOption(modelContainer.matrixControlOption) - matrixConfig = matrixOption.convertMatrixConfig(variables, asCodeEnabled) + matrixConfig = matrixOption.convertMatrixConfig(variables) contextCaseList = matrixConfig.getAllCombinations() if (contextCaseList.size > MATRIX_CASE_MAX_COUNT) { @@ -255,10 +257,9 @@ class InitializeMatrixGroupStageCmd( if (!it.key.isNullOrBlank()) customBuildEnv[it.key!!] = it.value ?: "" } val allContext = customBuildEnv.plus(contextCase) - val contextPair = if (asCodeEnabled) { + val contextPair = if (dialect.supportUseExpression()) { EnvReplacementParser.getCustomExecutionContextByMap(allContext) } else null - // 对自定义构建环境的做特殊解析 // customDispatchType决定customBaseOS是否计算，请勿填充默认值 val parsedInfo = matrixOption.customDispatchInfo?.let { self -> @@ -276,9 +277,17 @@ class InitializeMatrixGroupStageCmd( val mutexGroup = modelContainer.mutexGroup?.let { self -> self.copy( mutexGroupName = EnvReplacementParser.parse( - self.mutexGroupName, allContext, asCodeEnabled, contextPair + value = self.mutexGroupName, + contextMap = allContext, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair ), - linkTip = EnvReplacementParser.parse(self.linkTip, allContext, asCodeEnabled, contextPair) + linkTip = EnvReplacementParser.parse( + value = self.linkTip, + contextMap = allContext, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ) ) } val newSeq = context.containerSeq++ @@ -290,7 +299,12 @@ class InitializeMatrixGroupStageCmd( modelContainer.elements, context.executeCount, postParentIdMap, matrixTaskIds ) val newContainer = VMBuildContainer( - name = EnvReplacementParser.parse(modelContainer.name, allContext, asCodeEnabled, contextPair), + name = EnvReplacementParser.parse( + value = modelContainer.name, + contextMap = allContext, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ), id = newSeq.toString(), containerId = newSeq.toString(), containerHashId = modelContainerIdGenerator.getNextId(), @@ -315,13 +329,28 @@ class InitializeMatrixGroupStageCmd( }, buildEnv = buildEnv ?: modelContainer.buildEnv, thirdPartyAgentId = modelContainer.thirdPartyAgentId?.let { self -> - EnvReplacementParser.parse(self, allContext, asCodeEnabled, contextPair) + EnvReplacementParser.parse( + value = self, + contextMap = allContext, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ) }, thirdPartyAgentEnvId = modelContainer.thirdPartyAgentEnvId?.let { self -> - EnvReplacementParser.parse(self, allContext, asCodeEnabled, contextPair) + EnvReplacementParser.parse( + value = self, + contextMap = allContext, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ) }, thirdPartyWorkspace = modelContainer.thirdPartyWorkspace?.let { self -> - EnvReplacementParser.parse(self, allContext, asCodeEnabled, contextPair) + EnvReplacementParser.parse( + value = self, + contextMap = allContext, + onlyExpression = dialect.supportUseExpression(), + contextPair = contextPair + ) } ) newContainer.jobId?.let { matrixJobIds.add(it) } @@ -397,7 +426,7 @@ class InitializeMatrixGroupStageCmd( dependOnContainerId2JobIds = null ) matrixOption = checkAndFetchOption(modelContainer.matrixControlOption) - matrixConfig = matrixOption.convertMatrixConfig(variables, asCodeEnabled) + matrixConfig = matrixOption.convertMatrixConfig(variables) contextCaseList = matrixConfig.getAllCombinations() contextCaseList.forEach { contextCase -> @@ -411,22 +440,30 @@ class InitializeMatrixGroupStageCmd( val statusElements = generateMatrixElements( modelContainer.elements, context.executeCount, postParentIdMap, matrixTaskIds ) - val replacement = if (asCodeEnabled) { - EnvReplacementParser.getCustomExecutionContextByMap(contextCase) - } else null + val replacement = EnvReplacementParser.getCustomExecutionContextByMap(contextCase) val mutexGroup = modelContainer.mutexGroup?.let { self -> self.copy( mutexGroupName = EnvReplacementParser.parse( value = self.mutexGroupName, contextMap = contextCase, - onlyExpression = asCodeEnabled, + onlyExpression = dialect.supportUseExpression(), contextPair = replacement ), - linkTip = EnvReplacementParser.parse(self.linkTip, contextCase, asCodeEnabled, replacement) + linkTip = EnvReplacementParser.parse( + value = self.linkTip, + contextMap = contextCase, + onlyExpression = dialect.supportUseExpression(), + contextPair = replacement + ) ) } val newContainer = NormalContainer( - name = EnvReplacementParser.parse(modelContainer.name, contextCase, asCodeEnabled, replacement), + name = EnvReplacementParser.parse( + value = modelContainer.name, + contextMap = contextCase, + onlyExpression = dialect.supportUseExpression(), + contextPair = replacement + ), id = newSeq.toString(), containerId = newSeq.toString(), containerHashId = modelContainerIdGenerator.getNextId(), diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckPauseReviewStageCmd.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckPauseReviewStageCmd.kt index 5c6660e8245..9dd229e668b 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckPauseReviewStageCmd.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckPauseReviewStageCmd.kt @@ -29,6 +29,7 @@ package com.tencent.devops.process.engine.control.command.stage.impl import com.tencent.devops.common.auth.api.AuthProjectApi import com.tencent.devops.common.auth.code.PipelineAuthServiceCode +import com.tencent.devops.common.pipeline.dialect.PipelineDialectUtil import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.pojo.StagePauseCheck import com.tencent.devops.process.engine.common.BS_MANUAL_START_STAGE @@ -42,6 +43,7 @@ import com.tencent.devops.process.engine.pojo.event.PipelineBuildStageEvent import com.tencent.devops.process.engine.service.PipelineStageService import com.tencent.devops.process.service.BuildVariableService import com.tencent.devops.process.utils.PIPELINE_BUILD_NUM +import com.tencent.devops.process.utils.PIPELINE_DIALECT import com.tencent.devops.process.utils.PIPELINE_NAME import com.tencent.devops.process.utils.PIPELINE_START_USER_NAME import org.slf4j.LoggerFactory @@ -96,7 +98,8 @@ class CheckPauseReviewStageCmd( // #3742 进入暂停状态则刷新完状态后直接返回，等待手动触发 LOG.info("ENGINE|${event.buildId}|${event.source}|STAGE_PAUSE|${event.stageId}") - stage.checkIn?.parseReviewVariables(commandContext.variables, commandContext.pipelineAsCodeEnabled) + val dialect = PipelineDialectUtil.getPipelineDialect(commandContext.variables[PIPELINE_DIALECT]) + stage.checkIn?.parseReviewVariables(commandContext.variables, dialect = dialect) if (stage.checkIn != null) { checkReviewGroup(event.projectId, stage.checkIn!!) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt index 196a5097fa5..9f6aedadf5c 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt @@ -39,7 +39,6 @@ import com.tencent.devops.common.api.util.MessageUtil import com.tencent.devops.common.auth.api.ActionId import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.common.auth.api.AuthResourceType -import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.enums.VersionStatus @@ -101,8 +100,7 @@ class UserPipelineResourceImpl @Autowired constructor( private val auditService: AuditService, private val pipelineVersionFacadeService: PipelineVersionFacadeService, private val pipelineRuleService: PipelineRuleService, - private val pipelineRecentUseService: PipelineRecentUseService, - private val client: Client + private val pipelineRecentUseService: PipelineRecentUseService ) : UserPipelineResource { override fun hasCreatePermission(userId: String, projectId: String): Result { diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt index f5ad4172a56..b7a5ba7c12a 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt @@ -111,12 +111,12 @@ import org.springframework.beans.factory.annotation.Value import org.springframework.dao.DuplicateKeyException import org.springframework.stereotype.Service import java.net.URLEncoder +import java.time.LocalDateTime import java.util.LinkedList import java.util.concurrent.TimeUnit import javax.ws.rs.core.MediaType import javax.ws.rs.core.Response import javax.ws.rs.core.StreamingOutput -import java.time.LocalDateTime @Suppress("ALL") @Service @@ -140,7 +140,8 @@ class PipelineInfoFacadeService @Autowired constructor( private val transferService: PipelineTransferYamlService, private val yamlFacadeService: PipelineYamlFacadeService, private val operationLogService: PipelineOperationLogService, - private val pipelineAuthorizationService: PipelineAuthorizationService + private val pipelineAuthorizationService: PipelineAuthorizationService, + private val pipelineAsCodeService: PipelineAsCodeService ) { @Value("\${process.deletedPipelineStoreDays:30}") @@ -309,7 +310,9 @@ class PipelineInfoFacadeService @Autowired constructor( useLabelSettings: Boolean? = false, useConcurrencyGroup: Boolean? = false, description: String? = null, - yamlInfo: PipelineYamlVo? = null + yamlInfo: PipelineYamlVo? = null, + inheritedDialectSetting: Boolean? = true, + pipelineDialectSetting: String? = null ): DeployPipelineResult { val watcher = Watcher(id = "createPipeline|$projectId|$userId|$channelCode|$checkPermission|$instanceType|$fixPipelineId") @@ -429,6 +432,12 @@ class PipelineInfoFacadeService @Autowired constructor( } watcher.start("deployPipeline") + val pipelineDialect = pipelineAsCodeService.getPipelineDialect( + projectId = projectId, + asCodeSettings = setting?.pipelineAsCodeSettings, + inheritedDialectSetting = inheritedDialectSetting, + pipelineDialectSetting = pipelineDialectSetting + ) val result = pipelineRepositoryService.deployPipeline( model = instance, setting = setting, @@ -446,7 +455,10 @@ class PipelineInfoFacadeService @Autowired constructor( description = description, yaml = yaml, baseVersion = null, - yamlInfo = yamlInfo + yamlInfo = yamlInfo, + inheritedDialectSetting = inheritedDialectSetting, + pipelineDialectSetting = pipelineDialectSetting, + pipelineDialect = pipelineDialect ) pipelineId = result.pipelineId watcher.stop() @@ -599,12 +611,14 @@ class PipelineInfoFacadeService @Autowired constructor( } ?: newResource.model.name.ifBlank { yamlFileName } + val pipelineAsCodeSettings = + newResource.setting.pipelineAsCodeSettings?.copy(enable = true) ?: PipelineAsCodeSettings(enable = true) // 通过PAC模式创建或保存的流水线均打开PAC // 修正创建时的流水线名和增加PAC开关参数 val newSetting = newResource.setting.copy( projectId = projectId, pipelineName = pipelineName, - pipelineAsCodeSettings = PipelineAsCodeSettings(enable = true) + pipelineAsCodeSettings = pipelineAsCodeSettings ) return createPipeline( userId = userId, @@ -651,13 +665,15 @@ class PipelineInfoFacadeService @Autowired constructor( // 通过PAC模式创建或保存的流水线均打开PAC // 修正创建时的流水线名和增加PAC开关参数 val pipelineName = newResource.model.name.ifBlank { yamlFileName } + val pipelineAsCodeSettings = + newResource.setting.pipelineAsCodeSettings?.copy(enable = true) ?: PipelineAsCodeSettings(enable = true) val savedSetting = pipelineSettingFacadeService.saveSetting( userId = userId, projectId = projectId, pipelineId = pipelineId, setting = newResource.setting.copy( pipelineName = pipelineName, - pipelineAsCodeSettings = PipelineAsCodeSettings(enable = true) + pipelineAsCodeSettings = pipelineAsCodeSettings ), checkPermission = false, versionStatus = versionStatus, @@ -1109,6 +1125,11 @@ class PipelineInfoFacadeService @Autowired constructor( ) modelCheckPlugin.beforeDeleteElementInExistsModel(existModel, model, param) } + val pipelineSetting = savedSetting ?: pipelineSettingFacadeService.getSettingInfo(projectId, pipelineId) + val pipelineDialect = pipelineAsCodeService.getPipelineDialect( + projectId = projectId, + pipelineSetting?.pipelineAsCodeSettings + ) val deployResult = pipelineRepositoryService.deployPipeline( model = model, projectId = projectId, @@ -1123,7 +1144,8 @@ class PipelineInfoFacadeService @Autowired constructor( description = description, yaml = yaml, baseVersion = baseVersion, - yamlInfo = yamlInfo + yamlInfo = yamlInfo, + pipelineDialect = pipelineDialect ) // 审计 ActionAuditContext.current() diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineVersionFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineVersionFacadeService.kt index b9de79ed072..39ab5330a15 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineVersionFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineVersionFacadeService.kt @@ -105,7 +105,8 @@ class PipelineVersionFacadeService @Autowired constructor( private val pipelineViewGroupService: PipelineViewGroupService, private val pipelineBuildSummaryDao: PipelineBuildSummaryDao, private val pipelineBuildDao: PipelineBuildDao, - private val buildLogPrinter: BuildLogPrinter + private val buildLogPrinter: BuildLogPrinter, + private val pipelineAsCodeService: PipelineAsCodeService ) { companion object { @@ -319,7 +320,11 @@ class PipelineVersionFacadeService @Autowired constructor( create = false, versionStatus = VersionStatus.RELEASED, channelCode = pipeline.channelCode, - yamlInfo = request.yamlInfo + yamlInfo = request.yamlInfo, + pipelineDialect = pipelineAsCodeService.getPipelineDialect( + projectId = projectId, + asCodeSettings = originSetting.pipelineAsCodeSettings + ) ) val originYaml = pipelineYamlFacadeService.getPipelineYamlInfo(projectId, pipelineId, version) // 如果不匹配已有状态则报错，需要用户重新刷新页面 @@ -329,7 +334,7 @@ class PipelineVersionFacadeService @Autowired constructor( // 根据项目PAC状态进行接口调用 val enabled = originYaml != null || request.enablePac val targetSettings = originSetting.copy( - pipelineAsCodeSettings = PipelineAsCodeSettings(enabled) + pipelineAsCodeSettings = originSetting.pipelineAsCodeSettings?.copy(enable = enabled) ) val (versionStatus, branchName) = if ( enabled && request.targetAction == CodeTargetAction.CHECKOUT_BRANCH_AND_REQUEST_MERGE @@ -570,7 +575,9 @@ class PipelineVersionFacadeService @Autowired constructor( versionStatus = VersionStatus.COMMITTING, useSubscriptionSettings = request.useSubscriptionSettings, useLabelSettings = request.useLabelSettings, - useConcurrencyGroup = request.useConcurrencyGroup + useConcurrencyGroup = request.useConcurrencyGroup, + inheritedDialectSetting = request.inheritedDialect, + pipelineDialectSetting = request.pipelineDialect ) } @@ -644,7 +651,9 @@ class PipelineVersionFacadeService @Autowired constructor( baseVersion = resource.baseVersion, baseVersionName = baseResource?.versionName, yamlSupported = yamlSupported, - yamlInvalidMsg = msg + yamlInvalidMsg = msg, + updater = resource.updater ?: resource.creator, + updateTime = resource.updateTime?.timestampmilli() ) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/SubPipelineRepositoryService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/SubPipelineRepositoryService.kt index faf07365aad..0082bd70a6e 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/SubPipelineRepositoryService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/SubPipelineRepositoryService.kt @@ -6,6 +6,7 @@ import com.tencent.devops.common.api.util.EnvUtils import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.container.Stage +import com.tencent.devops.common.pipeline.container.TriggerContainer import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.pipeline.pojo.element.SubPipelineCallElement import com.tencent.devops.common.pipeline.pojo.element.atom.ElementCheckResult @@ -15,7 +16,6 @@ import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildLessAto import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.process.constant.ProcessMessageCode import com.tencent.devops.process.engine.dao.PipelineResourceDao -import com.tencent.devops.process.engine.extend.DefaultModelCheckPlugin import com.tencent.devops.process.engine.service.PipelineRepositoryService import com.tencent.devops.process.permission.PipelinePermissionService import com.tencent.devops.process.utils.PipelineVarUtil @@ -32,7 +32,6 @@ class SubPipelineRepositoryService @Autowired constructor( private val objectMapper: ObjectMapper, private val pipelineResDao: PipelineResourceDao, private val pipelineRepositoryService: PipelineRepositoryService, - private val defaultModelCheckPlugin: DefaultModelCheckPlugin, private val pipelinePermissionService: PipelinePermissionService ) { @@ -294,11 +293,11 @@ class SubPipelineRepositoryService @Autowired constructor( } private fun getContextMap(stages: List): Map { - val trigger = stages.getOrNull(0) - ?: throw ErrorCodeException(errorCode = ProcessMessageCode.ERROR_PIPELINE_MODEL_NEED_JOB) - // 检查触发容器 - val paramsMap = defaultModelCheckPlugin.checkTriggerContainer(trigger) - return PipelineVarUtil.fillVariableMap(paramsMap.mapValues { it.value.defaultValue.toString() }) + val triggerContainer = stages[0].containers[0] as TriggerContainer + val variables = triggerContainer.params.associate { param -> + param.id to param.defaultValue.toString() + } + return PipelineVarUtil.fillVariableMap(variables) } companion object { @@ -306,4 +305,4 @@ class SubPipelineRepositoryService @Autowired constructor( private val PIPELINE_ID_PATTERN = Pattern.compile("(p-)?[a-f\\d]{32}") private const val SUB_PIPELINE_EXEC_ATOM_CODE = "SubPipelineExec" } -} \ No newline at end of file +} diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt index 781f002957c..e1e5fca0e24 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt @@ -122,6 +122,7 @@ import com.tencent.devops.process.pojo.template.TemplateType import com.tencent.devops.process.pojo.template.TemplateVersion import com.tencent.devops.process.pojo.template.TemplateWithPermission import com.tencent.devops.process.service.ParamFacadeService +import com.tencent.devops.process.service.PipelineAsCodeService import com.tencent.devops.process.service.PipelineInfoFacadeService import com.tencent.devops.process.service.PipelineRemoteAuthService import com.tencent.devops.process.service.StageTagService @@ -180,7 +181,8 @@ class TemplateFacadeService @Autowired constructor( private val modelCheckPlugin: ModelCheckPlugin, private val pipelineSettingFacadeService: PipelineSettingFacadeService, private val templateCommonService: TemplateCommonService, - private val templateSettingService: TemplateSettingService + private val templateSettingService: TemplateSettingService, + private val pipelineAsCodeService: PipelineAsCodeService ) { @Value("\${template.maxSyncInstanceNum:10}") @@ -2321,11 +2323,14 @@ class TemplateFacadeService @Autowired constructor( errorCode = ProcessMessageCode.TEMPLATE_NAME_CAN_NOT_NULL ) } + // 模版先都统一使用项目配置 + val projectDialect = projectId?.let { pipelineAsCodeService.getProjectDialect(projectId = it) } modelCheckPlugin.checkModelIntegrity( model = template, projectId = projectId, userId = userId, - isTemplate = true + isTemplate = true, + pipelineDialect = projectDialect ) checkPipelineParam(template) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateSettingService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateSettingService.kt index 90c7630a1bd..844587c51f6 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateSettingService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateSettingService.kt @@ -6,9 +6,9 @@ import com.tencent.devops.common.pipeline.extend.ModelCheckPlugin import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting import com.tencent.devops.process.constant.ProcessMessageCode import com.tencent.devops.process.engine.dao.template.TemplateDao -import com.tencent.devops.process.engine.service.PipelineInfoExtService import com.tencent.devops.process.engine.service.PipelineRepositoryService import com.tencent.devops.process.permission.template.PipelineTemplatePermissionService +import com.tencent.devops.process.service.PipelineAsCodeService import com.tencent.devops.process.service.label.PipelineGroupService import com.tencent.devops.process.service.pipeline.PipelineSettingVersionService import org.jooq.DSLContext @@ -25,12 +25,12 @@ class TemplateSettingService @Autowired constructor( private val dslContext: DSLContext, private val pipelineGroupService: PipelineGroupService, private val pipelineRepositoryService: PipelineRepositoryService, - private val pipelineInfoExtService: PipelineInfoExtService, private val templateCommonService: TemplateCommonService, private val templateDao: TemplateDao, private val modelCheckPlugin: ModelCheckPlugin, private val pipelineSettingVersionService: PipelineSettingVersionService, - private val pipelineTemplatePermissionService: PipelineTemplatePermissionService + private val pipelineTemplatePermissionService: PipelineTemplatePermissionService, + private val pipelineAsCodeService: PipelineAsCodeService ) { fun updateTemplateSetting( projectId: String, @@ -144,6 +144,11 @@ class TemplateSettingService @Autowired constructor( labels.addAll(it.labels) } setting.labels = labels + val asCodeSettings = pipelineAsCodeService.getPipelineAsCodeSettings( + projectId = projectId, + asCodeSettings = setting.pipelineAsCodeSettings + ) + setting.pipelineAsCodeSettings = asCodeSettings return setting } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/exception/hanlder/YamlTriggerExceptionUtil.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/exception/hanlder/YamlTriggerExceptionUtil.kt index fc94adb36af..3f875e34f5c 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/exception/hanlder/YamlTriggerExceptionUtil.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/exception/hanlder/YamlTriggerExceptionUtil.kt @@ -52,7 +52,7 @@ object YamlTriggerExceptionUtil { PipelineTriggerFailedMsg(exception.message ?: PipelineTriggerReason.UNKNOWN_ERROR.detail) ) else -> Pair( - PipelineTriggerReason.UNKNOWN_ERROR.name, + PipelineTriggerReason.TRIGGER_FAILED.name, PipelineTriggerFailedMsg(exception.message ?: PipelineTriggerReason.UNKNOWN_ERROR.detail) ) } diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceProjectResource.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceProjectResource.kt index ecf07dc5631..302852b5ac3 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceProjectResource.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceProjectResource.kt @@ -36,13 +36,13 @@ import com.tencent.devops.common.auth.api.pojo.ProjectConditionDTO import com.tencent.devops.common.auth.api.pojo.SubjectScopeInfo import com.tencent.devops.project.pojo.OrgInfo import com.tencent.devops.project.pojo.ProjectBaseInfo +import com.tencent.devops.project.pojo.ProjectByConditionDTO import com.tencent.devops.project.pojo.ProjectCreateInfo import com.tencent.devops.project.pojo.ProjectCreateUserInfo import com.tencent.devops.project.pojo.ProjectOrganizationInfo import com.tencent.devops.project.pojo.ProjectProperties import com.tencent.devops.project.pojo.ProjectUpdateInfo import com.tencent.devops.project.pojo.ProjectVO -import com.tencent.devops.project.pojo.ProjectByConditionDTO import com.tencent.devops.project.pojo.Result import com.tencent.devops.project.pojo.enums.PluginDetailsDisplayOrder import com.tencent.devops.project.pojo.enums.ProjectChannelCode diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/user/UserProjectResource.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/user/UserProjectResource.kt index eda2451c862..e8f127c08cd 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/user/UserProjectResource.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/user/UserProjectResource.kt @@ -34,6 +34,7 @@ import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE import com.tencent.devops.common.api.pojo.Pagination import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.project.pojo.OperationalProductVO +import com.tencent.devops.project.pojo.ProjectByConditionDTO import com.tencent.devops.project.pojo.ProjectCollation import com.tencent.devops.project.pojo.ProjectCreateInfo import com.tencent.devops.project.pojo.ProjectDiffVO @@ -41,7 +42,6 @@ import com.tencent.devops.project.pojo.ProjectLogo import com.tencent.devops.project.pojo.ProjectSortType import com.tencent.devops.project.pojo.ProjectUpdateInfo import com.tencent.devops.project.pojo.ProjectVO -import com.tencent.devops.project.pojo.ProjectByConditionDTO import com.tencent.devops.project.pojo.Result import com.tencent.devops.project.pojo.enums.ProjectValidateType import io.swagger.v3.oas.annotations.Operation @@ -372,4 +372,16 @@ interface UserProjectResource { @PathParam("bgName") bgName: String ): Result> + + @Operation(summary = "获取项目级流水线方言, 流水线编辑/修改时调用") + @GET + @Path("{projectId}/pipelineDialect") + fun getPipelineDialect( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String + ): Result } diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectApprovalInfo.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectApprovalInfo.kt index c2e7fbb0947..e616522fab0 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectApprovalInfo.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectApprovalInfo.kt @@ -81,5 +81,7 @@ data class ProjectApprovalInfo( @get:Schema(title = "运营产品ID") val productId: Int? = null, @get:Schema(title = "运营产品名称") - val productName: String? = null + val productName: String? = null, + @get:Schema(title = "项目相关配置") + val properties: ProjectProperties? = null ) diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDiffVO.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDiffVO.kt index 4a61a381064..1096d030ade 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDiffVO.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDiffVO.kt @@ -120,5 +120,9 @@ data class ProjectDiffVO( @get:Schema(title = "运营产品名称") val productName: String? = null, @get:Schema(title = "审批中运营产品名称") - val afterProductName: String? = null + val afterProductName: String? = null, + @get:Schema(title = "流水线语言风格") + val pipelineDialect: String? = null, + @get:Schema(title = "审批中流水线语言风格") + val afterPipelineDialect: String? = null ) diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectProperties.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectProperties.kt index eeadad23319..95f524fe914 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectProperties.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectProperties.kt @@ -58,5 +58,7 @@ data class ProjectProperties( PluginDetailsDisplayOrder.LOG, PluginDetailsDisplayOrder.ARTIFACT, PluginDetailsDisplayOrder.CONFIG - ) + ), + @get:Schema(title = "流水线语法风格") + var pipelineDialect: String? = "CLASSIC" ) diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectUpdateInfo.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectUpdateInfo.kt index a61e85e4f31..22d2fd7c453 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectUpdateInfo.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectUpdateInfo.kt @@ -66,7 +66,7 @@ data class ProjectUpdateInfo( @get:Schema(title = "是否保密") var secrecy: Boolean = false, @get:Schema(title = "项目相关配置") - val properties: ProjectProperties? = null, + var properties: ProjectProperties? = null, @get:Schema(title = "项目最大可授权人员范围") val subjectScopes: List? = emptyList(), @get:Schema(title = "logo地址") diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectApprovalDao.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectApprovalDao.kt index 5918e9d4a2e..8dfd138cf41 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectApprovalDao.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectApprovalDao.kt @@ -36,6 +36,7 @@ import com.tencent.devops.model.project.tables.TProjectApproval import com.tencent.devops.model.project.tables.records.TProjectApprovalRecord import com.tencent.devops.project.pojo.ProjectApprovalInfo import com.tencent.devops.project.pojo.ProjectCreateInfo +import com.tencent.devops.project.pojo.ProjectProperties import com.tencent.devops.project.pojo.ProjectUpdateInfo import com.tencent.devops.project.pojo.enums.ProjectAuthSecrecyStatus import org.jooq.DSLContext @@ -78,7 +79,8 @@ class ProjectApprovalDao { TIPS_STATUS, PROJECT_TYPE, PRODUCT_ID, - PRODUCT_NAME + PRODUCT_NAME, + PROPERTIES ).values( projectCreateInfo.projectName, projectCreateInfo.englishName, @@ -102,7 +104,10 @@ class ProjectApprovalDao { tipsStatus, projectCreateInfo.projectType, projectCreateInfo.productId, - projectCreateInfo.productName + projectCreateInfo.productName, + projectCreateInfo.properties?.let { + JsonUtil.toJson(it, false) + } ).onDuplicateKeyUpdate() .set(PROJECT_NAME, projectCreateInfo.projectName) .set(DESCRIPTION, projectCreateInfo.description) @@ -122,6 +127,9 @@ class ProjectApprovalDao { .set(PROJECT_TYPE, projectCreateInfo.projectType) .set(PRODUCT_ID, projectCreateInfo.productId) .set(PRODUCT_NAME, projectCreateInfo.productName) + .set(PROPERTIES, projectCreateInfo.properties?.let { + JsonUtil.toJson(it, false) + }) .execute() } } @@ -156,6 +164,9 @@ class ProjectApprovalDao { .set(PROJECT_TYPE, projectUpdateInfo.projectType) .set(PRODUCT_ID, projectUpdateInfo.productId) .set(PRODUCT_NAME, projectUpdateInfo.productName) + .set(PROPERTIES, projectUpdateInfo.properties?.let { + JsonUtil.toJson(it, false) + }) .where(ENGLISH_NAME.eq(projectUpdateInfo.englishName)) .execute() } @@ -187,6 +198,9 @@ class ProjectApprovalDao { .set(PROJECT_TYPE, projectApprovalInfo.projectType) .set(PRODUCT_ID, projectApprovalInfo.productId) .set(PRODUCT_NAME, projectApprovalInfo.productName) + .set(PROPERTIES, projectApprovalInfo.properties?.let { + JsonUtil.toJson(it, false) + }) .where(ENGLISH_NAME.eq(projectApprovalInfo.englishName)) .execute() } @@ -287,7 +301,8 @@ class ProjectApprovalDao { tipsStatus = tipsStatus, projectType = projectType, productId = productId, - productName = productName + productName = productName, + properties = properties?.let { JsonUtil.to(it, ProjectProperties::class.java) } ) } } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectDao.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectDao.kt index e29c002f4b0..75b337443c8 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectDao.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectDao.kt @@ -48,9 +48,6 @@ import com.tencent.devops.project.pojo.enums.ProjectAuthSecrecyStatus import com.tencent.devops.project.pojo.enums.ProjectChannelCode import com.tencent.devops.project.pojo.user.UserDeptDetail import com.tencent.devops.project.util.ProjectUtils -import java.net.URLDecoder -import java.time.LocalDateTime -import java.util.Locale import org.jooq.Condition import org.jooq.DSLContext import org.jooq.Record @@ -61,6 +58,9 @@ import org.jooq.Result import org.jooq.impl.DSL import org.jooq.impl.DSL.lower import org.springframework.stereotype.Repository +import java.net.URLDecoder +import java.time.LocalDateTime +import java.util.Locale @Suppress("ALL") @Repository diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/UserProjectResourceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/UserProjectResourceImpl.kt index 827713fc742..5dc5d7e84de 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/UserProjectResourceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/UserProjectResourceImpl.kt @@ -39,6 +39,7 @@ import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.project.api.user.UserProjectResource import com.tencent.devops.project.constant.ProjectMessageCode.PROJECT_NOT_EXIST import com.tencent.devops.project.pojo.OperationalProductVO +import com.tencent.devops.project.pojo.ProjectByConditionDTO import com.tencent.devops.project.pojo.ProjectCollation import com.tencent.devops.project.pojo.ProjectCreateExtInfo import com.tencent.devops.project.pojo.ProjectCreateInfo @@ -47,7 +48,6 @@ import com.tencent.devops.project.pojo.ProjectLogo import com.tencent.devops.project.pojo.ProjectSortType import com.tencent.devops.project.pojo.ProjectUpdateInfo import com.tencent.devops.project.pojo.ProjectVO -import com.tencent.devops.project.pojo.ProjectByConditionDTO import com.tencent.devops.project.pojo.Result import com.tencent.devops.project.pojo.enums.ProjectChannelCode import com.tencent.devops.project.pojo.enums.ProjectValidateType @@ -272,4 +272,8 @@ class UserProjectResourceImpl @Autowired constructor( projectService.getOperationalProductsByBgName(bgName) ) } + + override fun getPipelineDialect(userId: String, projectId: String): Result { + return Result(projectService.getPipelineDialect(projectId = projectId)) + } } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectApprovalService.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectApprovalService.kt index 788a42bde66..0bf229b60c9 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectApprovalService.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectApprovalService.kt @@ -41,6 +41,7 @@ import com.tencent.devops.project.dao.ProjectUpdateHistoryDao import com.tencent.devops.project.pojo.ProjectApprovalInfo import com.tencent.devops.project.pojo.ProjectCreateExtInfo import com.tencent.devops.project.pojo.ProjectCreateInfo +import com.tencent.devops.project.pojo.ProjectProperties import com.tencent.devops.project.pojo.ProjectUpdateInfo import com.tencent.devops.project.pojo.enums.ProjectApproveStatus import com.tencent.devops.project.pojo.enums.ProjectTipsStatus @@ -257,6 +258,15 @@ class ProjectApprovalService @Autowired constructor( params = arrayOf(projectId), defaultMessage = "project $projectId is not exist" ) + // 属性只能变更前端展示的,其他的字段由op变更 + val projectProperties = projectInfo.properties?.let { JsonUtil.to(it, ProjectProperties::class.java) } + val updateProjectProperties = if (projectApprovalInfo.properties != null) { + projectProperties?.copy( + pipelineDialect = projectApprovalInfo.properties!!.pipelineDialect + ) + } else { + projectProperties + } val projectUpdateInfo = with(projectApprovalInfo) { ProjectUpdateInfo( projectName = projectName, @@ -277,7 +287,8 @@ class ProjectApprovalService @Autowired constructor( ccAppName = projectInfo.ccAppName, kind = projectInfo.kind, projectType = projectType ?: 0, - productId = projectApprovalInfo.productId + productId = projectApprovalInfo.productId, + properties = updateProjectProperties ) } val logoAddress = projectUpdateInfo.logoAddress diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectService.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectService.kt index b7ac77aa00a..c59da263686 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectService.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectService.kt @@ -35,6 +35,7 @@ import com.tencent.devops.common.auth.api.pojo.SubjectScopeInfo import com.tencent.devops.model.project.tables.records.TProjectRecord import com.tencent.devops.project.pojo.OperationalProductVO import com.tencent.devops.project.pojo.ProjectBaseInfo +import com.tencent.devops.project.pojo.ProjectByConditionDTO import com.tencent.devops.project.pojo.ProjectCollation import com.tencent.devops.project.pojo.ProjectCreateExtInfo import com.tencent.devops.project.pojo.ProjectCreateInfo @@ -47,7 +48,6 @@ import com.tencent.devops.project.pojo.ProjectSortType import com.tencent.devops.project.pojo.ProjectUpdateCreatorDTO import com.tencent.devops.project.pojo.ProjectUpdateInfo import com.tencent.devops.project.pojo.ProjectVO -import com.tencent.devops.project.pojo.ProjectByConditionDTO import com.tencent.devops.project.pojo.Result import com.tencent.devops.project.pojo.enums.PluginDetailsDisplayOrder import com.tencent.devops.project.pojo.enums.ProjectChannelCode @@ -290,4 +290,6 @@ interface ProjectService { englishName: String, pluginDetailsDisplayOrder: List ): Boolean + + fun getPipelineDialect(projectId: String): String } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt index e8eb2e6042f..2052f0b2ff3 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt @@ -62,6 +62,7 @@ import com.tencent.devops.common.auth.code.ProjectAuthServiceCode import com.tencent.devops.common.client.Client import com.tencent.devops.common.client.ClientTokenService import com.tencent.devops.common.event.dispatcher.SampleEventDispatcher +import com.tencent.devops.common.pipeline.dialect.PipelineDialectType import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.service.Profile import com.tencent.devops.common.service.utils.LogUtils @@ -550,6 +551,13 @@ abstract class AbsProjectServiceImpl @Autowired constructor( ) } } + // 属性只能变更前端展示的,其他的字段由op变更 + val properties = projectInfo.properties?.let { JsonUtil.to(it, ProjectProperties::class.java) } + ?: ProjectProperties() + if (projectUpdateInfo.properties != null) { + projectUpdateInfo.properties = + properties.copy(pipelineDialect = projectUpdateInfo.properties!!.pipelineDialect) + } // 判断是否需要审批,当修改最大授权范围/权限敏感/关联运营产品时需要审批 val (finalNeedApproval, newApprovalStatus) = getUpdateApprovalStatus( needApproval = needApproval, @@ -570,7 +578,9 @@ abstract class AbsProjectServiceImpl @Autowired constructor( originalProjectName = projectInfo.projectName, modifiedProjectName = projectUpdateInfo.projectName, finalNeedApproval = finalNeedApproval, - beforeSubjectScopes = JsonUtil.to(projectInfo.subjectScopes, object : TypeReference>() {}), + beforeSubjectScopes = JsonUtil.to( + projectInfo.subjectScopes, object : TypeReference>() {} + ), afterSubjectScopes = subjectScopes )) { modifyProjectAuthResource(resourceUpdateInfo) @@ -620,8 +630,12 @@ abstract class AbsProjectServiceImpl @Autowired constructor( afterProjectName = projectUpdateInfo.projectName, beforeProductId = projectInfo.productId, afterProductId = projectUpdateInfo.productId, - beforeOrganization = with(projectInfo) { getOrganizationStr(bgName, businessLineName, deptName, centerName) }, - afterOrganization = with(projectUpdateInfo) { getOrganizationStr(bgName, businessLineName, deptName, centerName) }, + beforeOrganization = with(projectInfo) { + getOrganizationStr(bgName, businessLineName, deptName, centerName) + }, + afterOrganization = with(projectUpdateInfo) { + getOrganizationStr(bgName, businessLineName, deptName, centerName) + }, beforeSubjectScopes = projectInfo.subjectScopes, afterSubjectScopes = subjectScopesStr, operator = userId, @@ -726,7 +740,10 @@ abstract class AbsProjectServiceImpl @Autowired constructor( // 判断是否需要审批 return if (approveStatus.isSuccess()) { val isSubjectScopesChange = isSubjectScopesChange( - beforeSubjectScopes = JsonUtil.to(projectInfo.subjectScopes, object : TypeReference>() {}), + beforeSubjectScopes = JsonUtil.to( + projectInfo.subjectScopes, + object : TypeReference>() {} + ), afterSubjectScopes = afterSubjectScopes ) // 当项目创建成功,则只有最大授权范围和项目性质修改才审批 @@ -1627,6 +1644,11 @@ abstract class AbsProjectServiceImpl @Autowired constructor( return true } + override fun getPipelineDialect(projectId: String): String { + return getByEnglishName(englishName = projectId)?.properties?.pipelineDialect + ?: PipelineDialectType.CLASSIC.name + } + companion object { const val MAX_PROJECT_NAME_LENGTH = 64 private val logger = LoggerFactory.getLogger(AbsProjectServiceImpl::class.java)!! diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/util/ProjectUtils.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/util/ProjectUtils.kt index d8c69e97734..3fa1e431c43 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/util/ProjectUtils.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/util/ProjectUtils.kt @@ -138,6 +138,8 @@ object ProjectUtils { JsonUtil.to(it, object : TypeReference>() {}) } return with(tProjectRecord) { + val projectProperties = properties?.let { JsonUtil.to(it, ProjectProperties::class.java) } + val projectApprovalProperties = projectApprovalInfo?.properties ProjectDiffVO( id = id, projectId = projectId, @@ -192,7 +194,9 @@ object ProjectUtils { productId = productId, afterProductId = projectApprovalInfo?.productId, productName = beforeProductName, - afterProductName = projectApprovalInfo?.productName + afterProductName = projectApprovalInfo?.productName, + pipelineDialect = projectProperties?.pipelineDialect, + afterPipelineDialect = projectApprovalProperties?.pipelineDialect ) } } diff --git a/src/backend/ci/core/worker/build.gradle.kts b/src/backend/ci/core/worker/build.gradle.kts index 6d112723b4f..52f512667d4 100644 --- a/src/backend/ci/core/worker/build.gradle.kts +++ b/src/backend/ci/core/worker/build.gradle.kts @@ -29,7 +29,6 @@ subprojects { configurations.forEach { it.exclude(group = "com.perforce", module = "*") - it.exclude(group = "com.google.guava", module = "*") it.exclude(group = "com.googlecode.javaewah", module = "*") it.exclude(group = "com.vdurmont", module = "*") it.exclude(group = "com.github.ulisesbocchio", module = "*") diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/BuildPushDockerImageTask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/BuildPushDockerImageTask.kt index 896a0935e1c..2a796da7d2d 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/BuildPushDockerImageTask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/BuildPushDockerImageTask.kt @@ -56,11 +56,11 @@ import com.tencent.devops.worker.common.logger.LoggerService import com.tencent.devops.worker.common.task.ITask import com.tencent.devops.worker.common.task.TaskClassType import com.tencent.devops.worker.common.task.script.CommandFactory -import java.io.File import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.Request import okhttp3.RequestBody import org.slf4j.LoggerFactory +import java.io.File @TaskClassType(classTypes = [BuildPushDockerImageElement.classType]) @Suppress("ALL") @@ -150,8 +150,7 @@ class BuildPushDockerImageTask : ITask() { runtimeVariables = runtimeVariables, projectId = projectId, dir = workspace, - buildEnvs = buildVariables.buildEnvs, - asCodeEnabled = buildVariables.pipelineAsCodeSettings?.enable + buildEnvs = buildVariables.buildEnvs ) LoggerService.addNormalLine("Start to build the docker image. imageName:$imageName; imageTag:$imageTag") @@ -165,8 +164,7 @@ class BuildPushDockerImageTask : ITask() { runtimeVariables = runtimeVariables, projectId = projectId, dir = workspace, - buildEnvs = buildVariables.buildEnvs, - asCodeEnabled = buildVariables.pipelineAsCodeSettings?.enable + buildEnvs = buildVariables.buildEnvs ) } catch (t: RuntimeException) { val message = MessageUtil.getMessageByLocale( @@ -191,8 +189,7 @@ class BuildPushDockerImageTask : ITask() { runtimeVariables = runtimeVariables, projectId = projectId, dir = workspace, - buildEnvs = buildVariables.buildEnvs, - asCodeEnabled = buildVariables.pipelineAsCodeSettings?.enable + buildEnvs = buildVariables.buildEnvs ) } } diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt index 82c9adc42a3..9fa3abb4582 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt @@ -37,6 +37,7 @@ import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.MessageUtil import com.tencent.devops.common.pipeline.EnvReplacementParser import com.tencent.devops.common.pipeline.NameAndValue +import com.tencent.devops.common.pipeline.dialect.PipelineDialectUtil import com.tencent.devops.common.pipeline.enums.BuildFormPropertyType import com.tencent.devops.common.pipeline.enums.BuildTaskStatus import com.tencent.devops.common.pipeline.pojo.BuildParameters @@ -45,6 +46,7 @@ import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.process.pojo.BuildJobResult import com.tencent.devops.process.pojo.BuildTask import com.tencent.devops.process.pojo.BuildVariables +import com.tencent.devops.process.utils.PIPELINE_DIALECT import com.tencent.devops.process.utils.PIPELINE_RETRY_COUNT import com.tencent.devops.process.utils.PipelineVarUtil import com.tencent.devops.worker.common.constants.WorkerMessageCode.BK_PREPARE_TO_BUILD @@ -54,7 +56,6 @@ import com.tencent.devops.worker.common.constants.WorkerMessageCode.UNKNOWN_ERRO import com.tencent.devops.worker.common.env.AgentEnv import com.tencent.devops.worker.common.env.BuildEnv import com.tencent.devops.worker.common.env.BuildType -import com.tencent.devops.worker.common.env.DockerEnv import com.tencent.devops.worker.common.exception.TaskExecuteExceptionDecorator import com.tencent.devops.worker.common.expression.SpecialFunctions import com.tencent.devops.worker.common.heartbeat.Heartbeat @@ -66,10 +67,10 @@ import com.tencent.devops.worker.common.task.TaskFactory import com.tencent.devops.worker.common.utils.CredentialUtils import com.tencent.devops.worker.common.utils.KillBuildProcessTree import com.tencent.devops.worker.common.utils.ShellUtil +import org.slf4j.LoggerFactory import java.io.File import java.io.FileNotFoundException import java.io.IOException -import org.slf4j.LoggerFactory import kotlin.system.exitProcess object Runner { @@ -176,7 +177,7 @@ object Runner { if (endBuildFlag) { // 启动失败，尝试结束构建 try { - EngineService.endBuild(emptyMap(), DockerEnv.getBuildId(), BuildJobResult(ignored.message)) + EngineService.endBuild(emptyMap(), "", BuildJobResult(ignored.message)) } catch (ignored: Exception) { logger.warn("End build catch unknown exceptions", ignored) } @@ -444,6 +445,7 @@ object Runner { // 填充插件级的ENV参数 val customEnvStr = buildTask.params?.get(Element::customEnv.name) + val dialect = PipelineDialectUtil.getPipelineDialect(jobBuildVariables.variables[PIPELINE_DIALECT]) if (customEnvStr != null) { val customEnv = try { JsonUtil.toOrNull(customEnvStr, object : TypeReference>() {}) @@ -459,7 +461,7 @@ object Runner { val value = EnvReplacementParser.parse( value = it.value ?: "", contextMap = jobVariables, - onlyExpression = jobBuildVariables.pipelineAsCodeSettings?.enable, + onlyExpression = dialect.supportUseExpression(), functions = SpecialFunctions.functions, output = SpecialFunctions.output ) diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/CIKeywordsService.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/CIKeywordsService.kt index cc7c015dc3c..6d9a2a81be0 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/CIKeywordsService.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/CIKeywordsService.kt @@ -6,6 +6,7 @@ import com.tencent.devops.common.expression.context.StringContextData import com.tencent.devops.worker.common.CI_TOKEN_CONTEXT import com.tencent.devops.worker.common.api.ApiFactory import com.tencent.devops.worker.common.api.engine.EngineBuildSDKApi +import org.slf4j.LoggerFactory object CIKeywordsService { private val buildApi = ApiFactory.create(EngineBuildSDKApi::class) @@ -26,6 +27,8 @@ object CIKeywordsService { override val key: String = "ci" ) : RuntimeNamedValue { override fun getValue(key: String): PipelineContextData? { + // TODO: 理论上不会走这里的代码逻辑，先打印日志保守点看看 + logger.info("CIKeywordsRuntimeNamedValue|ci.token") // 不是需要的关键字直接返回空 if (key != CI_TOKEN_CONTEXT.removePrefix("ci.")) { return null @@ -33,4 +36,6 @@ object CIKeywordsService { return StringContextData(getOrRequestToken() ?: "") } } + + private val logger = LoggerFactory.getLogger(CIKeywordsService::class.java) } diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/ITask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/ITask.kt index 90b707e2e39..6fe338fd723 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/ITask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/ITask.kt @@ -30,21 +30,22 @@ package com.tencent.devops.worker.common.task import com.tencent.devops.common.api.exception.TaskExecuteException import com.tencent.devops.common.api.pojo.ErrorCode import com.tencent.devops.common.api.pojo.ErrorType +import com.tencent.devops.common.pipeline.dialect.IPipelineDialect +import com.tencent.devops.common.pipeline.dialect.PipelineDialectUtil import com.tencent.devops.common.pipeline.pojo.BuildParameters import com.tencent.devops.process.pojo.BuildTask import com.tencent.devops.process.pojo.BuildVariables +import com.tencent.devops.process.utils.PIPELINE_DIALECT +import com.tencent.devops.process.utils.PIPELINE_VARIABLES_STRING_LENGTH_MAX import com.tencent.devops.worker.common.env.BuildEnv import com.tencent.devops.worker.common.env.BuildType import com.tencent.devops.worker.common.logger.LoggerService import java.io.File +import java.util.regex.Pattern import java.util.stream.Collectors -import org.slf4j.LoggerFactory @Suppress("NestedBlockDepth", "TooManyFunctions") abstract class ITask { - - private val logger = LoggerFactory.getLogger(ITask::class.java) - private val environment = HashMap() private val monitorData = HashMap() @@ -57,6 +58,13 @@ abstract class ITask { /* 存储常量的key */ private lateinit var constVar: List + /* */ + private lateinit var dialect: IPipelineDialect + + companion object { + // 有的插件输出的变量会带taskId,taskId包含-,所以需要保留 + private const val ENGLISH_NAME_PATTERN = "[A-Za-z_][A-Za-z_0-9.-]*" + } fun run( buildTask: BuildTask, @@ -67,6 +75,7 @@ abstract class ITask { .filter { it.readOnly == true } .map { it.key } .collect(Collectors.toList()) + dialect = PipelineDialectUtil.getPipelineDialect(buildVariables.variables[PIPELINE_DIALECT]) execute(buildTask, buildVariables, workspace) } @@ -95,23 +104,60 @@ abstract class ITask { ) protected fun addEnv(env: Map) { - if (this::constVar.isInitialized) { - var errFlag = false - env.forEach { (key, _) -> - if (key in constVar) { - LoggerService.addErrorLine("Variable $key is read-only and cannot be modified.") - errFlag = true - } + var errReadOnlyFlag = false + var errLongValueFlag = false + val errLongValueVars = mutableSetOf() + var errChineseVarName = false + val errChineseVars = mutableSetOf() + env.forEach { (key, value) -> + if (this::constVar.isInitialized && key in constVar) { + LoggerService.addErrorLine("Variable $key is read-only and cannot be modified.") + errReadOnlyFlag = true + } + if (value.length >= PIPELINE_VARIABLES_STRING_LENGTH_MAX) { + errLongValueVars.add(key) + errLongValueFlag = true + } + if (!Pattern.matches(ENGLISH_NAME_PATTERN, key)) { + errChineseVars.add(key) + errChineseVarName = true + } + } + if (errReadOnlyFlag) { + throw TaskExecuteException( + errorMsg = "[Finish task] status: false, errorType: ${ErrorType.USER.num}, " + + "errorCode: ${ErrorCode.USER_INPUT_INVAILD}, message: read-only cannot be modified.", + errorType = ErrorType.USER, + errorCode = ErrorCode.USER_INPUT_INVAILD + ) + } + if (this::dialect.isInitialized) { + if (errLongValueFlag && !dialect.supportLongVarValue()) { + LoggerService.addWarnLine("Variable $errLongValueVars value exceeds 4000 length limit.") + throw TaskExecuteException( + errorMsg = "[Finish task] status: false, errorType: ${ErrorType.USER.num}, " + + "errorCode: ${ErrorCode.USER_INPUT_INVAILD}," + + " message: variable value exceeds 4000 length limit.", + errorType = ErrorType.USER, + errorCode = ErrorCode.USER_INPUT_INVAILD + ) } - if (errFlag) { + if (errChineseVarName && !dialect.supportChineseVarName()) { + LoggerService.addWarnLine( + "Variable $errChineseVars name is illegal,Variable names can only use letters, " + + "numbers and underscores, " + + "and the first character cannot start with a number" + ) throw TaskExecuteException( errorMsg = "[Finish task] status: false, errorType: ${ErrorType.USER.num}, " + - "errorCode: ${ErrorCode.USER_INPUT_INVAILD}, message: read-only cannot be modified.", + "errorCode: ${ErrorCode.USER_INPUT_INVAILD}," + + " message: variable name is illegal.", errorType = ErrorType.USER, errorCode = ErrorCode.USER_INPUT_INVAILD ) } } + environment.putAll(env) } diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt index 03800282e45..ee8b1851af7 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt @@ -45,16 +45,19 @@ import com.tencent.devops.common.api.constant.VALUE import com.tencent.devops.common.api.enums.OSType import com.tencent.devops.common.api.exception.RemoteServiceException import com.tencent.devops.common.api.exception.TaskExecuteException +import com.tencent.devops.common.api.exception.VariableNotFoundException import com.tencent.devops.common.api.factory.BkDiskLruFileCacheFactory import com.tencent.devops.common.api.pojo.ErrorCode import com.tencent.devops.common.api.pojo.ErrorType -import com.tencent.devops.common.api.util.EnvUtils import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.MessageUtil +import com.tencent.devops.common.api.util.ObjectReplaceEnvVarUtil import com.tencent.devops.common.api.util.ShaUtils import com.tencent.devops.common.archive.element.ReportArchiveElement import com.tencent.devops.common.pipeline.EnvReplacementParser import com.tencent.devops.common.pipeline.container.VMBuildContainer +import com.tencent.devops.common.pipeline.dialect.IPipelineDialect +import com.tencent.devops.common.pipeline.dialect.PipelineDialectUtil import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.service.utils.CommonUtils @@ -69,6 +72,7 @@ import com.tencent.devops.process.utils.PIPELINE_ATOM_CODE import com.tencent.devops.process.utils.PIPELINE_ATOM_NAME import com.tencent.devops.process.utils.PIPELINE_ATOM_TIMEOUT import com.tencent.devops.process.utils.PIPELINE_ATOM_VERSION +import com.tencent.devops.process.utils.PIPELINE_DIALECT import com.tencent.devops.process.utils.PIPELINE_START_USER_ID import com.tencent.devops.process.utils.PIPELINE_STEP_ID import com.tencent.devops.process.utils.PIPELINE_TASK_NAME @@ -156,10 +160,9 @@ open class MarketAtomTask : ITask() { val workspacePath = workspace.absolutePath // 输出参数的用户命名空间：防止重名窘况 val namespace: String? = map["namespace"] as String? - val asCodeEnabled = buildVariables.pipelineAsCodeSettings?.enable == true logger.info( "${buildTask.buildId}|RUN_ATOM|taskName=$taskName|ver=$atomVersion|code=$atomCode" + - "|workspace=$workspacePath|asCodeEnabled=$asCodeEnabled" + "|workspace=$workspacePath" ) // 获取插件基本信息 @@ -227,13 +230,14 @@ open class MarketAtomTask : ITask() { ) ) + val dialect = PipelineDialectUtil.getPipelineDialect(variables[PIPELINE_DIALECT]) // 解析并打印插件执行传入的所有参数 val inputParams = map["input"]?.let { input -> parseInputParams( inputMap = input as Map, variables = variables.plus(getContainerVariables(buildTask, buildVariables, workspacePath)), acrossInfo = acrossInfo, - asCodeEnabled = asCodeEnabled + dialect = dialect ) } ?: emptyMap() printInput(atomData, inputParams, inputTemplate) @@ -248,7 +252,7 @@ open class MarketAtomTask : ITask() { buildTask.stepId?.let { variables = variables.plus(PIPELINE_STEP_ID to it) } - val inputVariables = if (asCodeEnabled) { + val inputVariables = if (dialect.supportUseExpression()) { // 如果开启PAC,插件入参增加旧变量，防止开启PAC后,插件获取参数失败 PipelineVarUtil.mixOldVarAndNewVar(variables.toMutableMap()) } else { @@ -493,11 +497,11 @@ open class MarketAtomTask : ITask() { inputMap: Map, variables: Map, acrossInfo: BuildTemplateAcrossInfo?, - asCodeEnabled: Boolean + dialect: IPipelineDialect ): Map { val atomParams = mutableMapOf() try { - if (asCodeEnabled) { + if (dialect.supportUseExpression()) { val customReplacement = EnvReplacementParser.getCustomExecutionContextByMap( variables = variables, extendNamedValueMap = listOf( @@ -510,7 +514,7 @@ open class MarketAtomTask : ITask() { atomParams[name] = EnvReplacementParser.parse( value = JsonUtil.toJson(value), contextMap = variables, - onlyExpression = true, + dialect = dialect, contextPair = customReplacement, functions = SpecialFunctions.functions, output = SpecialFunctions.output @@ -519,12 +523,20 @@ open class MarketAtomTask : ITask() { } else { inputMap.forEach { (name, value) -> // 修复插件input环境变量替换问题 #5682 - atomParams[name] = EnvUtils.parseEnv( - command = JsonUtil.toJson(value), - data = variables + atomParams[name] = JsonUtil.toJson( + ObjectReplaceEnvVarUtil.replaceEnvVar(value, variables) ).parseCredentialValue(null, acrossInfo?.targetProjectId) } } + } catch (exception: VariableNotFoundException) { + val errMessage = "Variable ${exception.variableKey} not found" + LoggerService.addErrorLine(errMessage) + throw TaskExecuteException( + errorMsg = "[Finish task] status: false, errorType: ${ErrorType.USER.num}, " + + "errorCode: ${ErrorCode.USER_INPUT_INVAILD}, message: $errMessage", + errorType = ErrorType.USER, + errorCode = ErrorCode.USER_INPUT_INVAILD + ) } catch (e: Throwable) { logger.error("plugin input illegal! ", e) LoggerService.addErrorLine("plugin input illegal! ${e.message}") diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ICommand.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ICommand.kt index 23f7f94753a..6be2d7876a1 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ICommand.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ICommand.kt @@ -30,6 +30,8 @@ package com.tencent.devops.worker.common.task.script import com.tencent.devops.common.api.util.KeyReplacement import com.tencent.devops.common.api.util.ReplacementUtils import com.tencent.devops.common.pipeline.EnvReplacementParser +import com.tencent.devops.common.pipeline.dialect.PipelineDialectUtil +import com.tencent.devops.process.utils.PIPELINE_DIALECT import com.tencent.devops.process.utils.PipelineVarUtil import com.tencent.devops.store.pojo.app.BuildEnv import com.tencent.devops.worker.common.CI_TOKEN_CONTEXT @@ -58,8 +60,7 @@ interface ICommand { jobId: String? = null, stepId: String? = null, charsetType: String? = null, - taskId: String? = null, - asCodeEnabled: Boolean? = null + taskId: String? = null ) fun parseTemplate( @@ -67,8 +68,7 @@ interface ICommand { command: String, variables: Map, dir: File, - taskId: String?, - asCodeEnabled: Boolean? + taskId: String? ): String { // 解析跨项目模板信息 val acrossTargetProjectId by lazy { @@ -83,11 +83,12 @@ interface ICommand { ).toMutableMap() // 增加上下文的替换 PipelineVarUtil.fillContextVarMap(contextMap) - return if (asCodeEnabled == true) { + val dialect = PipelineDialectUtil.getPipelineDialect(variables[PIPELINE_DIALECT]) + return if (dialect.supportUseExpression()) { EnvReplacementParser.parse( value = command, contextMap = contextMap, - onlyExpression = true, + dialect = dialect, contextPair = EnvReplacementParser.getCustomExecutionContextByMap( variables = contextMap, extendNamedValueMap = listOf( diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ScriptTask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ScriptTask.kt index e24f5fe3c51..dda609c17ab 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ScriptTask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ScriptTask.kt @@ -29,6 +29,7 @@ package com.tencent.devops.worker.common.task.script import com.tencent.bkrepo.repository.pojo.token.TokenType import com.tencent.devops.common.api.exception.TaskExecuteException +import com.tencent.devops.common.api.exception.VariableNotFoundException import com.tencent.devops.common.api.pojo.ErrorCode import com.tencent.devops.common.api.pojo.ErrorCode.USER_SCRIPT_TASK_FAIL import com.tencent.devops.common.api.pojo.ErrorType @@ -118,8 +119,16 @@ open class ScriptTask : ITask() { continueNoneZero = continueNoneZero.toBoolean(), errorMessage = "Fail to run the plugin", charsetType = charsetType, - taskId = buildTask.taskId, - asCodeEnabled = buildVariables.pipelineAsCodeSettings?.enable + taskId = buildTask.taskId + ) + } catch (exception: VariableNotFoundException) { + val errMessage = "Variable ${exception.variableKey} not found" + LoggerService.addErrorLine(errMessage) + throw TaskExecuteException( + errorMsg = "[Finish task] status: false, errorType: ${ErrorType.USER.num}, " + + "errorCode: ${ErrorCode.USER_INPUT_INVAILD}, message: $errMessage", + errorType = ErrorType.USER, + errorCode = ErrorCode.USER_INPUT_INVAILD ) } catch (ignore: Throwable) { logger.warn("Fail to run the script task", ignore) diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/bat/CommandBatImpl.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/bat/CommandBatImpl.kt index d75ca55bf48..373ec9e566d 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/bat/CommandBatImpl.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/bat/CommandBatImpl.kt @@ -47,10 +47,9 @@ class CommandBatImpl : ICommand { jobId: String?, stepId: String?, charsetType: String?, - taskId: String?, - asCodeEnabled: Boolean? + taskId: String? ) { - val realCommand = parseTemplate(buildId, script, taskParam.plus(runtimeVariables), dir, taskId, asCodeEnabled) + val realCommand = parseTemplate(buildId, script, taskParam.plus(runtimeVariables), dir, taskId) BatScriptUtil.execute( buildId = buildId, script = realCommand, diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/powershell/CommandPowerShellImpl.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/powershell/CommandPowerShellImpl.kt index 2d39b0731bc..66b82888753 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/powershell/CommandPowerShellImpl.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/powershell/CommandPowerShellImpl.kt @@ -47,8 +47,7 @@ class CommandPowerShellImpl : ICommand { jobId: String?, stepId: String?, charsetType: String?, - taskId: String?, - asCodeEnabled: Boolean? + taskId: String? ) { TODO("Not yet implemented") } diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/shell/CommandShellImpl.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/shell/CommandShellImpl.kt index 43537a1b8d6..e9a62255f99 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/shell/CommandShellImpl.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/shell/CommandShellImpl.kt @@ -47,16 +47,14 @@ class CommandShellImpl : ICommand { jobId: String?, stepId: String?, charsetType: String?, - taskId: String?, - asCodeEnabled: Boolean? + taskId: String? ) { val realCommand = parseTemplate( buildId = buildId, command = script, variables = taskParam.plus(runtimeVariables), dir = dir, - taskId = taskId, - asCodeEnabled = asCodeEnabled + taskId = taskId ) ShellUtil.execute( buildId = buildId, diff --git a/support-files/i18n/message_en_US.properties b/support-files/i18n/message_en_US.properties index 364ae164953..49734706707 100644 --- a/support-files/i18n/message_en_US.properties +++ b/support-files/i18n/message_en_US.properties @@ -134,6 +134,7 @@ 2100133=Trigger [{0}] Merge Request Accept event type has been adjusted to action:merged for Merge Request event. Please switch version to 2.latest or above. 2100135=SVN token is incorrect or SVN path does not have permission 2100136=SVN token is empty, please check the credential type. +2100137=variable [{0}] not found 2189500=daemon interrupted 2189501=An error occurred in the internal service call of the system 2189502=The execution request timed out diff --git a/support-files/i18n/message_zh_CN.properties b/support-files/i18n/message_zh_CN.properties index 4748ae998c7..98d9323cb83 100644 --- a/support-files/i18n/message_zh_CN.properties +++ b/support-files/i18n/message_zh_CN.properties @@ -134,6 +134,7 @@ 2100133=触发器[{0}]Merge Request Accept事件类型已调整为Merge Request事件对应的监听动作:合并MR,请切换版本至2.latest及以上 2100135=SVN token 不正确 或者 SVN 路径没有权限 2100136=SVN Token 为空, 请检查代码库的凭证类型 +2100137=变量[${0}]不存在 2189500=守护进程中断 2189501=系统内部服务调用出错 2189502=执行请求超时 diff --git a/support-files/i18n/process/message_en_US.properties b/support-files/i18n/process/message_en_US.properties index 39b358a3b7a..10965e0ac2c 100644 --- a/support-files/i18n/process/message_en_US.properties +++ b/support-files/i18n/process/message_en_US.properties @@ -51,7 +51,7 @@ 2101051=Store image is used in the model, but code is empty 2101052=Store image is used in the model, but version is empty 2101053=Image value of non-store BK-CI / third party sources cannot be empty -2101054=Variable names can only use letters, numbers and underscores, and the first character cannot start with a number +2101054=Variable [{0}] names can only use letters, numbers and underscores, and the first character cannot start with a number 2101055=Stage [{0}] admission configuration is incorrect 2101056=The pipeline description is too long. 2101057=The timing parameter [{0}] of a timing trigger cannot be triggered in seconds. @@ -247,6 +247,7 @@ 2101253=Condition expression in [{0}] is longer than {1} digits 2101254=Parameter {0} is required and cannot be empty 2101255=Start the build machine repeatedly. The current status of the build machine is: {0} +2101256=pipeline build parameter {0} value exceeds 4000 length limit ATOM_POST_EXECUTE_TIP=###Tip:this is the post-action hooked by [step{0}]{1}### # 公共变量 diff --git a/support-files/i18n/process/message_zh_CN.properties b/support-files/i18n/process/message_zh_CN.properties index 27f134c4dd6..e6dfa2b9517 100644 --- a/support-files/i18n/process/message_zh_CN.properties +++ b/support-files/i18n/process/message_zh_CN.properties @@ -51,7 +51,7 @@ 2101051=模型中使用了商店镜像，但code为空 2101052=模型中使用了商店镜像，但version为空 2101053=非商店蓝盾源/第三方源的镜像value不可为空 -2101054=变量名只能使用英文字母，数字和下划线，首字符不能以数字开头 +2101054=变量名[{0}]只能使用英文字母，数字和下划线，首字符不能以数字开头 2101055=Stage[{0}]准入配置不正确 2101056=流水线描述过长 2101057=定时触发器的定时参数[{0}]不能秒级触发 @@ -247,6 +247,7 @@ 2101253=[{0}]的条件表达式的长度超过{1}位 2101254=构建入参[{0}]必填，不能为空 2101255=重复启动构建机，当前构建机的状态为：{0} +2101256=流水线变量{0}值超出4000长度限制 ATOM_POST_EXECUTE_TIP=###Tip:this is the post-action hooked by [step{0}]{1}### ci.build-no=构建号，开启推荐版本号时有效 diff --git a/support-files/sql/1001_ci_project_ddl_mysql.sql b/support-files/sql/1001_ci_project_ddl_mysql.sql index 05ffb1d89de..377a2e01b5c 100644 --- a/support-files/sql/1001_ci_project_ddl_mysql.sql +++ b/support-files/sql/1001_ci_project_ddl_mysql.sql @@ -374,6 +374,7 @@ CREATE TABLE IF NOT EXISTS `T_PROJECT_APPROVAL` ( `PROJECT_TYPE` int(10) DEFAULT NULL COMMENT '项目类型', `PRODUCT_ID` int(10) DEFAULT NULL COMMENT '运营产品ID', `PRODUCT_NAME` VARCHAR(64) DEFAULT NULL comment '运营产品名称', + `PROPERTIES` text null comment '项目其他配置', PRIMARY KEY (`ID`) USING BTREE, UNIQUE KEY `project_name` (`PROJECT_NAME`) USING BTREE, UNIQUE KEY `english_name` (`ENGLISH_NAME`) USING BTREE diff --git a/support-files/sql/2004_v3.x/2030_ci_project-update_v3.0_mysql.sql b/support-files/sql/2004_v3.x/2030_ci_project-update_v3.0_mysql.sql index a0b29441394..436aefe0196 100644 --- a/support-files/sql/2004_v3.x/2030_ci_project-update_v3.0_mysql.sql +++ b/support-files/sql/2004_v3.x/2030_ci_project-update_v3.0_mysql.sql @@ -20,6 +20,15 @@ BEGIN ADD COLUMN `DOC_URL` VARCHAR(255) NOT NULL DEFAULT '' COMMENT '文档链接'; END IF; + IF NOT EXISTS(SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = db + AND TABLE_NAME = 'T_PROJECT_APPROVAL' + AND COLUMN_NAME = 'PROPERTIES') THEN + ALTER TABLE T_PROJECT_APPROVAL + ADD COLUMN `PROPERTIES` text null DEFAULT NULL comment '项目其他配置'; + END IF; + COMMIT; END DELIMITER ;