feat：流水线变量语法支持两种风格 #10576

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt

@@ -185,6 +185,7 @@ object CommonMessageCode {
 
     const val SVN_TOKEN_FAIL = "2100135" // SVN Token 不正确
     const val SVN_TOKEN_EMPTY = "2100136" // SVN Token 为空, 请检查代码库的凭证类型
+    const val ERROR_VARIABLE_NOT_FOUND = "2100137" // SVN Token 为空, 请检查代码库的凭证类型
     const val BK_CONTAINER_TIMED_OUT = "bkContainerTimedOut" // 创建容器超时
     const val BK_CREATION_FAILED_EXCEPTION_INFORMATION = "bkCreationFailedExceptionInformation" // 创建失败，异常信息
 

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/exception/VariableNotFoundException.kt

@@ -0,0 +1,43 @@
+/*
+ * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.
+ *
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company.  All rights reserved.
+ *
+ * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.
+ *
+ * A copy of the MIT License is included in this file.
+ *
+ *
+ * Terms of the MIT License:
+ * ---------------------------------------------------
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+ * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all copies or substantial portions of
+ * the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+ * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
+ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package com.tencent.devops.common.api.exception
+
+import com.tencent.devops.common.api.constant.CommonMessageCode
+
+/**
+ * 变量不存在异常
+ */
+class VariableNotFoundException(
+    val variableKey: String?,
+    errorCode: String = CommonMessageCode.ERROR_VARIABLE_NOT_FOUND
+) :
+    ErrorCodeException(
+        errorCode = errorCode,
+        defaultMessage = "variable $variableKey not found",
+        params = variableKey?.let { arrayOf(it) }
+    )

src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/pojo/PipelineAsCodeSettings.kt

@@ -32,5 +32,35 @@ import io.swagger.v3.oas.annotations.media.Schema
 @Schema(title = "设置-YAML流水线功能设置")
 data class PipelineAsCodeSettings(
     @get:Schema(title = "是否支持YAML流水线功能", required = true)
-    val enable: Boolean = false
-)
+    val enable: Boolean = false,
+    @get:Schema(title = "项目级流水线语法风格", required = false)
+    var projectDialect: String? = null,
+    @get:Schema(title = "是否继承项目流水线语言风格", required = false)
+    val inheritedDialect: Boolean? = true,
+    @get:Schema(title = "流水线语言风格", required = false)
+    var pipelineDialect: String? = null
+) {
+    companion object {
+        fun initDialect(inheritedDialect: Boolean?, pipelineDialect: String?): PipelineAsCodeSettings {
+            return PipelineAsCodeSettings(
+                inheritedDialect = inheritedDialect ?: true,
+                // 如果继承项目方言配置,置空pipelineDialect字段,防止数据库存储多余数据
+                pipelineDialect = if (inheritedDialect == false) {
+                    pipelineDialect
+                } else {
+                    null
+                }
+            )
+        }
+    }
+
+    /**
+     * 入库时,重置方言字段值
+     */
+    fun resetDialect() {
+        projectDialect = null
+        if (inheritedDialect != false) {
+            pipelineDialect = null
+        }
+    }
+}

src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionException.kt

@@ -82,12 +82,6 @@ class FunctionFormatException(override val message: String?) : ExpressionExcepti
     }
 }
 
-class ContextNotFoundException(override val message: String?) : ExpressionException() {
-    companion object {
-        fun contextNameNotFound(arg0: String) = ContextNotFoundException(
-            "Expression context $arg0 not found."
-        )
-    }
-}
+class ContextNotFoundException(override var message: String? = null) : ExpressionException()
 
 class ContextJsonFormatException(override val message: String?) : ExpressionException()

src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionParser.kt

@@ -27,9 +27,11 @@
 
 package com.tencent.devops.common.expression
 
+import com.tencent.devops.common.expression.ExpressionParser.legalizeExpression
 import com.tencent.devops.common.expression.context.ContextValueNode
 import com.tencent.devops.common.expression.context.DictionaryContextData
 import com.tencent.devops.common.expression.context.PipelineContextData
+import com.tencent.devops.common.expression.expression.EvaluationOptions
 import com.tencent.devops.common.expression.expression.ExpressionConstants
 import com.tencent.devops.common.expression.expression.IExpressionNode
 import com.tencent.devops.common.expression.expression.IFunctionInfo
@@ -74,8 +76,9 @@ object ExpressionParser {
         nameValue: List<NamedValueInfo>,
         fetchValue: Boolean
     ): Any? {
+        val options = EvaluationOptions(false)
         val result = createTree(expression.legalizeExpression(), null, nameValue, null)!!
-            .evaluate(null, context, null, null)
+            .evaluate(null, context, options, null)
         if (!fetchValue) {
             return result
         }
@@ -90,10 +93,32 @@ object ExpressionParser {
         val context = ExecutionContext(DictionaryContextData())
         val nameValue = mutableListOf<NamedValueInfo>()
         fillContextByMap(contextMap, context, nameValue)
+        val options = EvaluationOptions(false)
+        try {
+            return doEvaluateByMap(
+                expression = expression,
+                context = context,
+                nameValue = nameValue,
+                options = options,
+                fetchValue = fetchValue
+            )
+        } catch (e: Throwable) {
+            if (options.contextNotNull() && e is ContextNotFoundException) {
+                throw ContextNotFoundException("Expression context ${options.contextNotNull.errKey()} not found.")
+            }
+            throw e
+        }
+    }
 
+    private fun doEvaluateByMap(
+        expression: String,
+        context: ExecutionContext,
+        nameValue: MutableList<NamedValueInfo>,
+        options: EvaluationOptions,
+        fetchValue: Boolean
+    ): Any? {
         val result = createTree(expression.legalizeExpression(), null, nameValue, null)!!
-            .evaluate(null, context, null, null)
-
+            .evaluate(null, context, options, null)
         if (!fetchValue) {
             return result
         }
@@ -104,8 +129,9 @@ object ExpressionParser {
     /**
      * 将流水线变量转换为表达式上下文类型，存在如下情况
      * 1、a = str, 直接使用 string 类型的上下文保存即可
-     * 2、a.b.c = str, 将 a.b.c 升格为上下文中的嵌套 map 保存 既 a {b: {c: str}}
-     * 3、a.b.c = str 且 a.b = {"c": "str"}, 需要校验 a.b 所保存的 json 与 a.b.c 结构和数据是否相同后再升格
+     * 2、a.b.c = str, 将 a.b.c 转换为上下文中的嵌套 map 保存 既 a {b: {c: str}}
+     * 3、a.b.c = str 且 a.b = {"c": "str"}, 将 a.b 保存为兼容用户的数据类型，在不涉及引擎计算纯输出的情况下使用 a.b 的原数据，
+     * 在涉及引擎计算时，则使用 a.b.c 转换后的 map，同 2 中所述
      */
     fun fillContextByMap(
         contextMap: Map<String, String>,

src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/AbsDictionaryContextData.kt

@@ -1,18 +1,15 @@
 package com.tencent.devops.common.expression.context
 
 import com.fasterxml.jackson.databind.JsonNode
+import com.tencent.devops.common.expression.expression.sdk.CollectionPipelineResult
 import com.tencent.devops.common.expression.expression.sdk.IReadOnlyObject
 import com.tencent.devops.common.expression.utils.ExpressionJsonUtil
 import java.util.TreeMap
 
 /**
  * dict 的抽象类，总结公共方法
  */
-abstract class AbsDictionaryContextData :
-    PipelineContextData(PipelineContextDataType.DICTIONARY),
-    Iterable<Pair<String, PipelineContextData?>>,
-    IReadOnlyObject {
-
+abstract class AbsDictionaryContextData : PipelineContextData(PipelineContextDataType.DICTIONARY), IReadOnlyObject {
     protected open var mIndexLookup: TreeMap<String, Int>? = null
     protected open var mList: MutableList<DictionaryContextDataPair> = mutableListOf()
 
@@ -24,14 +21,6 @@ abstract class AbsDictionaryContextData :
             return emptyList()
         }
 
-    override fun tryGetValue(key: String): Pair<Any?, Boolean> {
-        if (mList.isNotEmpty() && indexLookup.containsKey(key)) {
-            return Pair(mList[indexLookup[key]!!].value, true)
-        }
-
-        return Pair(null, false)
-    }
-
     protected val indexLookup: MutableMap<String, Int>
         get() {
             if (mIndexLookup == null) {
@@ -51,6 +40,29 @@ abstract class AbsDictionaryContextData :
             return mList
         }
 
+    override operator fun get(key: String): PipelineContextData? {
+        val index = indexLookup[key] ?: return null
+        return list[index].value
+    }
+
+    override fun getRes(key: String): CollectionPipelineResult {
+        return if (containsKey(key)) {
+            CollectionPipelineResult(list.getOrNull(indexLookup[key]!!)?.value)
+        } else {
+            CollectionPipelineResult.noKey()
+        }
+    }
+
+    override fun toJson(): JsonNode {
+        val json = ExpressionJsonUtil.createObjectNode()
+        if (mList.isNotEmpty()) {
+            mList.forEach {
+                json.set<JsonNode>(it.key, it.value?.toJson())
+            }
+        }
+        return json
+    }
+
     operator fun set(k: String, value: PipelineContextData?) {
         // Existing
         val index = indexLookup[k]
@@ -64,22 +76,6 @@ abstract class AbsDictionaryContextData :
         }
     }
 
-    open operator fun get(k: String): PipelineContextData? {
-        // Existing
-        val index = indexLookup[k] ?: return null
-        return list[index].value
-    }
-
-    open operator fun IReadOnlyObject.get(key: String): Any? {
-        val index = indexLookup[key] ?: return null
-        return list[index].value
-    }
-
-    operator fun Pair<String, PipelineContextData>.get(key: Int): Pair<String, PipelineContextData?> {
-        val pair = mList[key]
-        return Pair(pair.key, pair.value)
-    }
-
     fun add(pairs: Iterable<Pair<String, PipelineContextData>>) {
         pairs.forEach { pair ->
             add(pair.first, pair.second)
@@ -94,18 +90,8 @@ abstract class AbsDictionaryContextData :
         list.add(DictionaryContextDataPair(key, value))
     }
 
-    override fun iterator(): Iterator<Pair<String, PipelineContextData?>> {
-        return mList.map { pair -> Pair(pair.key, pair.value); }.iterator()
-    }
-
-    override fun toJson(): JsonNode {
-        val json = ExpressionJsonUtil.createObjectNode()
-        if (mList.isNotEmpty()) {
-            mList.forEach {
-                json.set<JsonNode>(it.key, it.value?.toJson())
-            }
-        }
-        return json
+    fun containsKey(key: String): Boolean {
+        return mList.isNotEmpty() && indexLookup.containsKey(key)
     }
 
     protected data class DictionaryContextDataPair(

src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/ArrayContextData.kt

@@ -28,6 +28,7 @@
 package com.tencent.devops.common.expression.context
 
 import com.fasterxml.jackson.databind.JsonNode
+import com.tencent.devops.common.expression.expression.sdk.CollectionPipelineResult
 import com.tencent.devops.common.expression.expression.sdk.IReadOnlyArray
 import com.tencent.devops.common.expression.utils.ExpressionJsonUtil
 
@@ -40,11 +41,14 @@ class ArrayContextData : PipelineContextData(PipelineContextDataType.ARRAY), IRe
     override val count: Int
         get() = mItems.count()
 
-    fun add(item: PipelineContextData?) {
-        mItems.add(item)
-    }
+    override operator fun get(index: Int): PipelineContextData? = mItems[index]
 
-    override fun get(index: Int): Any? = mItems[index]
+    override fun getRes(index: Int): CollectionPipelineResult {
+        if (index >= 0 && index <= mItems.lastIndex) {
+            return CollectionPipelineResult(mItems[index])
+        }
+        return CollectionPipelineResult.noKey()
+    }
 
     override fun clone(): PipelineContextData {
         val result = ArrayContextData()
@@ -71,9 +75,17 @@ class ArrayContextData : PipelineContextData(PipelineContextDataType.ARRAY), IRe
         val list = mutableListOf<Any>()
         if (mItems.isNotEmpty()) {
             mItems.forEach {
+                if (it is DictionaryContextDataWithVal) {
+                    list.add(it.fetchValueNative())
+                    return@forEach
+                }
                 list.add(it?.fetchValue() ?: return@forEach)
             }
         }
         return list
     }
+
+    fun add(item: PipelineContextData?) {
+        mItems.add(item)
+    }
 }

src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/ContextValueNode.kt

@@ -27,14 +27,20 @@
 
 package com.tencent.devops.common.expression.context
 
+import com.tencent.devops.common.expression.ContextNotFoundException
 import com.tencent.devops.common.expression.ExecutionContext
 import com.tencent.devops.common.expression.expression.sdk.EvaluationContext
 import com.tencent.devops.common.expression.expression.sdk.NamedValue
 import com.tencent.devops.common.expression.expression.sdk.ResultMemory
 
 class ContextValueNode : NamedValue() {
     override fun evaluateCore(context: EvaluationContext): Pair<ResultMemory?, Any?> {
-        return Pair(null, (context.state as ExecutionContext).expressionValues[name])
+        val value = (context.state as ExecutionContext).expressionValues.getRes(name)
+        if (context.options.contextNotNull() && value.noKey()) {
+            context.options.contextNotNull.trace(name)
+            throw ContextNotFoundException()
+        }
+        return Pair(null, value.value)
     }
 
     override fun createNode(): NamedValue {
@@ -43,7 +49,7 @@ class ContextValueNode : NamedValue() {
 
     override fun subNameValueEvaluateCore(context: EvaluationContext): Pair<Any?, Boolean> {
         val values = (context.state as ExecutionContext).expressionValues
-        return if (values[name] != null) {
+        return if (values.containsKey(name)) {
             Pair(values[name], true)
         } else {
             Pair(name, false)

src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextData.kt

@@ -46,10 +46,14 @@ open class DictionaryContextData : AbsDictionaryContextData() {
         return result
     }
 
-    override fun fetchValue(): Map<String, Any> {
+    override fun fetchValue(): Any {
         val map = mutableMapOf<String, Any>()
         if (mList.isNotEmpty()) {
             mList.forEach {
+                if (it.value is DictionaryContextDataWithVal) {
+                    map[it.key] = it.value.fetchValueNative()
+                    return@forEach
+                }
                 map[it.key] = it.value?.fetchValue() ?: ""
             }
         }