Extending value evaluator in the python frontend (#1947)

* Extending value evaluator in the python frontend

This PR extends the value evaluator with python specifics. This also makes the `cpg-analysis` mandatory for the python language frontend.

* Added SystemInformation as an overlay node

* Added more tests

* Test

* Added arithmetic test

* ++

* Using handleHasInitializer for all things with initializers

* Update cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/SystemInformation.kt

Co-authored-by: KuechA <31155350+KuechA@users.noreply.github.com>

---------

Co-authored-by: KuechA <31155350+KuechA@users.noreply.github.com>

Author: oxisto

cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/MultiValueEvaluator.kt

@@ -64,14 +64,12 @@ class MultiValueEvaluator : ValueEvaluator() {
         this.path += node
 
         when (node) {
-            is FieldDeclaration -> {
-                return evaluateInternal(node.initializer, depth + 1)
-            }
-            is NewArrayExpression -> return evaluateInternal(node.initializer, depth + 1)
-            is VariableDeclaration -> return handleVariableDeclaration(node, depth)
+            is FieldDeclaration -> return handleHasInitializer(node, depth)
+            is NewArrayExpression -> return handleHasInitializer(node, depth)
+            is VariableDeclaration -> return handleHasInitializer(node, depth)
             // For a literal, we can just take its value, and we are finished
             is Literal<*> -> return node.value
-            is Reference -> return handleReference(node, depth)
+            is Reference -> return handlePrevDFG(node, depth)
             is UnaryOperator -> return handleUnaryOp(node, depth)
             is AssignExpression -> return handleAssignExpression(node, depth)
             is BinaryOperator -> return handleBinaryOperator(node, depth)
@@ -206,25 +204,30 @@ class MultiValueEvaluator : ValueEvaluator() {
     }
 
     /**
-     * Tries to compute the value of a reference. It therefore checks the incoming data flow edges.
+     * Tries to compute the value of a node based on its [Node.prevDFG].
      *
      * In contrast to the implementation of [ValueEvaluator], this one can handle more than one
      * value.
      */
-    override fun handleReference(expr: Reference, depth: Int): Collection<Any?> {
+    override fun handlePrevDFG(node: Node, depth: Int): Collection<Any?> {
         // For a reference, we are interested in its last assignment into the reference
         // denoted by the previous DFG edge. We need to filter out any self-references for READWRITE
         // references.
-        val prevDFG = filterSelfReferences(expr, expr.prevDFG.toList())
+        val prevDFG =
+            if (node is Reference) {
+                filterSelfReferences(node, node.prevDFG.toList())
+            } else {
+                node.prevDFG
+            }
 
         if (prevDFG.size == 1) {
             // There's only one incoming DFG edge, so we follow this one.
             val internalRes = evaluateInternal(prevDFG.first(), depth + 1)
             return (internalRes as? Collection<*>) ?: mutableSetOf(internalRes)
         }
 
-        if (prevDFG.size == 2 && prevDFG.all(::isSimpleForLoop)) {
-            return handleSimpleLoopVariable(expr, depth)
+        if (node is Reference && prevDFG.size == 2 && prevDFG.all(::isSimpleForLoop)) {
+            return handleSimpleLoopVariable(node, depth)
         }
 
         val result = mutableSetOf<Any?>()

cpg-analysis/src/main/kotlin/de/fraunhofer/aisec/cpg/analysis/ValueEvaluator.kt

@@ -26,6 +26,7 @@
 package de.fraunhofer.aisec.cpg.analysis
 
 import de.fraunhofer.aisec.cpg.graph.AccessValues
+import de.fraunhofer.aisec.cpg.graph.HasInitializer
 import de.fraunhofer.aisec.cpg.graph.HasOperatorCode
 import de.fraunhofer.aisec.cpg.graph.Node
 import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration
@@ -74,7 +75,7 @@ open class ValueEvaluator(
     open fun evaluate(node: Any?): Any? {
         if (node !is Node) return node
 
-        return evaluateInternal(node as? Node, 0)
+        return evaluateInternal(node, 0)
     }
 
     fun clearPath() {
@@ -87,11 +88,11 @@ open class ValueEvaluator(
         node?.let { this.path += it }
 
         when (node) {
-            is NewArrayExpression -> return evaluateInternal(node.initializer, depth)
-            is VariableDeclaration -> return handleVariableDeclaration(node, depth)
+            is NewArrayExpression -> return handleHasInitializer(node, depth)
+            is VariableDeclaration -> return handleHasInitializer(node, depth)
             // For a literal, we can just take its value, and we are finished
             is Literal<*> -> return node.value
-            is Reference -> return handleReference(node, depth)
+            is Reference -> return handlePrevDFG(node, depth)
             is UnaryOperator -> return handleUnaryOp(node, depth)
             is BinaryOperator -> return handleBinaryOperator(node, depth)
             // Casts are just a wrapper in this case, we are interested in the inner expression
@@ -108,10 +109,17 @@ open class ValueEvaluator(
         return cannotEvaluate(node, this)
     }
 
-    protected fun handleVariableDeclaration(node: VariableDeclaration, depth: Int): Any? {
-        // If we have an initializer, we can use it. However, we actually should just use the DFG
-        // instead and do something similar to handleReference
-        return evaluateInternal(node.initializer, depth + 1)
+    /**
+     * If a node declaration implements [HasInitializer], we can use the initializer to evaluate
+     * their value. If not, we can try to use [handlePrevDFG].
+     */
+    protected fun handleHasInitializer(node: HasInitializer, depth: Int): Any? {
+        // If we have an initializer, we can use it. Otherwise, we can fall back to the prevDFG
+        return if (node.initializer != null) {
+            evaluateInternal(node.initializer, depth + 1)
+        } else {
+            handlePrevDFG(node as Node, depth)
+        }
     }
 
     /** Under certain circumstances, an assignment can also be used as an expression. */
@@ -259,51 +267,63 @@ open class ValueEvaluator(
         }
     }
 
-    private fun handleGreater(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
+    protected open fun handleGreater(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
         return if (lhsValue is Number && rhsValue is Number) {
             lhsValue.compareTo(rhsValue) > 0
         } else {
             cannotEvaluate(expr, this)
         }
     }
 
-    private fun handleGEq(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
+    protected open fun handleGEq(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
         return if (lhsValue is Number && rhsValue is Number) {
             lhsValue.compareTo(rhsValue) >= 0
         } else {
             cannotEvaluate(expr, this)
         }
     }
 
-    private fun handleLess(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
+    protected open fun handleLess(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
         return if (lhsValue is Number && rhsValue is Number) {
             lhsValue.compareTo(rhsValue) < 0
         } else {
             cannotEvaluate(expr, this)
         }
     }
 
-    private fun handleLEq(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
+    protected open fun handleLEq(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
         return if (lhsValue is Number && rhsValue is Number) {
             lhsValue.compareTo(rhsValue) <= 0
         } else {
             cannotEvaluate(expr, this)
         }
     }
 
-    private fun handleEq(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
-        return if (lhsValue is Number && rhsValue is Number) {
-            lhsValue.compareTo(rhsValue) == 0
-        } else {
-            cannotEvaluate(expr, this)
+    protected open fun handleEq(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
+        return when {
+            lhsValue is Number && rhsValue is Number -> {
+                lhsValue.compareTo(rhsValue) == 0
+            }
+            lhsValue is String && rhsValue is String -> {
+                lhsValue == rhsValue
+            }
+            else -> {
+                cannotEvaluate(expr, this)
+            }
         }
     }
 
-    private fun handleNEq(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
-        return if (lhsValue is Number && rhsValue is Number) {
-            lhsValue.compareTo(rhsValue) != 0
-        } else {
-            cannotEvaluate(expr, this)
+    protected open fun handleNEq(lhsValue: Any?, rhsValue: Any?, expr: Expression?): Any? {
+        return when {
+            lhsValue is Number && rhsValue is Number -> {
+                lhsValue.compareTo(rhsValue) != 0
+            }
+            lhsValue is String && rhsValue is String -> {
+                lhsValue != rhsValue
+            }
+            else -> {
+                cannotEvaluate(expr, this)
+            }
         }
     }
 
@@ -390,15 +410,17 @@ open class ValueEvaluator(
         return cannotEvaluate(expr, this)
     }
 
-    /**
-     * Tries to compute the constant value of a reference. It therefore checks the incoming data
-     * flow edges.
-     */
-    protected open fun handleReference(expr: Reference, depth: Int): Any? {
+    /** Tries to compute the constant value of a node based on its [Node.prevDFG]. */
+    protected open fun handlePrevDFG(node: Node, depth: Int): Any? {
         // For a reference, we are interested into its last assignment into the reference
         // denoted by the previous DFG edge. We need to filter out any self-references for READWRITE
         // references.
-        val prevDFG = filterSelfReferences(expr, expr.prevDFG.toList())
+        val prevDFG =
+            if (node is Reference) {
+                filterSelfReferences(node, node.prevDFG.toList())
+            } else {
+                node.prevDFG
+            }
 
         return if (prevDFG.size == 1) {
             // There's only one incoming DFG edge, so we follow this one.
@@ -407,13 +429,13 @@ open class ValueEvaluator(
             // We cannot have more than ONE valid solution, so we need to abort
             log.warn(
                 "We cannot evaluate {}: It has more than 1 previous DFG edges, meaning that the value is probably affected by a branch.",
-                expr,
+                node,
             )
-            cannotEvaluate(expr, this)
+            cannotEvaluate(node, this)
         } else {
             // No previous DFG node
-            log.warn("We cannot evaluate {}: It has no previous DFG edges.", expr)
-            cannotEvaluate(expr, this)
+            log.warn("We cannot evaluate {}: It has no previous DFG edges.", node)
+            cannotEvaluate(node, this)
         }
     }
 

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/statements/expressions/NewArrayExpression.kt

@@ -25,6 +25,7 @@
  */
 package de.fraunhofer.aisec.cpg.graph.statements.expressions
 
+import de.fraunhofer.aisec.cpg.graph.HasInitializer
 import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration
 import de.fraunhofer.aisec.cpg.graph.edges.Edge.Companion.propertyEqualsList
 import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf
@@ -38,14 +39,14 @@ import org.neo4j.ogm.annotation.Relationship
  * combination with a [VariableDeclaration].
  */
 // TODO Merge and/or refactor with new Expression
-class NewArrayExpression : Expression() {
+class NewArrayExpression : Expression(), HasInitializer {
     @Relationship("INITIALIZER") var initializerEdge = astOptionalEdgeOf<Expression>()
 
     /**
      * The initializer of the expression, if present. Many languages, such as Java, either specify
      * [dimensions] or an initializer.
      */
-    var initializer by unwrapping(NewArrayExpression::initializerEdge)
+    override var initializer by unwrapping(NewArrayExpression::initializerEdge)
 
     /**
      * Specifies the dimensions of the array that is to be created. Many languages, such as Java,

cpg-language-python/build.gradle.kts

@@ -43,6 +43,9 @@ dependencies {
     // jep for python support
     implementation(libs.jep)
 
+    // the cpg-analysis project helps supporting a dynamically invoked language
+    implementation(projects.cpgAnalysis)
+
     // to evaluate some test cases
-    testImplementation(project(":cpg-analysis"))
+    testImplementation(projects.cpgAnalysis)
 }

cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt

@@ -25,6 +25,7 @@
  */
 package de.fraunhofer.aisec.cpg.frontends.python
 
+import de.fraunhofer.aisec.cpg.TranslationConfiguration
 import de.fraunhofer.aisec.cpg.TranslationContext
 import de.fraunhofer.aisec.cpg.frontends.Language
 import de.fraunhofer.aisec.cpg.frontends.LanguageFrontend
@@ -48,6 +49,19 @@ import kotlin.io.path.pathString
 import kotlin.io.path.relativeToOrNull
 import kotlin.math.min
 
+/**
+ * The [LanguageFrontend] for Python. It uses the JEP library to interact with Python's AST.
+ *
+ * It requires the Python interpreter (and the JEP library) to be installed on the system. The
+ * frontend registers two additional passes.
+ *
+ * ## Adding dynamic variable declarations
+ *
+ * The [PythonAddDeclarationsPass] adds dynamic declarations to the CPG. Python does not have the
+ * concept of a "declaration", but rather values are assigned to variables and internally variable
+ * are represented by a dictionary. This pass adds a declaration for each variable that is assigned
+ * a value (on the first assignment).
+ */
 @RegisterExtraPass(PythonAddDeclarationsPass::class)
 class PythonLanguageFrontend(language: Language<PythonLanguageFrontend>, ctx: TranslationContext) :
     LanguageFrontend<Python.AST.AST, Python.AST.AST?>(language, ctx) {
@@ -76,10 +90,13 @@ class PythonLanguageFrontend(language: Language<PythonLanguageFrontend>, ctx: Tr
             it.set("content", fileContent)
             it.set("filename", file.absolutePath)
             it.exec("import ast")
+            it.exec("import sys")
             it.exec("parsed = ast.parse(content, filename=filename, type_comments=True)")
 
             val pyAST = it.getValue("parsed") as PyObject
+
             val tud = pythonASTtoCPG(pyAST, file.toPath())
+            populateSystemInformation(config, tud)
 
             if (config.matchCommentsToNodes) {
                 it.exec("import tokenize")
@@ -96,6 +113,7 @@ class PythonLanguageFrontend(language: Language<PythonLanguageFrontend>, ctx: Tr
                     (it.getValue("tokenList") as? ArrayList<*>) ?: TODO("Cannot get tokens of $it")
                 addCommentsToCPG(tud, pyTokens, pyCommentCode)
             }
+
             return tud
         }
     }
@@ -334,6 +352,39 @@ class PythonLanguageFrontend(language: Language<PythonLanguageFrontend>, ctx: Tr
         }
 }
 
+/**
+ * Populate system information from defined symbols that represent our environment. We add it as an
+ * overlay node to our [TranslationUnitDeclaration].
+ */
+fun populateSystemInformation(
+    config: TranslationConfiguration,
+    tu: TranslationUnitDeclaration,
+): SystemInformation {
+    var sysInfo =
+        SystemInformation(
+            platform = config.symbols["PYTHON_PLATFORM"],
+            // We need to populate the version info "in-order", to ensure that we do not
+            // set the micro version if minor and major are not set, i.e., there must not be a
+            // "gap" in the granularity of version numbers
+            versionInfo =
+                config.symbols["PYTHON_VERSION_MAJOR"]?.toLong()?.let { major ->
+                    val minor = config.symbols["PYTHON_VERSION_MINOR"]?.toLong()
+                    val micro =
+                        if (minor != null) config.symbols["PYTHON_VERSION_MICRO"]?.toLong()
+                        else null
+                    VersionInfo(major, minor, micro)
+                },
+        )
+    sysInfo.underlyingNode = tu
+    return sysInfo
+}
+
+/** Returns the system information overlay node from the [TranslationUnitDeclaration]. */
+val TranslationUnitDeclaration.sysInfo: SystemInformation?
+    get() {
+        return this.overlays.firstOrNull { it is SystemInformation } as? SystemInformation
+    }
+
 /**
  * This function maps Python's `ast` objects to out internal [Python] representation.
  *