diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt index 492c140edd4..a54f8615c41 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/ScopeManager.kt @@ -240,6 +240,7 @@ class ScopeManager : ScopeProvider { is TryStatement, is IfStatement, is CatchClause, + is CollectionComprehension, is Block -> LocalScope(nodeToScope) is FunctionDeclaration -> FunctionScope(nodeToScope) is RecordDeclaration -> RecordScope(nodeToScope) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt index 7ada182428b..d4fb769e558 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Node.kt @@ -369,3 +369,15 @@ abstract class Node : const val EMPTY_NAME = "" } } + +/** + * Works similar to [apply] but before executing [block], it enters the scope for this object and + * afterward leaves the scope again. + */ +inline fun T.applyWithScope(block: T.() -> Unit): T { + return this.apply { + ctx?.scopeManager?.enterScope(this) + block() + ctx?.scopeManager?.leaveScope(this) + } +} diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt index 3a4e343b7e7..e8ff3b326c8 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/TypeBuilder.kt @@ -101,7 +101,7 @@ fun LanguageProvider.objectType( ): Type { // First, we check, whether this is a built-in type, to avoid necessary allocations of simple // types - val builtIn = language?.getSimpleTypeOf(name.toString()) + val builtIn = language.getSimpleTypeOf(name.toString()) if (builtIn != null) { return builtIn } diff --git a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/ExtensionsTest.kt b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/ExtensionsTest.kt index aaa949caca2..f9eb9cbed4a 100644 --- a/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/ExtensionsTest.kt +++ b/cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/helpers/ExtensionsTest.kt @@ -28,18 +28,27 @@ package de.fraunhofer.aisec.cpg.helpers import de.fraunhofer.aisec.cpg.GraphExamples.Companion.testFrontend import de.fraunhofer.aisec.cpg.TranslationConfiguration import de.fraunhofer.aisec.cpg.frontends.TestLanguage +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.Name +import de.fraunhofer.aisec.cpg.graph.applyWithScope import de.fraunhofer.aisec.cpg.graph.builder.body import de.fraunhofer.aisec.cpg.graph.builder.declare import de.fraunhofer.aisec.cpg.graph.builder.function import de.fraunhofer.aisec.cpg.graph.builder.problemDecl import de.fraunhofer.aisec.cpg.graph.builder.translationResult import de.fraunhofer.aisec.cpg.graph.builder.translationUnit +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.problems +import de.fraunhofer.aisec.cpg.graph.statements.DeclarationStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CollectionComprehension import de.fraunhofer.aisec.cpg.graph.statements.expressions.ProblemExpression +import de.fraunhofer.aisec.cpg.graph.variables import de.fraunhofer.aisec.cpg.test.BaseTest import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertIs import kotlin.test.assertNotNull +import kotlin.test.assertNull internal class ExtensionsTest : BaseTest() { val problemDeclText = "This is a problem declaration." @@ -75,4 +84,19 @@ internal class ExtensionsTest : BaseTest() { "Failed to find the problem expression.", ) } + + @Test + fun testApplyWithScopeWithoutCtxAndScopeManager() { + val collectionComprehension = + CollectionComprehension().applyWithScope { + val varA = VariableDeclaration() + varA.name = Name("a") + val declarationStatement = DeclarationStatement() + declarationStatement.addDeclaration(varA) + this.statement = declarationStatement + } + val varA = collectionComprehension.variables["a"] + assertIs(varA) + assertNull(varA.scope) + } } diff --git a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt index f556615cb97..68e7aa908ca 100644 --- a/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt +++ b/cpg-core/src/testFixtures/kotlin/de/fraunhofer/aisec/cpg/test/TestUtils.kt @@ -223,18 +223,18 @@ fun compareLineFromLocationIfExists(n: Node, startLine: Boolean, toCompare: Int) } /** Asserts, that the expression given in [expression] refers to the expected declaration [b]. */ -fun assertRefersTo(expression: Expression?, b: Declaration?) { +fun assertRefersTo(expression: Expression?, b: Declaration?, message: String? = null) { if (expression is Reference) { - assertEquals(b, (expression as Reference?)?.refersTo) + assertEquals(b, (expression as Reference?)?.refersTo, message) } else { fail("not a reference") } } /** Asserts, that the expression given in [expression] does not refer to the declaration [b]. */ -fun assertNotRefersTo(expression: Expression?, b: Declaration?) { +fun assertNotRefersTo(expression: Expression?, b: Declaration?, message: String? = null) { if (expression is Reference) { - assertNotEquals(b, (expression as Reference?)?.refersTo) + assertNotEquals(b, (expression as Reference?)?.refersTo, message) } else { fail("not a reference") } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 3da8338b34b..f43bd5316c3 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -105,7 +105,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : * [CollectionComprehension]. */ private fun handleGeneratorExp(node: Python.AST.GeneratorExp): CollectionComprehension { - return newCollectionComprehension(rawNode = node).apply { + return newCollectionComprehension(rawNode = node).applyWithScope { statement = handle(node.elt) comprehensionExpressions += node.generators.map { handleComprehension(it, node) } type = objectType("Generator") @@ -117,10 +117,10 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : * into a [CollectionComprehension]. */ private fun handleListComprehension(node: Python.AST.ListComp): CollectionComprehension { - return newCollectionComprehension(rawNode = node).apply { + return newCollectionComprehension(rawNode = node).applyWithScope { statement = handle(node.elt) comprehensionExpressions += node.generators.map { handleComprehension(it, node) } - type = objectType("list") // TODO: Replace this once we have dedicated types + type = primitiveType("list") } } @@ -129,10 +129,10 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : * a [CollectionComprehension]. */ private fun handleSetComprehension(node: Python.AST.SetComp): CollectionComprehension { - return newCollectionComprehension(rawNode = node).apply { + return newCollectionComprehension(rawNode = node).applyWithScope { this.statement = handle(node.elt) this.comprehensionExpressions += node.generators.map { handleComprehension(it, node) } - this.type = objectType("set") // TODO: Replace this once we have dedicated types + this.type = primitiveType("set") } } @@ -141,7 +141,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : * into a [CollectionComprehension]. */ private fun handleDictComprehension(node: Python.AST.DictComp): CollectionComprehension { - return newCollectionComprehension(rawNode = node).apply { + return newCollectionComprehension(rawNode = node).applyWithScope { this.statement = newKeyValueExpression( key = handle(node.key), @@ -149,7 +149,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : rawNode = node, ) this.comprehensionExpressions += node.generators.map { handleComprehension(it, node) } - this.type = objectType("dict") // TODO: Replace this once we have dedicated types + this.type = primitiveType("dict") } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt index d6760829c48..d5246432a0c 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt @@ -37,6 +37,8 @@ import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CollectionComprehension +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ComprehensionExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference @@ -67,15 +69,15 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx), L } /** - * This function checks for each [AssignExpression] whether there is already a matching variable - * or not. New variables can be one of: + * This function checks for each [AssignExpression], [ComprehensionExpression] and + * [ForEachStatement] whether there is already a matching variable or not. New variables can be + * one of: * - [FieldDeclaration] if we are currently in a record * - [VariableDeclaration] otherwise - * - * TODO: loops */ private fun handle(node: Node?) { when (node) { + is ComprehensionExpression -> handleComprehensionExpression(node) is AssignExpression -> handleAssignExpression(node) is ForEachStatement -> handleForEach(node) else -> { @@ -194,6 +196,27 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx), L this.base.name == scopeManager.currentMethod?.receiver?.name } + /** + * Generates a new [VariableDeclaration] for [Reference] (and those included in a + * [InitializerListExpression]) in the [ComprehensionExpression.variable]. + */ + private fun handleComprehensionExpression(comprehensionExpression: ComprehensionExpression) { + when (val variable = comprehensionExpression.variable) { + is Reference -> { + variable.access = AccessValues.WRITE + handleWriteToReference(variable) + } + is InitializerListExpression -> { + variable.initializers.forEach { + (it as? Reference)?.let { ref -> + ref.access = AccessValues.WRITE + handleWriteToReference(ref) + } + } + } + } + } + /** * Generates a new [VariableDeclaration] if [target] is a [Reference] and there is no existing * declaration yet. @@ -223,6 +246,15 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx), L } private fun handleAssignExpression(assignExpression: AssignExpression) { + val parentCollectionComprehensions = ArrayDeque() + var parentCollectionComprehension = + assignExpression.firstParentOrNull() + while (assignExpression.operatorCode == ":=" && parentCollectionComprehension != null) { + scopeManager.leaveScope(parentCollectionComprehension) + parentCollectionComprehensions.addLast(parentCollectionComprehension) + parentCollectionComprehension = + parentCollectionComprehension.firstParentOrNull() + } for (target in assignExpression.lhs) { handleAssignmentToTarget(assignExpression, target, setAccessValue = false) // If the lhs is an InitializerListExpression, we have to handle the individual elements @@ -233,6 +265,11 @@ class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx), L } } } + while ( + assignExpression.operatorCode == ":=" && parentCollectionComprehensions.isNotEmpty() + ) { + scopeManager.enterScope(parentCollectionComprehensions.removeLast()) + } } // New variables can also be declared as `variable` in a [ForEachStatement] diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/CollectionComprehensionTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/CollectionComprehensionTest.kt new file mode 100644 index 00000000000..8439a4d7031 --- /dev/null +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/CollectionComprehensionTest.kt @@ -0,0 +1,1922 @@ +/* + * Copyright (c) 2025, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.python + +import de.fraunhofer.aisec.cpg.TranslationResult +import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.functions +import de.fraunhofer.aisec.cpg.graph.get +import de.fraunhofer.aisec.cpg.graph.invoke +import de.fraunhofer.aisec.cpg.graph.records +import de.fraunhofer.aisec.cpg.graph.refs +import de.fraunhofer.aisec.cpg.graph.scopes.LocalScope +import de.fraunhofer.aisec.cpg.graph.statements +import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.CollectionComprehension +import de.fraunhofer.aisec.cpg.graph.statements.expressions.ComprehensionExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.KeyValueExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Literal +import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import de.fraunhofer.aisec.cpg.graph.statements.expressions.SubscriptExpression +import de.fraunhofer.aisec.cpg.graph.variables +import de.fraunhofer.aisec.cpg.test.analyze +import de.fraunhofer.aisec.cpg.test.assertLiteralValue +import de.fraunhofer.aisec.cpg.test.assertLocalName +import de.fraunhofer.aisec.cpg.test.assertNotRefersTo +import de.fraunhofer.aisec.cpg.test.assertRefersTo +import java.nio.file.Path +import kotlin.test.assertEquals +import kotlin.test.assertIs +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertSame +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class CollectionComprehensionTest { + private lateinit var result: TranslationResult + + @BeforeAll + fun setup() { + val topLevel = Path.of("src", "test", "resources", "python") + result = + analyze(listOf(topLevel.resolve("comprehension.py").toFile()), topLevel, true) { + it.registerLanguage() + it.symbols( + mapOf( + "PYTHON_VERSION_MAJOR" to "3", + "PYTHON_VERSION_MINOR" to "0", + "PYTHON_VERSION_MICRO" to "0", + ) + ) + } + assertNotNull(result) + } + + @Test + fun testComprehensionExpressionTuple() { + // Get the function tuple_comp + val tupleComp = result.functions["tuple_comp"] + assertIs( + tupleComp, + "There must be a function called \"tuple_comp\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + // Get the body + val body = tupleComp.body + assertIs( + body, + "The body of each function is modeled as a Block in the CPG. This must also apply to the function \"tuple_comp\".", + ) + // The first statement is expected to be an assignment of a list comprehension with an if to + // a variable "a" + val tupleAsVariableAssignment = body.statements[0] + assertIs( + tupleAsVariableAssignment, + "The statement is expected to be an AssignExpression", + ) + val tupleAsVariable = tupleAsVariableAssignment.rhs[0] + assertIs( + tupleAsVariable, + "The right hand side must be a CollectionComprehension representing python's list comprehension \"[bar(k, v) for (k, v) in x]\".", + ) + val barCall = tupleAsVariable.statement + assertIs( + barCall, + "The statement inside the list comprehension is expected to be a call to bar with arguments k and v", + ) + assertLocalName("bar", barCall, "The CallExpression calls bar()") + val argK = barCall.arguments[0] + assertIs(argK, "The first argument of bar() is expected to be a reference k") + assertLocalName("k", argK, "The first argument of bar() is expected to be a reference k") + val argV = barCall.arguments[1] + assertIs(argV, "The second argument of bar() is expected to be a reference v") + assertLocalName("v", argV, "The second argument of bar() is expected to be a reference v") + assertEquals( + 1, + tupleAsVariable.comprehensionExpressions.size, + "There is expected to be a single comprehension expression (\"for (k, v) in x\")", + ) + val initializerListExpression = tupleAsVariable.comprehensionExpressions[0].variable + assertIs( + initializerListExpression, + "The variable is expected to be actually tuple which is represented as an InitializerListExpression in the CPG", + ) + val variableK = initializerListExpression.initializers[0] + assertIs( + variableK, + "The first element in the tuple is expected to be a variable reference \"k\"", + ) + assertLocalName( + "k", + variableK, + "The first element in the tuple is expected to be a variable reference \"k\"", + ) + val variableV = initializerListExpression.initializers[1] + assertIs( + variableV, + "The second element in the tuple is expected to be a variable reference \"v\"", + ) + assertLocalName( + "v", + variableV, + "The second element in the tuple is expected to be a variable reference \"V\"", + ) + + // Check that the declarations exist for the variables k and v + val declarationK = variableK.refersTo + assertIs(declarationK, "The refersTo should be a VariableDeclaration") + assertIs( + declarationK.scope, + "The scope of the variable is the local scope belonging to the list comprehension. In particular, it is not the FunctionScope.", + ) + assertEquals( + tupleAsVariable, + declarationK.scope?.astNode, + "The scope of the variable is the local scope belonging to the list comprehension. In particular, it is not the FunctionScope.", + ) + assertRefersTo( + argK, + declarationK, + "The argument k of the call also refers to the variable k declared in the comprehension expression.", + ) + val declarationV = variableV.refersTo + assertIs(declarationV, "The refersTo should be a VariableDeclaration") + assertIs( + declarationV.scope, + "The scope of the variable is the local scope belonging to the list comprehension. In particular, it is not the FunctionScope.", + ) + assertEquals( + tupleAsVariable, + declarationV.scope?.astNode, + "The scope of the variable is the local scope belonging to the list comprehension. In particular, it is not the FunctionScope.", + ) + assertRefersTo( + argV, + declarationV, + "The argument v of the call also refers to the variable v declared in the comprehension expression.", + ) + } + + @Test + fun testListComprehensions() { + val listCompFunctionDeclaration = result.functions["list_comp"] + assertIs( + listCompFunctionDeclaration, + "There must be a function called \"list_comp\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + val paramX = listCompFunctionDeclaration.parameters[0] + assertIs( + paramX, + "The function \"list_comp\" has a parameter called \"x^\".", + ) + assertLocalName("x", paramX, "The function \"list_comp\" has a parameter called \"x\".") + + val body = listCompFunctionDeclaration.body + assertIs( + body, + "The body of each function is modeled as a Block in the CPG. This must also apply to the function \"list_comp\".", + ) + + val singleWithIfAssignment = body.statements[0] + assertIs( + singleWithIfAssignment, + "The first statement in the body is \"a = [foo(i) for i in x if i == 10}^\" which should be represented by an AssignExpression in the CPG.", + ) + val singleWithIf = singleWithIfAssignment.rhs[0] + assertIs( + singleWithIf, + "The right hand side of the assignment \"a = [foo(i) for i in x if i == 10]\" is expected to be modeled as a CollectionComprehension \"[foo(i) for i in x if i == 10]\" in the CPG.", + ) + var statement = singleWithIf.statement + var variable = singleWithIf.comprehensionExpressions[0].variable + assertIs( + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertLocalName( + "foo", + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertEquals( + 1, + singleWithIf.comprehensionExpressions.size, + "The CollectionComprehension \"[foo(i) for i in x if i == 10]\" has exactly one comprehensionExpressions which is \"for i in x if i == 10\"", + ) + assertIs( + variable.scope, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertSame( + singleWithIf, + variable.scope?.astNode, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertLocalName( + "i", + variable, + "The variable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + variable, + "The variable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + singleWithIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"x\"", + ) + assertLocalName( + "x", + singleWithIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"x\"", + ) + val ifPredicate = singleWithIf.comprehensionExpressions[0].predicate + assertIs( + ifPredicate, + "The two predicates \"if i == 10\" is expected to be represented by a binary operator \"==\" in the CPG.", + ) + assertEquals( + "==", + ifPredicate.operatorCode, + "The two predicates \"if i == 10\" is expected to be represented by a binary operator \"==\" in the CPG.", + ) + + val singleWithoutIfAssignment = body.statements[1] + assertIs( + singleWithoutIfAssignment, + "The second statement in the body is \"b = [foo(i) for i in x]\" which should be represented by an AssignExpression in the CPG.", + ) + val singleWithoutIf = singleWithoutIfAssignment.rhs[0] + assertIs( + singleWithoutIf, + "The right hand side of the assignment \"b = [foo(i) for i in x]\" is expected to be modeled as a CollectionComprehension \"[foo(i) for i in x]\" in the CPG.", + ) + statement = singleWithoutIf.statement + variable = singleWithoutIf.comprehensionExpressions[0].variable + assertIs( + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertLocalName( + "foo", + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertEquals( + 1, + singleWithoutIf.comprehensionExpressions.size, + "The CollectionComprehension \"[foo(i) for i in x]\" has exactly one comprehensionExpressions which is \"for i in x\"", + ) + assertIs( + variable.scope, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertSame( + singleWithoutIf, + variable.scope?.astNode, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertLocalName( + "i", + variable, + "The variable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + variable, + "The variable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + singleWithoutIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"x\"", + ) + assertLocalName( + "x", + singleWithoutIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"x\"", + ) + assertNull( + singleWithoutIf.comprehensionExpressions[0].predicate, + "The comprehension expression \"for i in x\" should not have any predicate.", + ) + + val singleWithDoubleIfAssignment = body.statements[2] + assertIs( + singleWithDoubleIfAssignment, + "The third statement in the body is \"c = [foo(i) for i in x if i == 10 if i < 20]\" which should be represented by an AssignExpression in the CPG.", + ) + val singleWithDoubleIf = singleWithDoubleIfAssignment.rhs[0] + assertIs( + singleWithDoubleIf, + "The right hand side of the assignment \"c = [foo(i) for i in x if i == 10 if i < 20]\" is expected to be modeled as a CollectionComprehension \"[foo(i) for i in x if i == 10 if i < 20]\" in the CPG.", + ) + statement = singleWithDoubleIf.statement + variable = singleWithDoubleIf.comprehensionExpressions[0].variable + assertIs( + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertLocalName( + "foo", + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertEquals( + 1, + singleWithDoubleIf.comprehensionExpressions.size, + "The CollectionComprehension \"[foo(i) for i in x if i == 10 if i < 20]\" has exactly one comprehensionExpressions which is \"for i in x if i == 10 if i < 20\"", + ) + assertIs( + variable.scope, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertSame( + singleWithDoubleIf, + variable.scope?.astNode, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertLocalName( + "i", + variable, + "The variable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + variable, + "The variable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + singleWithDoubleIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"x\"", + ) + assertLocalName( + "x", + singleWithDoubleIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"x\"", + ) + val doubleIfPredicate = singleWithDoubleIf.comprehensionExpressions[0].predicate + assertIs( + doubleIfPredicate, + "The two predicates \"if i == 10 if i < 20\" are expected to be connected with the binary operator \"and\" in the CPG.", + ) + assertEquals( + "and", + doubleIfPredicate.operatorCode, + "The two predicates \"if i == 10 if i < 20\" are expected to be connected with the binary operator \"and\" in the CPG.", + ) + + val doubleAssignment = body.statements[3] + assertIs( + doubleAssignment, + "The third statement in the body is \"d = [foo(i) for z in y if z in x for i in z if i == 10 ]\" which should be represented by an AssignExpression in the CPG.", + ) + val double = doubleAssignment.rhs[0] + assertIs( + double, + "The right hand side of the assignment \"d = [foo(i) for z in y if z in x for i in z if i == 10 ]\" is expected to be modeled as a CollectionComprehension \"[foo(i) for z in y if z in x for i in z if i == 10 ]\" in the CPG.", + ) + statement = double.statement + assertIs( + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertLocalName( + "foo", + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertEquals( + 2, + double.comprehensionExpressions.size, + "The CollectionComprehension \"[foo(i) for z in y if z in x for i in z if i == 10 ]\" has two comprehension expressions which are \"for z in y if z in x\" and \"for i in z if i == 10\"", + ) + } + + @Test + fun testSetComprehensions() { + val setCompFunctionDeclaration = result.functions["set_comp"] + assertIs( + setCompFunctionDeclaration, + "There must be a function called \"set_comp\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + val body = setCompFunctionDeclaration.body + assertIs( + body, + "The body of each function is modeled as a Block in the CPG. This must also apply to the function \"set_comp\".", + ) + val singleWithIfAssignment = body.statements[0] + assertIs( + singleWithIfAssignment, + "The first statement in the body is \"a = {foo(i) for i in x if i == 10}^\" which should be represented by an AssignExpression in the CPG.", + ) + val singleWithIf = singleWithIfAssignment.rhs[0] + assertIs( + singleWithIf, + "The right hand side of the assignment \"a = {foo(i) for i in x if i == 10}\" is expected to be modeled as a CollectionComprehension \"{foo(i) for i in x if i == 10}\" in the CPG.", + ) + var statement = singleWithIf.statement + var variable = singleWithIf.comprehensionExpressions[0].variable + assertIs( + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertLocalName( + "foo", + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertEquals( + 1, + singleWithIf.comprehensionExpressions.size, + "The CollectionComprehension \"{foo(i) for i in x if i == 10}\" has exactly one comprehensionExpressions which is \"for i in x if i == 10\"", + ) + assertIs( + variable.scope, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertSame( + singleWithIf, + variable.scope?.astNode, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertLocalName( + "i", + variable, + "The variable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + variable, + "The variable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + singleWithIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"x\"", + ) + assertLocalName( + "x", + singleWithIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"x\"", + ) + val ifPredicate = singleWithIf.comprehensionExpressions[0].predicate + assertIs( + ifPredicate, + "The two predicates \"if i == 10\" is expected to be represented by a binary operator \"==\" in the CPG.", + ) + assertEquals( + "==", + ifPredicate.operatorCode, + "The two predicates \"if i == 10\" is expected to be represented by a binary operator \"==\" in the CPG.", + ) + + val singleWithoutIfAssignment = body.statements[1] + assertIs( + singleWithoutIfAssignment, + "The second statement in the body is \"b = {foo(i) for i in x}\" which should be represented by an AssignExpression in the CPG.", + ) + val singleWithoutIf = singleWithoutIfAssignment.rhs[0] + assertIs( + singleWithoutIf, + "The right hand side of the assignment \"b = {foo(i) for i in x}\" is expected to be modeled as a CollectionComprehension \"{foo(i) for i in x}\" in the CPG.", + ) + statement = singleWithoutIf.statement + variable = singleWithoutIf.comprehensionExpressions[0].variable + assertIs( + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertLocalName( + "foo", + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertEquals( + 1, + singleWithoutIf.comprehensionExpressions.size, + "The CollectionComprehension \"{foo(i) for i in x}\" has exactly one comprehensionExpressions which is \"for i in x\"", + ) + assertIs( + variable.scope, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertSame( + singleWithoutIf, + variable.scope?.astNode, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertLocalName( + "i", + variable, + "The variable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + variable, + "The variable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + singleWithoutIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"x\"", + ) + assertLocalName( + "x", + singleWithoutIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"x\"", + ) + assertNull( + singleWithoutIf.comprehensionExpressions[0].predicate, + "The comprehension expression \"for i in x\" should not have any predicate.", + ) + + val singleWithDoubleIfAssignment = body.statements[2] + assertIs( + singleWithDoubleIfAssignment, + "The third statement in the body is \"c = {foo(i) for i in x if i == 10 if i < 20}\" which should be represented by an AssignExpression in the CPG.", + ) + val singleWithDoubleIf = singleWithDoubleIfAssignment.rhs[0] + assertIs( + singleWithDoubleIf, + "The right hand side of the assignment \"c = {foo(i) for i in x if i == 10 if i < 20}\" is expected to be modeled as a CollectionComprehension \"{foo(i) for i in x if i == 10 if i < 20}\" in the CPG.", + ) + statement = singleWithDoubleIf.statement + variable = singleWithDoubleIf.comprehensionExpressions[0].variable + assertIs( + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertLocalName( + "foo", + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertEquals( + 1, + singleWithDoubleIf.comprehensionExpressions.size, + "The CollectionComprehension \"{foo(i) for i in x if i == 10 if i < 20}\" has exactly one comprehensionExpressions which is \"for i in x if i == 10 if i < 20\"", + ) + assertIs( + variable.scope, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertSame( + singleWithDoubleIf, + variable.scope?.astNode, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertLocalName( + "i", + variable, + "The variable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + variable, + "The variable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + singleWithDoubleIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"x\"", + ) + assertLocalName( + "x", + singleWithDoubleIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"x\"", + ) + val doubleIfPredicate = singleWithDoubleIf.comprehensionExpressions[0].predicate + assertIs( + doubleIfPredicate, + "The two predicates \"if i == 10 if i < 20\" are expected to be connected with the binary operator \"and\" in the CPG.", + ) + assertEquals( + "and", + doubleIfPredicate.operatorCode, + "The two predicates \"if i == 10 if i < 20\" are expected to be connected with the binary operator \"and\" in the CPG.", + ) + + val doubleAssignment = body.statements[3] + assertIs( + doubleAssignment, + "The third statement in the body is \"d = {foo(i) for z in y if z in x for i in z if i == 10 }\" which should be represented by an AssignExpression in the CPG.", + ) + val double = doubleAssignment.rhs[0] + assertIs( + double, + "The right hand side of the assignment \"d = {foo(i) for z in y if z in x for i in z if i == 10 }\" is expected to be modeled as a CollectionComprehension \"{foo(i) for z in y if z in x for i in z if i == 10 }\" in the CPG.", + ) + statement = double.statement + assertIs( + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertLocalName( + "foo", + statement, + "The CollectionComprehension has the statement \"foo(i)\" which is expected to be modeled as a CallExpression with localName \"foo\".", + ) + assertEquals( + 2, + double.comprehensionExpressions.size, + "The CollectionComprehension \"{foo(i) for z in y if z in x for i in z if i == 10 }\" has two comprehension expressions which are \"for z in y if z in x\" and \"for i in z if i == 10\"", + ) + } + + @Test + fun testDictComprehensions() { + val dictCompFunctionDeclaration = result.functions["dict_comp"] + assertIs( + dictCompFunctionDeclaration, + "There must be a function called \"dict_comp\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + val body = dictCompFunctionDeclaration.body + assertIs( + body, + "The body of each function is modeled as a Block in the CPG. This must also apply to the function \"dict_comp\".", + ) + val singleWithIfAssignment = body.statements[0] + assertIs( + singleWithIfAssignment, + "The first statement in the body is \"a = {i: foo(i) for i in x if i == 10}^\" which should be represented by an AssignExpression in the CPG.", + ) + val singleWithIf = singleWithIfAssignment.rhs[0] + assertIs( + singleWithIf, + "The right hand side of the assignment \"a = {i: foo(i) for i in x if i == 10}\" is expected to be modeled as a CollectionComprehension \"{i: foo(i) for i in x if i == 10}\" in the CPG.", + ) + var statement = singleWithIf.statement + var variable = singleWithIf.comprehensionExpressions[0].variable + assertIs( + statement, + "The CollectionComprehension has the statement \"i: foo(i)\" which is expected to be modeled as a KeyValueExpression.", + ) + assertIs( + statement.key, + "The key of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a Reference with localName \"i\"", + ) + assertLocalName( + "i", + statement.key, + "The key of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + statement.value, + "The value of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a CallExpression with localName \"foo\"", + ) + assertLocalName( + "foo", + statement.value, + "The value of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a CallExpression with localName \"foo\"", + ) + assertEquals( + 1, + singleWithIf.comprehensionExpressions.size, + "The CollectionComprehension \"{i: foo(i) for i in x if i == 10}\" has exactly one comprehensionExpressions which is \"for i in x if i == 10\"", + ) + assertIs( + variable.scope, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertSame( + singleWithIf, + variable.scope?.astNode, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertLocalName( + "i", + variable, + "The variable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + variable, + "The variable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + singleWithIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"x\"", + ) + assertLocalName( + "x", + singleWithIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10\" is expected to be a Reference with localName \"x\"", + ) + val ifPredicate = singleWithIf.comprehensionExpressions[0].predicate + assertIs( + ifPredicate, + "The two predicates \"if i == 10\" is expected to be represented by a binary operator \"==\" in the CPG.", + ) + assertEquals( + "==", + ifPredicate.operatorCode, + "The two predicates \"if i == 10\" is expected to be represented by a binary operator \"==\" in the CPG.", + ) + + val singleWithoutIfAssignment = body.statements[1] + assertIs( + singleWithoutIfAssignment, + "The second statement in the body is \"b = {i: foo(i) for i in x}\" which should be represented by an AssignExpression in the CPG.", + ) + val singleWithoutIf = singleWithoutIfAssignment.rhs[0] + assertIs( + singleWithoutIf, + "The right hand side of the assignment \"b = {i: foo(i) for i in x}\" is expected to be modeled as a CollectionComprehension \"{i: foo(i) for i in x}\" in the CPG.", + ) + statement = singleWithoutIf.statement + variable = singleWithoutIf.comprehensionExpressions[0].variable + assertIs( + statement, + "The CollectionComprehension has the statement \"i: foo(i)\" which is expected to be modeled as a KeyValueExpression.", + ) + assertIs( + statement.key, + "The key of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a Reference with localName \"i\"", + ) + assertLocalName( + "i", + statement.key, + "The key of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + statement.value, + "The value of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a CallExpression with localName \"foo\"", + ) + assertLocalName( + "foo", + statement.value, + "The value of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a CallExpression with localName \"foo\"", + ) + assertEquals( + 1, + singleWithoutIf.comprehensionExpressions.size, + "The CollectionComprehension \"{i: foo(i) for i in x}\" has exactly one comprehensionExpressions which is \"for i in x\"", + ) + assertIs( + variable.scope, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertSame( + singleWithoutIf, + variable.scope?.astNode, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertLocalName( + "i", + variable, + "The variable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + variable, + "The variable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + singleWithoutIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"x\"", + ) + assertLocalName( + "x", + singleWithoutIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x\" is expected to be a Reference with localName \"x\"", + ) + assertNull( + singleWithoutIf.comprehensionExpressions[0].predicate, + "The comprehension expression \"for i in x\" should not have any predicate.", + ) + + val singleWithDoubleIfAssignment = body.statements[2] + assertIs( + singleWithDoubleIfAssignment, + "The third statement in the body is \"c = {i: foo(i) for i in x if i == 10 if i < 20}\" which should be represented by an AssignExpression in the CPG.", + ) + val singleWithDoubleIf = singleWithDoubleIfAssignment.rhs[0] + assertIs( + singleWithDoubleIf, + "The right hand side of the assignment \"c = {i: foo(i) for i in x if i == 10 if i < 20}\" is expected to be modeled as a CollectionComprehension \"{i: foo(i) for i in x if i == 10 if i < 20}\" in the CPG.", + ) + statement = singleWithDoubleIf.statement + variable = singleWithDoubleIf.comprehensionExpressions[0].variable + assertIs( + statement, + "The CollectionComprehension has the statement \"i: foo(i)\" which is expected to be modeled as a KeyValueExpression.", + ) + assertIs( + statement.key, + "The key of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a Reference with localName \"i\"", + ) + assertLocalName( + "i", + statement.key, + "The key of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + statement.value, + "The value of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a CallExpression with localName \"foo\"", + ) + assertLocalName( + "foo", + statement.value, + "The value of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a CallExpression with localName \"foo\"", + ) + assertEquals( + 1, + singleWithDoubleIf.comprehensionExpressions.size, + "The CollectionComprehension \"{i: foo(i) for i in x if i == 10 if i < 20}\" has exactly one comprehensionExpressions which is \"for i in x if i == 10 if i < 20\"", + ) + assertIs( + variable.scope, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertSame( + singleWithDoubleIf, + variable.scope?.astNode, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertLocalName( + "i", + variable, + "The variable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + variable, + "The variable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + singleWithDoubleIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"x\"", + ) + assertLocalName( + "x", + singleWithDoubleIf.comprehensionExpressions[0].iterable, + "The iterable of the comprehension expression \"for i in x if i == 10 if i < 20\" is expected to be a Reference with localName \"x\"", + ) + val doubleIfPredicate = singleWithDoubleIf.comprehensionExpressions[0].predicate + assertIs( + doubleIfPredicate, + "The two predicates \"if i == 10 if i < 20\" are expected to be connected with the binary operator \"and\" in the CPG.", + ) + assertEquals( + "and", + doubleIfPredicate.operatorCode, + "The two predicates \"if i == 10 if i < 20\" are expected to be connected with the binary operator \"and\" in the CPG.", + ) + + val doubleAssignment = body.statements[3] + assertIs( + doubleAssignment, + "The third statement in the body is \"d = {i: foo(i) for z in y if z in x for i in z if i == 10 }\" which should be represented by an AssignExpression in the CPG.", + ) + val double = doubleAssignment.rhs[0] + assertIs( + double, + "The right hand side of the assignment \"d = {i: foo(i) for z in y if z in x for i in z if i == 10 }\" is expected to be modeled as a CollectionComprehension \"{i: foo(i) for z in y if z in x for i in z if i == 10 }\" in the CPG.", + ) + statement = double.statement + assertIs( + statement, + "The CollectionComprehension has the statement \"i: foo(i)\" which is expected to be modeled as a KeyValueExpression.", + ) + assertIs( + statement.key, + "The key of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a Reference with localName \"i\"", + ) + assertLocalName( + "i", + statement.key, + "The key of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a Reference with localName \"i\"", + ) + assertIs( + statement.value, + "The value of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a CallExpression with localName \"foo\"", + ) + assertLocalName( + "foo", + statement.value, + "The value of the CollectionComprehension of the KeyValueExpression \"i: foo(i)\" is expected to be a CallExpression with localName \"foo\"", + ) + assertEquals( + 2, + double.comprehensionExpressions.size, + "The CollectionComprehension \"{i: foo(i) for z in y if z in x for i in z if i == 10 }\" has two comprehension expressions which are \"for z in y if z in x\" and \"for i in z if i == 10\"", + ) + } + + @Test + fun testGeneratorExpr() { + val generatorFunctionDeclaration = result.functions["generator"] + assertIs( + generatorFunctionDeclaration, + "There must be a function called \"generator\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + val body = generatorFunctionDeclaration.body + assertIs( + body, + "The body of each function is modeled as a Block in the CPG. This must also apply to the function \"generator\".", + ) + val singleWithIfAssignment = body.statements[0] + assertIs(singleWithIfAssignment) + val singleWithIf = singleWithIfAssignment.rhs[0] + assertIs(singleWithIf) + assertIs(singleWithIf.statement) + assertEquals(1, singleWithIf.comprehensionExpressions.size) + var variable = singleWithIf.comprehensionExpressions[0].variable + assertIs( + variable.scope, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertSame( + singleWithIf, + variable.scope?.astNode, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertLocalName("i", variable) + assertIs(singleWithIf.comprehensionExpressions[0].iterable) + assertLocalName("range", singleWithIf.comprehensionExpressions[0].iterable) + val ifPredicate = singleWithIf.comprehensionExpressions[0].predicate + assertIs(ifPredicate) + assertEquals("==", ifPredicate.operatorCode) + + val singleWithoutIfAssignment = body.statements[1] + assertIs(singleWithoutIfAssignment) + val singleWithoutIf = singleWithoutIfAssignment.rhs[0] + assertIs(singleWithoutIf) + assertIs(singleWithoutIf.statement) + assertEquals(1, singleWithoutIf.comprehensionExpressions.size) + variable = singleWithoutIf.comprehensionExpressions[0].variable + assertIs( + variable.scope, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertSame( + singleWithoutIf, + variable.scope?.astNode, + "The scope of the variable is expected to be a local scope belonging to the list comprehension.", + ) + assertLocalName("i", variable) + assertIs(singleWithIf.comprehensionExpressions[0].iterable) + assertLocalName("range", singleWithIf.comprehensionExpressions[0].iterable) + assertNull(singleWithoutIf.comprehensionExpressions[0].predicate) + } + + @Test + /** + * This test ensures that variables in a comprehension do not bind to the outer scope. See + * [testCompBindingAssignExpr] for exceptions. + */ + fun testCompBinding() { + val compBindingFunctionDeclaration = result.functions["comp_binding"] + assertIs( + compBindingFunctionDeclaration, + "There must be a function called \"comp_binding\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + val xDeclaration = compBindingFunctionDeclaration.variables.firstOrNull() + assertIs(xDeclaration) + + assertEquals( + 5, + compBindingFunctionDeclaration.variables.size, + "Expected five variables. One for the \"outside\" x and one for each of the four comprehensions.", + ) + + assertEquals( + 2, + xDeclaration.usages.size, + "Expected two usages: one for the initial assignment and one for the usage in \"print(x)\".", + ) + + val comprehensions = + compBindingFunctionDeclaration.body.statements.filterIsInstance< + CollectionComprehension + >() + assertEquals(4, comprehensions.size, "Expected to find four comprehensions.") + + comprehensions.forEach { + it.refs("x").forEach { ref -> assertNotRefersTo(ref, xDeclaration) } + } + } + + @Test + /** + * This test ensures that variables in a comprehension do not bind to the outer scope if they + * are used in an `AssignExpr`. See https://peps.python.org/pep-0572/#scope-of-the-target + */ + fun testCompBindingAssignExpr() { + val compBindingAssignExprFunctionDeclaration = result.functions["comp_binding_assign_expr"] + assertIs( + compBindingAssignExprFunctionDeclaration, + "There must be a function called \"comp_binding_assign_expr\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + val xDeclaration = compBindingAssignExprFunctionDeclaration.variables["x"] + assertIs( + xDeclaration, + "There must be a VariableDeclaration with the local name \"x\" inside the function.", + ) + + assertEquals( + 2, + compBindingAssignExprFunctionDeclaration.variables.size, + "Expected two variables. One for the \"outside\" x and one for the \"temp\" inside the comprehension.", + ) + + assertEquals( + 3, + xDeclaration.usages.size, + "Expected three usages: one for the initial assignment, one for the comprehension and one for the usage in \"print(x)\".", + ) + + val comprehension = + compBindingAssignExprFunctionDeclaration.body.statements.singleOrNull { + it is CollectionComprehension + } + assertNotNull(comprehension) + val xRef = comprehension.refs("x").singleOrNull() + assertNotNull(xRef) + assertRefersTo(xRef, xDeclaration) + } + + @Test + /** + * This test ensures that variables in a comprehension do not bind to the outer scope if they + * are used in an `AssignExpr`. See https://peps.python.org/pep-0572/#scope-of-the-target + */ + fun testCompBindingAssignExprNested() { + val compBindingAssignExprNestedFunctionDeclaration = + result.functions["comp_binding_assign_expr_nested"] + assertIs( + compBindingAssignExprNestedFunctionDeclaration, + "There must be a function called \"comp_binding_assign_expr_nested\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + val xDeclaration = compBindingAssignExprNestedFunctionDeclaration.variables["x"] + assertIs( + xDeclaration, + "There must be a VariableDeclaration with the local name \"x\" inside the function.", + ) + + assertEquals( + 3, + compBindingAssignExprNestedFunctionDeclaration.variables.size, + "Expected two variables. One for the \"outside\" x, one for the \"temp\" inside the comprehension and one for the \"a\" inside the comprehension.", + ) + + assertEquals( + 3, + xDeclaration.usages.size, + "Expected three usages: one for the initial assignment, one for the comprehension and one for the usage in \"print(x)\".", + ) + val body = compBindingAssignExprNestedFunctionDeclaration.body + assertIs(body, "The body of a function must be a Block.") + val outerComprehension = body.statements.singleOrNull { it is CollectionComprehension } + assertIs( + outerComprehension, + "There must be exactly one CollectionComprehension (the list comprehension) in the statement of the body. Note: The inner collection comprehension would be reached by the extension function Node::statements which does not apply here.", + ) + val innerComprehension = outerComprehension.statement + assertIs( + innerComprehension, + "The inner comprehension is the statement of the outer list comprehension", + ) + val xRef = innerComprehension.refs("x").singleOrNull() + assertNotNull( + xRef, + "There is only one usage of \"x\" which is inside the inner comprehension's statement.", + ) + assertRefersTo( + xRef, + xDeclaration, + "The reference of \"x\" inside the inner comprehension's statement refers to the variable declared outside the comprehensions.", + ) + } + + @Test + fun testCompBindingListAssignment() { + val comprehensionWithListAssignmentFunctionDeclaration = + result.functions["comprehension_with_list_assignment"] + assertIs( + comprehensionWithListAssignmentFunctionDeclaration, + "There must be a function called \"comprehension_with_list_assignment\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + // Get the body + val body = comprehensionWithListAssignmentFunctionDeclaration.body + assertIs( + body, + "The body of each function is modeled as a Block in the CPG. This must also apply to the function \"comprehension_with_list_assignment\".", + ) + + val listBInitialization = body.statements[0] + assertIs( + listBInitialization, + "The first statement of the function \"comprehension_with_list_assignment\" is expected to be the initialization of list \"b\" by the statement \"b = [0, 1, 2]\" which is expected to be represented by an AssignExpression in the CPG.", + ) + val refBFirstStatement = listBInitialization.lhs[0] + assertIs( + refBFirstStatement, + "The left hand side of the assignment \"b = [0, 1, 2]\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + assertLocalName( + "b", + refBFirstStatement, + "The left hand side of the assignment \"b = [0, 1, 2]\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + val bDeclaration = listBInitialization.variables["b"] + assertIs( + bDeclaration, + "There must be a VariableDeclaration with the local name \"b\" inside the first statement of the function \"comprehension_with_list_assignment\".", + ) + assertRefersTo( + refBFirstStatement, + bDeclaration, + "The reference \"b\" is expected to refer to the variable declaration \"b\" in the same statement.", + ) + + // Check if the AST of the list comprehension fits our expectations. + val listComprehensionWithTupleAndAssignmentToListElement = body.statements[1] + assertIs( + listComprehensionWithTupleAndAssignmentToListElement, + "The second statement of the function \"comprehension_with_list_assignment\" is expected to be python's list comprehension \"[a for (a, b[0]) in [(1, 2), (2, 4), (3, 6)]]\" which is represented by a CollectionComprehension in the CPG", + ) + + val comprehensionExpression = + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions + .singleOrNull() + assertEquals( + 1, + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions.size, + "There is expected to be exactly one CollectionComprehension in the list comprehension \"[a for (a, b[0]) in [(1, 2), (2, 4), (3, 6)]]\". It represents the code's part \"for (a, b[0]) in [(1, 2), (2, 4), (3, 6)]\"", + ) + assertIs( + comprehensionExpression, + "There is expected to be exactly one ComprehensionExpression in the list comprehension \"[a for (a, b[0]) in [(1, 2), (2, 4), (3, 6)]]\". It represents the code's part \"for (a, b[0]) in [(1, 2), (2, 4), (3, 6)]\"", + ) + + val tuple = comprehensionExpression.variable + assertIs( + tuple, + "The variable of the ComprehensionExpression is the tuple \"(a, b[0])\" which is expected to be represented by an InitializerListExpression in the CPG.", + ) + assertEquals( + 2, + tuple.initializers.size, + "The tuple \"(a, b[0])\" represented by an InitializerListExpression in the CPG is expected to have exactly two elements.", + ) + val refA = tuple.initializers[0] + assertIs( + refA, + "The first element of the tuple \"(a, b[0])\" is expected to be represented by a Reference with localName \"a\" in the CPG.", + ) + assertLocalName( + "a", + refA, + "The first element of the tuple \"(a, b[0])\" is expected to be represented by a Reference with localName \"a\" in the CPG.", + ) + val accessB0 = tuple.initializers[1] + assertIs( + accessB0, + "The second element of the tuple \"(a, b[0])\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Literal with value \"0\".", + ) + val refB = accessB0.arrayExpression + assertIs( + refB, + "The second element of the tuple \"(a, b[0])\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Literal with value \"0\" (with kotlin type Long ).", + ) + assertLocalName( + "b", + refB, + "The second element of the tuple \"(a, b[0])\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Literal with value \"0\" (with kotlin type Long ).", + ) + val index = accessB0.subscriptExpression + assertIs>( + index, + "The second element of the tuple \"(a, b[0])\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Literal with value \"0\" (with kotlin type Long ).", + ) + assertLiteralValue( + 0L, + index, + "The second element of the tuple \"(a, b[0])\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Literal with value \"0\" (with kotlin type Long ).", + ) + + // Now the actually interesting part: We check for variables belonging to the references. + val variableDeclarationA = refA.refersTo + assertIs( + variableDeclarationA, + "We expect that the reference \"a\" refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertIs( + variableDeclarationA.scope, + "We expect that the reference \"a\" refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertEquals( + listComprehensionWithTupleAndAssignmentToListElement, + variableDeclarationA.scope?.astNode, + "We expect that the reference \"a\" refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertRefersTo( + refB, + bDeclaration, + "We expect that the reference \"b\" in the tuple refers to the VariableDeclaration of \"b\" which is added outside the list comprehension (in statement 0).", + ) + } + + @Test + fun testComprehensionWithListAssignmentAndIndexVariable() { + val comprehensionWithListAssignmentAndIndexVariableFunctionDeclaration = + result.functions["comprehension_with_list_assignment_and_index_variable"] + assertIs( + comprehensionWithListAssignmentAndIndexVariableFunctionDeclaration, + "There must be a function called \"comprehension_with_list_assignment_and_index_variable\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + // Get the body + val body = comprehensionWithListAssignmentAndIndexVariableFunctionDeclaration.body + assertIs( + body, + "The body of each function is modeled as a Block in the CPG. This must also apply to the function \"comprehension_with_list_assignment_and_index_variable\".", + ) + + val listBInitialization = body.statements[0] + assertIs( + listBInitialization, + "The first statement of the function \"comprehension_with_list_assignment_and_index_variable\" is expected to be the initialization of list \"b\" by the statement \"b = [0, 1, 2]\" which is expected to be represented by an AssignExpression in the CPG.", + ) + val refBFirstStatement = listBInitialization.lhs[0] + assertIs( + refBFirstStatement, + "The left hand side of the assignment \"b = [0, 1, 2]\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + assertLocalName( + "b", + refBFirstStatement, + "The left hand side of the assignment \"b = [0, 1, 2]\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + val bDeclaration = listBInitialization.variables["b"] + assertIs( + bDeclaration, + "There must be a VariableDeclaration with the local name \"b\" inside the first statement of the function \"comprehension_with_list_assignment_and_index_variable\".", + ) + assertRefersTo( + refBFirstStatement, + bDeclaration, + "The reference \"b\" is expected to refer to the variable declaration \"b\" in the same statement.", + ) + + // Check if the AST of the list comprehension fits our expectations. + val listComprehensionWithTupleAndAssignmentToListElement = body.statements[1] + assertIs( + listComprehensionWithTupleAndAssignmentToListElement, + "The second statement of the function \"comprehension_with_list_assignment_and_index_variable\" is expected to be python's list comprehension \"[a for (a, b[a]) in [(0, 'this'), (1, 'is'), (2, 'fun')]]\" which is represented by a CollectionComprehension in the CPG", + ) + + val comprehensionExpression = + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions + .singleOrNull() + assertEquals( + 1, + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions.size, + "There is expected to be exactly one CollectionComprehension in the list comprehension \"[a for (a, b[a]) in [(0, 'this'), (1, 'is'), (2, 'fun')]]\". It represents the code's part \"for (a, b[a]) in [(0, 'this'), (1, 'is'), (2, 'fun')]\"", + ) + assertIs( + comprehensionExpression, + "There is expected to be exactly one ComprehensionExpression in the list comprehension \"[a for (a, b[a]) in [(0, 'this'), (1, 'is'), (2, 'fun')]]\". It represents the code's part \"for (a, b[a]) in [(0, 'this'), (1, 'is'), (2, 'fun')]\"", + ) + + val tuple = comprehensionExpression.variable + assertIs( + tuple, + "The variable of the ComprehensionExpression is the tuple \"(a, b[a])\" which is expected to be represented by an InitializerListExpression in the CPG.", + ) + assertEquals( + 2, + tuple.initializers.size, + "The tuple \"(a, b[a])\" represented by an InitializerListExpression in the CPG is expected to have exactly two elements.", + ) + val refA = tuple.initializers[0] + assertIs( + refA, + "The first element of the tuple \"(a, b[a])\" is expected to be represented by a Reference with localName \"a\" in the CPG.", + ) + assertLocalName( + "a", + refA, + "The first element of the tuple \"(a, b[a])\" is expected to be represented by a Reference with localName \"a\" in the CPG.", + ) + val accessBA = tuple.initializers[1] + assertIs( + accessBA, + "The second element of the tuple \"(a, b[a])\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"a\" which refers to the same variable as the tuple's first element.", + ) + val refB = accessBA.arrayExpression + assertIs( + refB, + "The second element of the tuple \"(a, b[a])\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"a\" which refers to the same variable as the tuple's first element.", + ) + assertLocalName( + "b", + refB, + "The second element of the tuple \"(a, b[a])\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is Reference \"a\" which refers to the same variable as the tuple's first element.", + ) + val index = accessBA.subscriptExpression + assertIs( + index, + "The second element of the tuple \"(a, b[a])\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is Reference \"a\" which refers to the same variable as the tuple's first element.", + ) + assertLocalName( + "a", + index, + "The second element of the tuple \"(a, b[a])\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Literal with value \"0\" (with kotlin type Long ).", + ) + + // Now the actually interesting part: We check for variables belonging to the references. + val variableDeclarationA = refA.refersTo + assertIs( + variableDeclarationA, + "We expect that the reference \"a\" in the first element of the tuple refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertRefersTo( + index, + variableDeclarationA, + "We expect that the reference \"a\" in the second element of the tuple refers to the same VariableDeclaration as the first element of the tuple.", + ) + assertIs( + variableDeclarationA.scope, + "We expect that the reference \"a\" refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertEquals( + listComprehensionWithTupleAndAssignmentToListElement, + variableDeclarationA.scope?.astNode, + "We expect that the reference \"a\" refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertRefersTo( + refB, + bDeclaration, + "We expect that the reference \"b\" in the tuple refers to the VariableDeclaration of \"b\" which is added outside the list comprehension (in statement 0).", + ) + } + + @Test + fun testComprehensionWithListAssignmentAndIndexVariableReversed() { + val comprehensionWithListAssignmentAndIndexVariableReversedFunctionDeclaration = + result.functions["comprehension_with_list_assignment_and_index_variable_reversed"] + assertIs( + comprehensionWithListAssignmentAndIndexVariableReversedFunctionDeclaration, + "There must be a function called \"comprehension_with_list_assignment_and_index_variable_reversed\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + // Get the body + val body = comprehensionWithListAssignmentAndIndexVariableReversedFunctionDeclaration.body + assertIs( + body, + "The body of each function is modeled as a Block in the CPG. This must also apply to the function \"comprehension_with_list_assignment_and_index_variable_reversed\".", + ) + + val listBInitialization = body.statements[0] + assertIs( + listBInitialization, + "The first statement of the function \"comprehension_with_list_assignment_and_index_variable_reversed\" is expected to be the initialization of list \"b\" by the statement \"b = [0, 1, 2]\" which is expected to be represented by an AssignExpression in the CPG.", + ) + val refBFirstStatement = listBInitialization.lhs[0] + assertIs( + refBFirstStatement, + "The left hand side of the assignment \"b = [0, 1, 2]\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + assertLocalName( + "b", + refBFirstStatement, + "The left hand side of the assignment \"b = [0, 1, 2]\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + val bDeclaration = listBInitialization.variables["b"] + assertIs( + bDeclaration, + "There must be a VariableDeclaration with the local name \"b\" inside the first statement of the function \"comprehension_with_list_assignment_and_index_variable_reversed\".", + ) + assertRefersTo( + refBFirstStatement, + bDeclaration, + "The reference \"b\" is expected to refer to the variable declaration \"b\" in the same statement.", + ) + + val localAAssignment = body.statements[1] + assertIs( + localAAssignment, + "The first statement of the function \"comprehension_with_list_assignment_and_index_variable_reversed\" is expected to be the initialization of the local variable \"a\" by the statement \"a = 1\" which is expected to be represented by an AssignExpression in the CPG.", + ) + val localARef = localAAssignment.lhs[0] + assertIs( + localARef, + "The left hand side of the assignment \"a = 1\" is expected to be represented by a Reference with localName \"a\" in the CPG.", + ) + val aDeclaration = localAAssignment.variables["a"] + assertIs( + aDeclaration, + "There must be a VariableDeclaration with the local name \"a\" inside the first statement of the function \"comprehension_with_list_assignment_and_index_variable_reversed\".", + ) + assertRefersTo( + localARef, + aDeclaration, + "The reference \"a\" is expected to refer to the variable declaration \"a\" in the same statement.", + ) + + // Check if the AST of the list comprehension fits our expectations. + val listComprehensionWithTupleAndAssignmentToListElement = body.statements[2] + assertIs( + listComprehensionWithTupleAndAssignmentToListElement, + "The third statement of the function \"comprehension_with_list_assignment_and_index_variable_reversed\" is expected to be python's list comprehension \"[a for (b[a], a) in [('this', 0), ('is', 1), ('fun', 2)]]\" which is represented by a CollectionComprehension in the CPG", + ) + + val comprehensionExpression = + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions + .singleOrNull() + assertEquals( + 1, + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions.size, + "There is expected to be exactly one CollectionComprehension in the list comprehension \"[a for (b[a], a) in [('this', 0), ('is', 1), ('fun', 2)]]\". It represents the code's part \"for (b[a], a) in [('this', 0), ('is', 1), ('fun', 2)]\"", + ) + assertIs( + comprehensionExpression, + "There is expected to be exactly one ComprehensionExpression in the list comprehension \"[a for (b[a], a) in [('this', 0), ('is', 1), ('fun', 2)]]\". It represents the code's part \"for (b[a], a) in [('this', 0), ('is', 1), ('fun', 2)]\"", + ) + + val tuple = comprehensionExpression.variable + assertIs( + tuple, + "The variable of the ComprehensionExpression is the tuple \"(b[a], a)\" which is expected to be represented by an InitializerListExpression in the CPG.", + ) + assertEquals( + 2, + tuple.initializers.size, + "The tuple \"(b[a], a)\" represented by an InitializerListExpression in the CPG is expected to have exactly two elements.", + ) + val accessBA = tuple.initializers[0] + assertIs( + accessBA, + "The first element of the tuple \"(b[a], a)\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"a\" which refers to no variable available at this point.", + ) + val refB = accessBA.arrayExpression + assertIs( + refB, + "The first element of the tuple \"(b[a], a)\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"a\" which refers to no variable available at this point.", + ) + assertLocalName( + "b", + refB, + "The first element of the tuple \"(b[a], a)\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"a\" which refers to no variable available at this point.", + ) + val index = accessBA.subscriptExpression + assertIs( + index, + "The first element of the tuple \"(b[a], a)\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"a\" which refers to no variable available at this point.", + ) + assertLocalName( + "a", + index, + "The first element of the tuple \"(b[a], a)\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"a\" which refers to no variable available at this point.", + ) + val refA = tuple.initializers[1] + assertIs( + refA, + "The second element of the tuple \"(b[a], a)\" is expected to be represented by a Reference with localName \"a\" in the CPG.", + ) + assertLocalName( + "a", + refA, + "The second element of the tuple \"(b[a], a)\" is expected to be represented by a Reference with localName \"a\" in the CPG.", + ) + + // Now the actually interesting part: We check for variables belonging to the references. + val innerVariableDeclarationA = refA.refersTo + assertIs( + innerVariableDeclarationA, + "We expect that the reference \"a\" in the second element of the tuple refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertIs( + innerVariableDeclarationA.scope, + "We expect that the reference \"a\" refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertEquals( + listComprehensionWithTupleAndAssignmentToListElement, + innerVariableDeclarationA.scope?.astNode, + "We expect that the reference \"a\" refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertRefersTo( + refB, + bDeclaration, + "We expect that the reference \"b\" in the tuple refers to the VariableDeclaration of \"b\" which is added outside the list comprehension (in statement 0).", + ) + assertEquals( + 0, + index.prevDFG.size, + "We expect that the reference \"a\" used as an index in the first element of the tuple does not have any incoming data flows which somewhat simulates that it's not initialized at this point in time which is also why python crashes.", + ) + assertNotRefersTo( + index, + aDeclaration, + "We expect that the reference \"a\" used as an index in the first element of the tuple does not refer to the same VariableDeclaration as the second element of the tuple nor to the local variable nor does it have an own VariableDeclaration since python would just crash.", + ) + /*assertNotRefersTo( + index, + innerVariableDeclarationA, + "We expect that the reference \"a\" used as an index in the first element of the tuple does not refer to the same VariableDeclaration as the second element of the tuple nor to the local variable nor does it have an own VariableDeclaration since python would just crash.", + ) + assertNull( + index.refersTo, + "We expect that the reference \"a\" used as an index in the first element of the tuple does not refer to the same VariableDeclaration as the second element of the tuple nor to the local variable nor does it have an own VariableDeclaration since python would just crash.", + )*/ + } + + @Test + fun testComprehensionWithListAssignmentAndLocalIndexVariable() { + val comprehensionWithListAssignmentAndLocalIndexVariableFunctionDeclaration = + result.functions["comprehension_with_list_assignment_and_local_index_variable"] + assertIs( + comprehensionWithListAssignmentAndLocalIndexVariableFunctionDeclaration, + "There must be a function called \"comprehension_with_list_assignment_and_local_index_variable\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + // Get the body + val body = comprehensionWithListAssignmentAndLocalIndexVariableFunctionDeclaration.body + assertIs( + body, + "The body of each function is modeled as a Block in the CPG. This must also apply to the function \"comprehension_with_list_assignment_and_local_index_variable\".", + ) + + val listBInitialization = body.statements[0] + assertIs( + listBInitialization, + "The first statement of the function \"comprehension_with_list_assignment_and_local_index_variable\" is expected to be the initialization of list \"b\" by the statement \"b = [0, 1, 2]\" which is expected to be represented by an AssignExpression in the CPG.", + ) + val refBFirstStatement = listBInitialization.lhs[0] + assertIs( + refBFirstStatement, + "The left hand side of the assignment \"b = [0, 1, 2]\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + assertLocalName( + "b", + refBFirstStatement, + "The left hand side of the assignment \"b = [0, 1, 2]\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + val bDeclaration = listBInitialization.variables["b"] + assertIs( + bDeclaration, + "There must be a VariableDeclaration with the local name \"b\" inside the first statement of the function \"comprehension_with_list_assignment_and_local_index_variable\".", + ) + assertRefersTo( + refBFirstStatement, + bDeclaration, + "The reference \"b\" is expected to refer to the variable declaration \"b\" in the same statement.", + ) + + val localCAssignment = body.statements[1] + assertIs( + localCAssignment, + "The first statement of the function \"comprehension_with_list_assignment_and_local_index_variable\" is expected to be the initialization of the local variable \"c\" by the statement \"c = 1\" which is expected to be represented by an AssignExpression in the CPG.", + ) + val localCRef = localCAssignment.lhs[0] + assertIs( + localCRef, + "The left hand side of the assignment \"c = 1\" is expected to be represented by a Reference with localName \"c\" in the CPG.", + ) + val cDeclaration = localCAssignment.variables["c"] + assertIs( + cDeclaration, + "There must be a VariableDeclaration with the local name \"c\" inside the first statement of the function \"comprehension_with_list_assignment_and_local_index_variable\".", + ) + assertRefersTo( + localCRef, + cDeclaration, + "The reference \"c\" is expected to refer to the variable declaration \"c\" in the same statement.", + ) + + // Check if the AST of the list comprehension fits our expectations. + val listComprehensionWithTupleAndAssignmentToListElement = body.statements[2] + assertIs( + listComprehensionWithTupleAndAssignmentToListElement, + "The third statement of the function \"comprehension_with_list_assignment_and_local_index_variable\" is expected to be python's list comprehension \"[a for (b[c], a) in [('this', 0), ('is', 1), ('fun', 2)]]\" which is represented by a CollectionComprehension in the CPG", + ) + + val comprehensionExpression = + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions + .singleOrNull() + assertEquals( + 1, + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions.size, + "There is expected to be exactly one CollectionComprehension in the list comprehension \"[a for (b[c], a) in [('this', 0), ('is', 1), ('fun', 2)]]\". It represents the code's part \"for (b[c], a) in [('this', 0), ('is', 1), ('fun', 2)]\"", + ) + assertIs( + comprehensionExpression, + "There is expected to be exactly one ComprehensionExpression in the list comprehension \"[a for (b[c], a) in [('this', 0), ('is', 1), ('fun', 2)]]\". It represents the code's part \"for (b[c], a) in [('this', 0), ('is', 1), ('fun', 2)]\"", + ) + + val tuple = comprehensionExpression.variable + assertIs( + tuple, + "The variable of the ComprehensionExpression is the tuple \"(b[c], a)\" which is expected to be represented by an InitializerListExpression in the CPG.", + ) + assertEquals( + 2, + tuple.initializers.size, + "The tuple \"(b[c], a)\" represented by an InitializerListExpression in the CPG is expected to have exactly two elements.", + ) + val accessBA = tuple.initializers[0] + assertIs( + accessBA, + "The first element of the tuple \"(b[c], a)\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"c\" which refers to the local variable.", + ) + val refB = accessBA.arrayExpression + assertIs( + refB, + "The first element of the tuple \"(b[c], a)\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"c\" which refers to the local variable.", + ) + assertLocalName( + "b", + refB, + "The first element of the tuple \"(b[c], a)\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"c\" which refers to the local variable.", + ) + val index = accessBA.subscriptExpression + assertIs( + index, + "The first element of the tuple \"(b[c], a)\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"c\" which refers to the local variable.", + ) + assertLocalName( + "c", + index, + "The first element of the tuple \"(b[c], a)\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Reference \"c\" which refers to the local variable.", + ) + val refA = tuple.initializers[1] + assertIs( + refA, + "The second element of the tuple \"(b[c], a)\" is expected to be represented by a Reference with localName \"a\" in the CPG.", + ) + assertLocalName( + "a", + refA, + "The second element of the tuple \"(b[c], a)\" is expected to be represented by a Reference with localName \"a\" in the CPG.", + ) + + // Now the actually interesting part: We check for variables belonging to the references. + val variableDeclarationA = refA.refersTo + assertIs( + variableDeclarationA, + "We expect that the reference \"a\" in the second element of the tuple refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertIs( + variableDeclarationA.scope, + "We expect that the reference \"a\" refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertEquals( + listComprehensionWithTupleAndAssignmentToListElement, + variableDeclarationA.scope?.astNode, + "We expect that the reference \"a\" refers to a VariableDeclaration with localName \"a\" which is not null and whose scope is the LocalScope of the list comprehension.", + ) + assertRefersTo( + refB, + bDeclaration, + "We expect that the reference \"b\" in the tuple refers to the VariableDeclaration of \"b\" which is added outside the list comprehension (in statement 0).", + ) + assertRefersTo( + index, + cDeclaration, + "We expect that the reference \"c\" used as an index in the first element of the tuple refers to the local variable \"c\" (in statement 1).", + ) + } + + @Test + fun testListComprehensionToListIndex() { + val moreLoopVariablesFunctionDeclaration = + result.functions["list_comprehension_to_list_index"] + assertIs( + moreLoopVariablesFunctionDeclaration, + "There must be a function called \"list_comprehension_to_list_index\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + // Get the body + val body = moreLoopVariablesFunctionDeclaration.body + assertIs( + body, + "The body of each function is modeled as a Block in the CPG. This must also apply to the function \"list_comprehension_to_list_index\".", + ) + + val listBInitialization = body.statements[0] + assertIs( + listBInitialization, + "The first statement of the function \"list_comprehension_to_list_index\" is expected to be the initialization of list \"b\" by the statement \"b = [0, 1, 2]\" which is expected to be represented by an AssignExpression in the CPG.", + ) + val refBFirstStatement = listBInitialization.lhs[0] + assertIs( + refBFirstStatement, + "The left hand side of the assignment \"b = [0, 1, 2]\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + assertLocalName( + "b", + refBFirstStatement, + "The left hand side of the assignment \"b = [0, 1, 2]\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + val bDeclaration = listBInitialization.variables["b"] + assertIs( + bDeclaration, + "There must be a VariableDeclaration with the local name \"b\" inside the first statement of the function \"list_comprehension_to_list_index\".", + ) + assertRefersTo( + refBFirstStatement, + bDeclaration, + "The reference \"b\" is expected to refer to the variable declaration \"b\" in the same statement.", + ) + + // Check if the AST of the list comprehension fits our expectations. + val listComprehensionWithTupleAndAssignmentToListElement = body.statements[1] + assertIs( + listComprehensionWithTupleAndAssignmentToListElement, + "The second statement of the function \"list_comprehension_to_list_index\" is expected to be python's list comprehension \"[b[0] for b[0] in ['this', 'is', 'fun']]\" which is represented by a CollectionComprehension in the CPG", + ) + + val comprehensionExpression = + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions + .singleOrNull() + assertEquals( + 1, + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions.size, + "There is expected to be exactly one CollectionComprehension in the list comprehension \"[b[0] for b[0] in ['this', 'is', 'fun']]\". It represents the code's part \"for b[0] in ['this', 'is', 'fun']\"", + ) + assertIs( + comprehensionExpression, + "There is expected to be exactly one ComprehensionExpression in the list comprehension \"[b[0] for b[0] in ['this', 'is', 'fun']]\". It represents the code's part \"for b[0] in ['this', 'is', 'fun']\"", + ) + + val accessB0 = comprehensionExpression.variable + assertIs( + accessB0, + "The control variable of the ComprehensionExpression is \"b[0]\" which is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Literal with value \"0\".", + ) + val refB = accessB0.arrayExpression + assertIs( + refB, + "The control variable \"b[0]\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Literal with value \"0\" (with kotlin type Long ).", + ) + assertLocalName( + "b", + refB, + "The control variable \"b[0]\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Literal with value \"0\" (with kotlin type Long ).", + ) + val index = accessB0.subscriptExpression + assertIs>( + index, + "The control variable \"b[0]\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Literal with value \"0\" (with kotlin type Long ).", + ) + assertLiteralValue( + 0L, + index, + "The control variable \"b[0]\" is expected to be represented by a SubscriptExpression with in the CPG. We expect that the base is a Reference with localName \"b\" and the subscriptExpression representing the index is a Literal with value \"0\" (with kotlin type Long ).", + ) + + // Now the actually interesting part: We check for variables belonging to the references. + assertRefersTo( + refB, + bDeclaration, + "We expect that the reference \"b\" in the control variable refers to the VariableDeclaration of \"b\" which is added outside the list comprehension (in statement 0).", + ) + } + + @Test + fun testListComprehensionToField() { + val listComprehensionToFieldFunctionDeclaration = + result.functions["list_comprehension_to_field"] + assertIs( + listComprehensionToFieldFunctionDeclaration, + "There must be a function called \"list_comprehension_to_field\" in the file. It must be neither null nor any other class than a FunctionDeclaration.", + ) + + // Get the body + val body = listComprehensionToFieldFunctionDeclaration.body + assertIs( + body, + "The body of each function is modeled as a Block in the CPG. This must also apply to the function \"list_comprehension_to_field\".", + ) + + val listBInitialization = body.statements[0] + assertIs( + listBInitialization, + "The first statement of the function \"list_comprehension_to_field\" is expected to be the initialization of list \"b\" by the statement \"b = Magic()\" which is expected to be represented by an AssignExpression in the CPG.", + ) + val refBFirstStatement = listBInitialization.lhs[0] + assertIs( + refBFirstStatement, + "The left hand side of the assignment \"b = Magic()\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + assertLocalName( + "b", + refBFirstStatement, + "The left hand side of the assignment \"b = Magic())\" is expected to be represented by a Reference with localName \"b\" in the CPG.", + ) + val bDeclaration = listBInitialization.variables["b"] + assertIs( + bDeclaration, + "There must be a VariableDeclaration with the local name \"b\" inside the first statement of the function \"list_comprehension_to_field\".", + ) + assertRefersTo( + refBFirstStatement, + bDeclaration, + "The reference \"b\" is expected to refer to the variable declaration \"b\" in the same statement.", + ) + + // Check if the AST of the list comprehension fits our expectations. + val listComprehensionWithTupleAndAssignmentToListElement = body.statements[1] + assertIs( + listComprehensionWithTupleAndAssignmentToListElement, + "The second statement of the function \"list_comprehension_to_field\" is expected to be python's list comprehension \"[b.a for b.a in ['this', 'is', 'fun']]\" which is represented by a CollectionComprehension in the CPG.", + ) + + val comprehensionExpression = + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions + .singleOrNull() + assertEquals( + 1, + listComprehensionWithTupleAndAssignmentToListElement.comprehensionExpressions.size, + "There is expected to be exactly one CollectionComprehension in the list comprehension \"[b.a for b.a in ['this', 'is', 'fun']]\". It represents the code's part \"for b.a in ['this', 'is', 'fun']\"", + ) + assertIs( + comprehensionExpression, + "There is expected to be exactly one ComprehensionExpression in the list comprehension \"[b.a for b.a in ['this', 'is', 'fun']]\". It represents the code's part \"for b.a in ['this', 'is', 'fun']\"", + ) + + val bMemberA = comprehensionExpression.variable + assertIs( + bMemberA, + "The control variable \"b.a\" is expected to be represented by a MemberExpression in the CPG. We expect that the base is a Reference with localName \"b\" and the localName of the MemberExpression is \"a\".", + ) + val refB = bMemberA.base + assertIs( + refB, + "The control variable \"b.a\" is expected to be represented by a MemberExpression in the CPG. We expect that the base is a Reference with localName \"b\" and the localName of the MemberExpression is \"a\".", + ) + assertLocalName( + "b", + refB, + "The control variable \"b.a\" is expected to be represented by a MemberExpression in the CPG. We expect that the base is a Reference with localName \"b\" and the localName of the MemberExpression is \"a\".", + ) + assertLocalName( + "a", + bMemberA, + "The control variable \"b.a\" is expected to be represented by a MemberExpression in the CPG. We expect that the base is a Reference with localName \"b\" and the localName of the MemberExpression is \"a\".", + ) + + // Now the actually interesting part: We check for variables belonging to the references. + assertRefersTo( + refB, + bDeclaration, + "We expect that the reference \"b\" used in the control variable refers to the VariableDeclaration of \"b\" which is added outside the list comprehension (in statement 0).", + ) + + val magicClass = result.records["Magic"] + assertIs( + magicClass, + "There must be a class called \"Magic\" in the file. It must be neither null nor any other class than a RecordDeclaration which is expected to model python classes in the CPG.", + ) + assertEquals( + 1, + magicClass.fields.size, + "We expect exactly one field inside the record declaration representing the class \"Magic\" and that's the field which we expect to represent the class' attribute \"a\".", + ) + val fieldA = magicClass.fields["a"] + assertIs( + fieldA, + "We expect exactly one field inside the record declaration representing the class \"Magic\" and that's the field which we expect to represent the class' attribute \"a\".", + ) + + assertRefersTo( + bMemberA, + fieldA, + "We expect that the member expression \"b.a\" used as control variable refers to the FieldDeclaration \"a\" of the class \"Magic\".", + ) + } +} diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt index 2f81c21425d..47b61fc7853 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandlerTest.kt @@ -26,249 +26,12 @@ package de.fraunhofer.aisec.cpg.frontends.python import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Block -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CallExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.CollectionComprehension -import de.fraunhofer.aisec.cpg.graph.statements.expressions.KeyValueExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.test.analyze -import de.fraunhofer.aisec.cpg.test.assertLiteralValue -import de.fraunhofer.aisec.cpg.test.assertLocalName +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.test.* import java.nio.file.Path import kotlin.test.* class ExpressionHandlerTest { - @Test - fun testListComprehensions() { - val topLevel = Path.of("src", "test", "resources", "python") - val result = - analyze(listOf(topLevel.resolve("comprehension.py").toFile()), topLevel, true) { - it.registerLanguage() - } - assertNotNull(result) - val listComp = result.functions["listComp"] - assertNotNull(listComp) - - val body = listComp.body - assertIs(body) - val singleWithIfAssignment = body.statements[0] - assertIs(singleWithIfAssignment) - val singleWithIf = singleWithIfAssignment.rhs[0] - assertIs(singleWithIf) - assertIs(singleWithIf.statement) - assertEquals(1, singleWithIf.comprehensionExpressions.size) - assertLocalName("i", singleWithIf.comprehensionExpressions[0].variable) - assertIs(singleWithIf.comprehensionExpressions[0].iterable) - assertLocalName("x", singleWithIf.comprehensionExpressions[0].iterable) - val ifPredicate = singleWithIf.comprehensionExpressions[0].predicate - assertIs(ifPredicate) - assertEquals("==", ifPredicate.operatorCode) - - val singleWithoutIfAssignment = body.statements[1] - assertIs(singleWithoutIfAssignment) - val singleWithoutIf = singleWithoutIfAssignment.rhs[0] - assertIs(singleWithoutIf) - assertIs(singleWithoutIf.statement) - assertEquals(1, singleWithoutIf.comprehensionExpressions.size) - assertLocalName("i", singleWithoutIf.comprehensionExpressions[0].variable) - assertIs(singleWithoutIf.comprehensionExpressions[0].iterable) - assertLocalName("x", singleWithoutIf.comprehensionExpressions[0].iterable) - assertNull(singleWithoutIf.comprehensionExpressions[0].predicate) - - val singleWithDoubleIfAssignment = body.statements[2] - assertIs(singleWithDoubleIfAssignment) - val singleWithDoubleIf = singleWithDoubleIfAssignment.rhs[0] - assertIs(singleWithDoubleIf) - assertIs(singleWithDoubleIf.statement) - assertEquals(1, singleWithDoubleIf.comprehensionExpressions.size) - assertLocalName("i", singleWithDoubleIf.comprehensionExpressions[0].variable) - assertIs(singleWithDoubleIf.comprehensionExpressions[0].iterable) - assertLocalName("x", singleWithDoubleIf.comprehensionExpressions[0].iterable) - val doubleIfPredicate = singleWithDoubleIf.comprehensionExpressions[0].predicate - assertIs(doubleIfPredicate) - assertEquals("and", doubleIfPredicate.operatorCode) - - val doubleAssignment = body.statements[3] as? AssignExpression - assertIs(doubleAssignment) - val double = doubleAssignment.rhs[0] as? CollectionComprehension - assertNotNull(double) - assertIs(double.statement) - assertEquals(2, double.comprehensionExpressions.size) - // TODO: Add tests on the comprehension expressions - } - - @Test - fun testSetComprehensions() { - val topLevel = Path.of("src", "test", "resources", "python") - val result = - analyze(listOf(topLevel.resolve("comprehension.py").toFile()), topLevel, true) { - it.registerLanguage() - } - assertNotNull(result) - val listComp = result.functions["setComp"] - assertNotNull(listComp) - - val body = listComp.body as? Block - assertNotNull(body) - val singleWithIfAssignment = body.statements[0] - assertIs(singleWithIfAssignment) - val singleWithIf = singleWithIfAssignment.rhs[0] - assertIs(singleWithIf) - assertIs(singleWithIf.statement) - assertEquals(1, singleWithIf.comprehensionExpressions.size) - assertLocalName("i", singleWithIf.comprehensionExpressions[0].variable) - assertIs(singleWithIf.comprehensionExpressions[0].iterable) - assertLocalName("x", singleWithIf.comprehensionExpressions[0].iterable) - val ifPredicate = singleWithIf.comprehensionExpressions[0].predicate - assertIs(ifPredicate) - assertEquals("==", ifPredicate.operatorCode) - - val singleWithoutIfAssignment = body.statements[1] - assertIs(singleWithoutIfAssignment) - val singleWithoutIf = singleWithoutIfAssignment.rhs[0] - assertIs(singleWithoutIf) - assertIs(singleWithoutIf.statement) - assertEquals(1, singleWithoutIf.comprehensionExpressions.size) - assertLocalName("i", singleWithoutIf.comprehensionExpressions[0].variable) - assertIs(singleWithoutIf.comprehensionExpressions[0].iterable) - assertLocalName("x", singleWithoutIf.comprehensionExpressions[0].iterable) - assertNull(singleWithoutIf.comprehensionExpressions[0].predicate) - - val singleWithDoubleIfAssignment = body.statements[2] - assertIs(singleWithDoubleIfAssignment) - val singleWithDoubleIf = singleWithDoubleIfAssignment.rhs[0] - assertIs(singleWithDoubleIf) - assertIs(singleWithDoubleIf.statement) - assertEquals(1, singleWithDoubleIf.comprehensionExpressions.size) - assertLocalName("i", singleWithDoubleIf.comprehensionExpressions[0].variable) - assertIs(singleWithDoubleIf.comprehensionExpressions[0].iterable) - assertLocalName("x", singleWithDoubleIf.comprehensionExpressions[0].iterable) - val doubleIfPredicate = singleWithDoubleIf.comprehensionExpressions[0].predicate - assertIs(doubleIfPredicate) - assertEquals("and", doubleIfPredicate.operatorCode) - - val doubleAssignment = body.statements[3] - assertIs(doubleAssignment) - val double = doubleAssignment.rhs[0] - assertIs(double) - assertIs(double.statement) - assertEquals(2, double.comprehensionExpressions.size) - } - - @Test - fun testDictComprehensions() { - val topLevel = Path.of("src", "test", "resources", "python") - val result = - analyze(listOf(topLevel.resolve("comprehension.py").toFile()), topLevel, true) { - it.registerLanguage() - } - assertNotNull(result) - val listComp = result.functions["dictComp"] - assertNotNull(listComp) - - val body = listComp.body as? Block - assertNotNull(body) - val singleWithIfAssignment = body.statements[0] - assertIs(singleWithIfAssignment) - val singleWithIf = singleWithIfAssignment.rhs[0] - assertIs(singleWithIf) - var statement = singleWithIf.statement - assertIs(statement) - assertIs(statement.key) - assertLocalName("i", statement.key) - assertIs(statement.value) - assertEquals(1, singleWithIf.comprehensionExpressions.size) - assertLocalName("i", singleWithIf.comprehensionExpressions[0].variable) - assertIs(singleWithIf.comprehensionExpressions[0].iterable) - assertLocalName("x", singleWithIf.comprehensionExpressions[0].iterable) - val ifPredicate = singleWithIf.comprehensionExpressions[0].predicate - assertIs(ifPredicate) - assertEquals("==", ifPredicate.operatorCode) - - val singleWithoutIfAssignment = body.statements[1] - assertIs(singleWithoutIfAssignment) - val singleWithoutIf = singleWithoutIfAssignment.rhs[0] - assertIs(singleWithoutIf) - statement = singleWithIf.statement - assertIs(statement) - assertIs(statement.key) - assertLocalName("i", statement.key) - assertIs(statement.value) - assertEquals(1, singleWithoutIf.comprehensionExpressions.size) - assertLocalName("i", singleWithoutIf.comprehensionExpressions[0].variable) - assertIs(singleWithoutIf.comprehensionExpressions[0].iterable) - assertLocalName("x", singleWithoutIf.comprehensionExpressions[0].iterable) - assertNull(singleWithoutIf.comprehensionExpressions[0].predicate) - - val singleWithDoubleIfAssignment = body.statements[2] - assertIs(singleWithDoubleIfAssignment) - val singleWithDoubleIf = singleWithDoubleIfAssignment.rhs[0] - assertIs(singleWithDoubleIf) - statement = singleWithIf.statement - assertIs(statement) - assertIs(statement.key) - assertLocalName("i", statement.key) - assertIs(statement.value) - assertEquals(1, singleWithDoubleIf.comprehensionExpressions.size) - assertLocalName("i", singleWithDoubleIf.comprehensionExpressions[0].variable) - assertIs(singleWithDoubleIf.comprehensionExpressions[0].iterable) - assertLocalName("x", singleWithDoubleIf.comprehensionExpressions[0].iterable) - val doubleIfPredicate = singleWithDoubleIf.comprehensionExpressions[0].predicate - assertIs(doubleIfPredicate) - assertEquals("and", doubleIfPredicate.operatorCode) - - val doubleAssignment = body.statements[3] as? AssignExpression - assertIs(doubleAssignment) - val double = doubleAssignment.rhs[0] as? CollectionComprehension - assertNotNull(double) - statement = singleWithIf.statement - assertIs(statement) - assertIs(statement.key) - assertLocalName("i", statement.key) - assertIs(statement.value) - assertEquals(2, double.comprehensionExpressions.size) - } - - @Test - fun testGeneratorExpr() { - val topLevel = Path.of("src", "test", "resources", "python") - val result = - analyze(listOf(topLevel.resolve("comprehension.py").toFile()), topLevel, true) { - it.registerLanguage() - } - assertNotNull(result) - val listComp = result.functions["generator"] - assertNotNull(listComp) - - val body = listComp.body as? Block - assertNotNull(body) - val singleWithIfAssignment = body.statements[0] - assertIs(singleWithIfAssignment) - val singleWithIf = singleWithIfAssignment.rhs[0] - assertIs(singleWithIf) - assertIs(singleWithIf.statement) - assertEquals(1, singleWithIf.comprehensionExpressions.size) - assertLocalName("i", singleWithIf.comprehensionExpressions[0].variable) - assertIs(singleWithIf.comprehensionExpressions[0].iterable) - assertLocalName("range", singleWithIf.comprehensionExpressions[0].iterable) - val ifPredicate = singleWithIf.comprehensionExpressions[0].predicate - assertIs(ifPredicate) - assertEquals("==", ifPredicate.operatorCode) - - val singleWithoutIfAssignment = body.statements[1] - assertIs(singleWithoutIfAssignment) - val singleWithoutIf = singleWithoutIfAssignment.rhs[0] - assertIs(singleWithoutIf) - assertIs(singleWithoutIf.statement) - assertEquals(1, singleWithoutIf.comprehensionExpressions.size) - assertLocalName("i", singleWithoutIf.comprehensionExpressions[0].variable) - assertIs(singleWithIf.comprehensionExpressions[0].iterable) - assertLocalName("range", singleWithIf.comprehensionExpressions[0].iterable) - assertNull(singleWithoutIf.comprehensionExpressions[0].predicate) - } - @Test fun testBoolOps() { val topLevel = Path.of("src", "test", "resources", "python") diff --git a/cpg-language-python/src/test/resources/python/comprehension.py b/cpg-language-python/src/test/resources/python/comprehension.py index 7d6f2568ade..207ae093194 100644 --- a/cpg-language-python/src/test/resources/python/comprehension.py +++ b/cpg-language-python/src/test/resources/python/comprehension.py @@ -1,19 +1,20 @@ def foo(arg): return 7 -def listComp(x, y): +def list_comp(x, y): a = [foo(i) for i in x if i == 10] b = [foo(i) for i in x] c = {foo(i) for i in x if i == 10 if i < 20} d = [foo(i) for z in y if z in x for i in z if i == 10 ] + foo(i) -def setComp(x, y): +def set_comp(x, y): a = {foo(i) for i in x if i == 10} b = {foo(i) for i in x} c = {foo(i) for i in x if i == 10 if i < 20} d = {foo(i) for z in y if z in x for i in z if i == 10 } -def dictComp(x, y): +def dict_comp(x, y): a = {i: foo(i) for i in x if i == 10} b = {i: foo(i) for i in x} c = {i: foo(i) for i in x if i == 10 if i < 20} @@ -21,4 +22,66 @@ def dictComp(x, y): def generator(x, y): a = (i**2 for i in range(10) if i == 10) - b = (i**2 for i in range(10)) \ No newline at end of file + b = (i**2 for i in range(10)) + +def bar(k, v): + return k+v + +def tuple_comp(x): + a = [bar(k, v) for (k, v) in x] + +def comp_binding(foo): + # As of Python 3, none of the comprehensions should bind to the outer x + x = 42 + [x for x in foo] + {x for x in foo} + {x: x for x in foo} + {x**2 for x in foo} + print(x) # this prints 42 + +def comp_binding_assign_expr(foo): + # https://peps.python.org/pep-0572/#scope-of-the-target + x = 42 + [(x := temp) for temp in foo] + print(x) # doesn't print 42 + +def comp_binding_assign_expr_nested(foo): + # https://peps.python.org/pep-0572/#scope-of-the-target + x = 42 + [[(x := temp) for temp in foo] for a in foo] + print(x) # doesn't print 42 + +def comprehension_with_list_assignment(): + b = [0, 1, 2] + [a for (a, b[0]) in [(1, 2), (2, 4), (3, 6)]] + print(b) # prints [6, 1, 2] + +def comprehension_with_list_assignment_and_index_variable(): + b = [0, 1, 2] + [a for (a, b[a]) in [(0, 'this'), (1, 'is'), (2, 'fun')]] + print(b) # prints ['this', 'is', 'fun'] + +def comprehension_with_list_assignment_and_index_variable_reversed(): + b = [0, 1, 2] + a = 1 + [a for (b[a], a) in [('this', 0), ('is', 1), ('fun', 2)]] # This crashes because the "a" in the tuple shadows the outer variable and "UnboundLocalError: cannot access local variable 'a' where it is not associated with a value". + print(b) # prints nothing due to crash + +def comprehension_with_list_assignment_and_local_index_variable(): + b = [0, 1, 2] + c = 1 + [a for (b[c], a) in [('this', 0), ('is', 1), ('fun', 2)]] + print(b) # prints [0, 'fun', 2] + +def list_comprehension_to_list_index(): + b = [0, 1, 2] + [b[0] for b[0] in ['this', 'is', 'fun']] + print(b) # prints ['fun', 1, 2] + +class Magic: + a = 7 + +def list_comprehension_to_field(): + b = Magic() + [b.a for b.a in ['this', 'is', 'fun']] + print(b.a) # prints 'fun' \ No newline at end of file