From 11f62db5629dd040fa317105c44686ca8de268de Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 29 Jan 2025 22:33:54 +0100 Subject: [PATCH 1/8] Providing a new `HasDynamicVariable` language trait Work in progress. Also tries to merge the python declaration pass into the symbol resolver. --- .../aisec/cpg/frontends/LanguageTraits.kt | 11 + .../aisec/cpg/passes/SymbolResolver.kt | 7 + .../python/DynamicDeclarationHelper.kt | 203 ++++++++++++++ .../cpg/frontends/python/PythonLanguage.kt | 34 ++- .../python/PythonLanguageFrontend.kt | 3 - .../cpg/passes/PythonAddDeclarationsPass.kt | 252 ------------------ 6 files changed, 251 insertions(+), 259 deletions(-) create mode 100644 cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt delete mode 100644 cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt index c28bf7b2dd..206cfbb210 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/LanguageTraits.kt @@ -31,6 +31,7 @@ import de.fraunhofer.aisec.cpg.graph.HasOperatorCode import de.fraunhofer.aisec.cpg.graph.HasOverloadedOperation import de.fraunhofer.aisec.cpg.graph.LanguageProvider import de.fraunhofer.aisec.cpg.graph.Name +import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.FunctionDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.RecordDeclaration import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration @@ -298,6 +299,16 @@ inline infix fun KClass.of( return Pair(T::class, operatorCode) } +/** + * A language trait that specifies that this language has dynamic declarations, meaning that + * declarations can be added to the symbol table at runtime. Since we are a static analysis tools, + * we can only deliver an approximation to the actual behaviour. + */ +interface HasDynamicDeclarations : LanguageTrait { + + fun SymbolResolver.provideDeclaration(ref: Reference): Declaration? +} + /** Checks whether the name for a function (as [CharSequence]) is a known operator name. */ context(LanguageProvider) val CharSequence.isKnownOperatorName: Boolean diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt index cd6ca6cc57..c75d01bccd 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt @@ -194,6 +194,13 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { val language = ref.language val helperType = ref.resolutionHelper?.type + // Before we resolve anything, we give languages with dynamic declarations a chance to + // declare any variables they need. This is most likely only needed for WRITE references, + // but we let the language decide that + if (language is HasDynamicDeclarations) { + with(language) { provideDeclaration(ref) } + } + // Ignore references to anonymous identifiers, if the language supports it (e.g., the _ // identifier in Go) if ( diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt new file mode 100644 index 0000000000..c86d96e0ec --- /dev/null +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2025, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.python + +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.declarations.Declaration +import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration +import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration +import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope +import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference +import de.fraunhofer.aisec.cpg.graph.types.InitializerTypePropagation +import de.fraunhofer.aisec.cpg.graph.types.UnknownType +import de.fraunhofer.aisec.cpg.passes.Pass +import de.fraunhofer.aisec.cpg.passes.Pass.Companion.log + +/** + * This function will create a new dynamic [VariableDeclaration] if there is a write access to the + * [ref]. + */ +internal fun Pass<*>.handleWriteToReference(ref: Reference): VariableDeclaration? { + if (ref.access != AccessValues.WRITE) { + return null + } + + // If this is a member expression, and we do not know the base's type, we cannot create a + // declaration + if (ref is MemberExpression && ref.base.type is UnknownType) { + return null + } + + // Look for a potential scope modifier for this reference + var targetScope = + scopeManager.currentScope?.predefinedLookupScopes[ref.name.toString()]?.targetScope + + // Try to see whether our symbol already exists. There are basically three rules to follow + // here. + var symbol = + when { + // When a target scope is set, then we have a `global` or `nonlocal` keyword for + // this symbol, and we need to start looking in this scope + targetScope != null -> scopeManager.lookupSymbolByNodeName(ref, targetScope) + // When we have a qualified reference (such as `self.a`), we do not have any + // specific restrictions, because the lookup will anyway be a qualified lookup, + // and it will consider only the scope of `self`. + ref.name.isQualified() -> scopeManager.lookupSymbolByNodeName(ref) + // In any other case, we need to restrict the lookup to the current scope. The + // main reason for this is that Python requires the `global` keyword in functions + // for assigning to global variables. See + // https://docs.python.org/3/reference/simple_stmts.html#the-global-statement. So + // basically we need to ignore all global variables at this point and only look + // for local ones. + else -> + scopeManager.lookupSymbolByNodeName(ref) { it.scope == scopeManager.currentScope } + } + + // If the symbol is already defined in the designed scope, there is nothing to create + if (symbol.isNotEmpty()) return null + + // First, check if we need to create a field + var decl: VariableDeclaration? = + when { + // Check, whether we are referring to a "self.X", which would create a field + scopeManager.isInRecord && scopeManager.isInFunction && ref.refersToReceiver -> { + // We need to temporarily jump into the scope of the current record to + // add the field. These are instance attributes + scopeManager.withScope(scopeManager.firstScopeIsInstanceOrNull()) { + newFieldDeclaration(ref.name) + } + } + scopeManager.isInRecord && scopeManager.isInFunction && ref is MemberExpression -> { + // If this is any other member expression, we are usually not interested in + // creating fields, except if this is a receiver + return null + } + scopeManager.isInRecord && !scopeManager.isInFunction -> { + // We end up here for fields declared directly in the class body. These are + // class attributes; more or less static fields. + newFieldDeclaration(scopeManager.currentNamespace.fqn(ref.name.localName)) + } + else -> { + null + } + } + + // If we didn't create any declaration up to this point and are still here, we need to + // create a (local) variable. We need to take scope modifications into account. + if (decl == null) { + decl = + if (targetScope != null) { + scopeManager.withScope(targetScope) { newVariableDeclaration(ref.name) } + } else { + newVariableDeclaration(ref.name) + } + } + + decl.code = ref.code + decl.location = ref.location + decl.isImplicit = true + decl.language = ref.language + + log.debug( + "Creating dynamic {} {} in {}", + if (decl is FieldDeclaration) { + "field" + } else { + "variable" + }, + decl.name, + decl.scope, + ) + + // Make sure we add the declaration at the correct place, i.e. with the scope we set at the + // creation time + scopeManager.withScope(decl.scope) { scopeManager.addDeclaration(decl) } + + return decl +} + +context(Pass<*>) +private val Reference.refersToReceiver: Boolean + get() { + return this is MemberExpression && + this.base.name == scopeManager.currentMethod?.receiver?.name + } + +/** + * Generates a new [VariableDeclaration] if [target] is a [Reference] and there is no existing + * declaration yet. + */ +internal fun Pass<*>.handleAssignmentToTarget( + assignExpression: AssignExpression, + target: Node, + setAccessValue: Boolean = false, +) { + (target as? Reference)?.let { + if (setAccessValue) it.access = AccessValues.WRITE + val handled = handleWriteToReference(target) + if (handled is Declaration) { + // We cannot assign an initializer here because this will lead to duplicate + // DFG edges, but we need to propagate the type information from our value + // to the declaration. We therefore add the declaration to the observers of + // the value's type, so that it gets informed and can change its own type + // accordingly. + assignExpression + .findValue(target) + ?.registerTypeObserver(InitializerTypePropagation(handled)) + + // Add it to our assign expression, so that we can find it in the AST + assignExpression.declarations += handled + } + } +} + +/*private fun Pass<*>.handleAssignExpression(assignExpression: AssignExpression) { + for (target in assignExpression.lhs) { + handleAssignmentToTarget(assignExpression, target, setAccessValue = false) + // If the lhs is an InitializerListExpression, we have to handle the individual elements + // in the initializers. + (target as? InitializerListExpression)?.let { + it.initializers.forEach { initializer -> + handleAssignmentToTarget(assignExpression, initializer, setAccessValue = true) + } + } + } +}*/ + +// New variables can also be declared as `variable` in a [ForEachStatement] +private fun Pass<*>.handleForEach(node: ForEachStatement) { + when (val forVar = node.variable) { + is Reference -> { + val handled = handleWriteToReference(forVar) + if (handled is Declaration) { + handled.let { node.addDeclaration(it) } + } + } + } +} diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt index 3a6c530259..8cf5241c7a 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt @@ -29,13 +29,14 @@ import de.fraunhofer.aisec.cpg.frontends.* import de.fraunhofer.aisec.cpg.graph.HasOverloadedOperation import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.autoType +import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration import de.fraunhofer.aisec.cpg.graph.scopes.Symbol -import de.fraunhofer.aisec.cpg.graph.statements.expressions.BinaryOperator -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.graph.statements.expressions.UnaryOperator +import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement +import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.Util.warnWithFileLocation +import de.fraunhofer.aisec.cpg.passes.SymbolResolver import kotlin.reflect.KClass import org.neo4j.ogm.annotation.Transient @@ -45,7 +46,8 @@ class PythonLanguage : HasShortCircuitOperators, HasOperatorOverloading, HasFunctionStyleConstruction, - HasMemberExpressionAmbiguity { + HasMemberExpressionAmbiguity, + HasDynamicDeclarations { override val fileExtensions = listOf("py", "pyi") override val namespaceDelimiter = "." @Transient @@ -228,6 +230,30 @@ class PythonLanguage : return super.tryCast(type, targetType, hint, targetHint) } + override fun SymbolResolver.provideDeclaration(ref: Reference): Declaration? { + // Completely hacky + if (ref.astParent is AssignExpression) { + handleAssignmentToTarget( + ref.astParent!! as AssignExpression, + ref, + setAccessValue = false, + ) + } else if (ref.astParent is InitializerListExpression) { + handleAssignmentToTarget( + ref.astParent!!.astParent as AssignExpression, + ref, + setAccessValue = false, + ) + } else if (ref.astParent is ForEachStatement) { + val handled = handleWriteToReference(ref) + if (handled is Declaration) { + handled.let { (ref.astParent as ForEachStatement).addDeclaration(it) } + } + } + + return null + } + companion object { /** * This is a "modifier" to differentiate parameters in functions that are "positional" only. diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt index 088985c59b..9046509a31 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt @@ -36,8 +36,6 @@ import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration import de.fraunhofer.aisec.cpg.graph.types.AutoType import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.helpers.CommentMatcher -import de.fraunhofer.aisec.cpg.passes.PythonAddDeclarationsPass -import de.fraunhofer.aisec.cpg.passes.configuration.RegisterExtraPass import de.fraunhofer.aisec.cpg.sarif.PhysicalLocation import de.fraunhofer.aisec.cpg.sarif.Region import java.io.File @@ -62,7 +60,6 @@ import kotlin.math.min * are represented by a dictionary. This pass adds a declaration for each variable that is assigned * a value (on the first assignment). */ -@RegisterExtraPass(PythonAddDeclarationsPass::class) class PythonLanguageFrontend(language: Language, ctx: TranslationContext) : LanguageFrontend(language, ctx) { val lineSeparator = "\n" // TODO diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt deleted file mode 100644 index d6760829c4..0000000000 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/PythonAddDeclarationsPass.kt +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Copyright (c) 2023, Fraunhofer AISEC. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * $$$$$$\ $$$$$$$\ $$$$$$\ - * $$ __$$\ $$ __$$\ $$ __$$\ - * $$ / \__|$$ | $$ |$$ / \__| - * $$ | $$$$$$$ |$$ |$$$$\ - * $$ | $$ ____/ $$ |\_$$ | - * $$ | $$\ $$ | $$ | $$ | - * \$$$$$ |$$ | \$$$$$ | - * \______/ \__| \______/ - * - */ -package de.fraunhofer.aisec.cpg.passes - -import de.fraunhofer.aisec.cpg.TranslationContext -import de.fraunhofer.aisec.cpg.frontends.Language -import de.fraunhofer.aisec.cpg.frontends.UnknownLanguage -import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage -import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguageFrontend -import de.fraunhofer.aisec.cpg.graph.* -import de.fraunhofer.aisec.cpg.graph.declarations.Declaration -import de.fraunhofer.aisec.cpg.graph.declarations.FieldDeclaration -import de.fraunhofer.aisec.cpg.graph.declarations.VariableDeclaration -import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope -import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement -import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.InitializerListExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression -import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference -import de.fraunhofer.aisec.cpg.graph.types.InitializerTypePropagation -import de.fraunhofer.aisec.cpg.graph.types.UnknownType -import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker -import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteBefore -import de.fraunhofer.aisec.cpg.passes.configuration.RequiredFrontend - -@ExecuteBefore(ImportResolver::class) -@ExecuteBefore(SymbolResolver::class) -@RequiredFrontend(PythonLanguageFrontend::class) -class PythonAddDeclarationsPass(ctx: TranslationContext) : ComponentPass(ctx), LanguageProvider { - - lateinit var walker: SubgraphWalker.ScopedWalker - - override fun cleanup() { - // nothing to do - } - - override fun accept(p0: Component) { - walker = SubgraphWalker.ScopedWalker(ctx.scopeManager) - walker.registerHandler { _, _, currNode -> handle(currNode) } - - for (tu in p0.translationUnits) { - walker.iterate(tu) - } - } - - /** - * This function checks for each [AssignExpression] whether there is already a matching variable - * or not. New variables can be one of: - * - [FieldDeclaration] if we are currently in a record - * - [VariableDeclaration] otherwise - * - * TODO: loops - */ - private fun handle(node: Node?) { - when (node) { - is AssignExpression -> handleAssignExpression(node) - is ForEachStatement -> handleForEach(node) - else -> { - // Nothing to do for all other types of nodes - } - } - } - - /** - * This function will create a new dynamic [VariableDeclaration] if there is a write access to - * the [ref]. - */ - private fun handleWriteToReference(ref: Reference): VariableDeclaration? { - if (ref.access != AccessValues.WRITE) { - return null - } - - // If this is a member expression, and we do not know the base's type, we cannot create a - // declaration - if (ref is MemberExpression && ref.base.type is UnknownType) { - return null - } - - // Look for a potential scope modifier for this reference - var targetScope = - scopeManager.currentScope?.predefinedLookupScopes[ref.name.toString()]?.targetScope - - // Try to see whether our symbol already exists. There are basically three rules to follow - // here. - var symbol = - when { - // When a target scope is set, then we have a `global` or `nonlocal` keyword for - // this symbol, and we need to start looking in this scope - targetScope != null -> scopeManager.lookupSymbolByNodeName(ref, targetScope) - // When we have a qualified reference (such as `self.a`), we do not have any - // specific restrictions, because the lookup will anyway be a qualified lookup, - // and it will consider only the scope of `self`. - ref.name.isQualified() -> scopeManager.lookupSymbolByNodeName(ref) - // In any other case, we need to restrict the lookup to the current scope. The - // main reason for this is that Python requires the `global` keyword in functions - // for assigning to global variables. See - // https://docs.python.org/3/reference/simple_stmts.html#the-global-statement. So - // basically we need to ignore all global variables at this point and only look - // for local ones. - else -> - scopeManager.lookupSymbolByNodeName(ref) { - it.scope == scopeManager.currentScope - } - } - - // If the symbol is already defined in the designed scope, there is nothing to create - if (symbol.isNotEmpty()) return null - - // First, check if we need to create a field - var decl: VariableDeclaration? = - when { - // Check, whether we are referring to a "self.X", which would create a field - scopeManager.isInRecord && scopeManager.isInFunction && ref.refersToReceiver -> { - // We need to temporarily jump into the scope of the current record to - // add the field. These are instance attributes - scopeManager.withScope(scopeManager.firstScopeIsInstanceOrNull()) { - newFieldDeclaration(ref.name) - } - } - scopeManager.isInRecord && scopeManager.isInFunction && ref is MemberExpression -> { - // If this is any other member expression, we are usually not interested in - // creating fields, except if this is a receiver - return null - } - scopeManager.isInRecord && !scopeManager.isInFunction -> { - // We end up here for fields declared directly in the class body. These are - // class attributes; more or less static fields. - newFieldDeclaration(scopeManager.currentNamespace.fqn(ref.name.localName)) - } - else -> { - null - } - } - - // If we didn't create any declaration up to this point and are still here, we need to - // create a (local) variable. We need to take scope modifications into account. - if (decl == null) { - decl = - if (targetScope != null) { - scopeManager.withScope(targetScope) { newVariableDeclaration(ref.name) } - } else { - newVariableDeclaration(ref.name) - } - } - - decl.code = ref.code - decl.location = ref.location - decl.isImplicit = true - - log.debug( - "Creating dynamic {} {} in {}", - if (decl is FieldDeclaration) { - "field" - } else { - "variable" - }, - decl.name, - decl.scope, - ) - - // Make sure we add the declaration at the correct place, i.e. with the scope we set at the - // creation time - scopeManager.withScope(decl.scope) { scopeManager.addDeclaration(decl) } - - return decl - } - - private val Reference.refersToReceiver: Boolean - get() { - return this is MemberExpression && - this.base.name == scopeManager.currentMethod?.receiver?.name - } - - /** - * Generates a new [VariableDeclaration] if [target] is a [Reference] and there is no existing - * declaration yet. - */ - private fun handleAssignmentToTarget( - assignExpression: AssignExpression, - target: Node, - setAccessValue: Boolean = false, - ) { - (target as? Reference)?.let { - if (setAccessValue) it.access = AccessValues.WRITE - val handled = handleWriteToReference(target) - if (handled is Declaration) { - // We cannot assign an initializer here because this will lead to duplicate - // DFG edges, but we need to propagate the type information from our value - // to the declaration. We therefore add the declaration to the observers of - // the value's type, so that it gets informed and can change its own type - // accordingly. - assignExpression - .findValue(target) - ?.registerTypeObserver(InitializerTypePropagation(handled)) - - // Add it to our assign expression, so that we can find it in the AST - assignExpression.declarations += handled - } - } - } - - private fun handleAssignExpression(assignExpression: AssignExpression) { - for (target in assignExpression.lhs) { - handleAssignmentToTarget(assignExpression, target, setAccessValue = false) - // If the lhs is an InitializerListExpression, we have to handle the individual elements - // in the initializers. - (target as? InitializerListExpression)?.let { - it.initializers.forEach { initializer -> - handleAssignmentToTarget(assignExpression, initializer, setAccessValue = true) - } - } - } - } - - // New variables can also be declared as `variable` in a [ForEachStatement] - private fun handleForEach(node: ForEachStatement) { - when (val forVar = node.variable) { - is Reference -> { - val handled = handleWriteToReference(forVar) - if (handled is Declaration) { - handled.let { node.addDeclaration(it) } - } - } - } - } - - override val language: Language<*> - get() = ctx.config.languages.firstOrNull { it is PythonLanguage } ?: UnknownLanguage -} From 2c4533064b75e95c55064558907c5ce3e150c7aa Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 29 Jan 2025 23:03:10 +0100 Subject: [PATCH 2/8] More tests pass --- .../cpg/graph/declarations/RecordDeclaration.kt | 1 + .../aisec/cpg/passes/SymbolResolver.kt | 7 +++++++ .../python/DynamicDeclarationHelper.kt | 17 +++++++++++------ .../cpg/frontends/python/SymbolResolverTest.kt | 6 ++++++ .../src/test/resources/python/fields.py | 2 +- 5 files changed, 26 insertions(+), 7 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt index 23440b37c2..ac1b9086da 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt @@ -169,6 +169,7 @@ open class RecordDeclaration : get() { val list = mutableListOf() + // list += statements list += fields list += methods list += constructors diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt index c75d01bccd..3fec8cbb78 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt @@ -319,6 +319,13 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { val base = current.base val language = current.language + // Before we resolve anything, we give languages with dynamic declarations a chance to + // declare any variables they need. This is most likely only needed for WRITE references, + // but we let the language decide that + if (language is HasDynamicDeclarations) { + with(language) { provideDeclaration(current) } + } + // We need to adjust certain types of the base in case of a "super" expression, and we // delegate this to the language. If that is successful, we can continue with regular // resolving. diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt index c86d96e0ec..0314a37c20 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt @@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.AssignExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.MemberExpression import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.InitializerTypePropagation +import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.UnknownType import de.fraunhofer.aisec.cpg.passes.Pass import de.fraunhofer.aisec.cpg.passes.Pass.Companion.log @@ -93,16 +94,20 @@ internal fun Pass<*>.handleWriteToReference(ref: Reference): VariableDeclaration newFieldDeclaration(ref.name) } } - scopeManager.isInRecord && scopeManager.isInFunction && ref is MemberExpression -> { - // If this is any other member expression, we are usually not interested in - // creating fields, except if this is a receiver - return null - } - scopeManager.isInRecord && !scopeManager.isInFunction -> { + scopeManager.isInRecord && !scopeManager.isInFunction && ref !is MemberExpression -> { // We end up here for fields declared directly in the class body. These are // class attributes; more or less static fields. newFieldDeclaration(scopeManager.currentNamespace.fqn(ref.name.localName)) } + ref is MemberExpression && ref.base.type is ObjectType -> { + // If this is a member expression for a known object type, we can create a field for + // the type + (ref.base.type as ObjectType).recordDeclaration?.let { + scopeManager.withScope(scopeManager.lookupScope(it)) { + newFieldDeclaration(ref.name) + } + } + } else -> { null } diff --git a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/SymbolResolverTest.kt b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/SymbolResolverTest.kt index ec7d2c01b7..aeb8b812d4 100644 --- a/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/SymbolResolverTest.kt +++ b/cpg-language-python/src/test/kotlin/de/fraunhofer/aisec/cpg/frontends/python/SymbolResolverTest.kt @@ -59,6 +59,12 @@ class SymbolResolverTest { aRefs.filterIsInstance().forEach { assertRefersTo(it, fieldA) } aRefs.filter { it !is MemberExpression }.forEach { assertRefersTo(it, globalA) } + // First assignment of copyA (a) should refer to global A, since the static class context is + // executed before the methods + val copyA = result.fields["copyA"] + assertNotNull(copyA) + assertRefersTo(copyA.firstAssignment, globalA) + // We should only have one reference to "os" -> the member expression "self.os" val osRefs = result.refs("os") assertEquals(1, osRefs.size) diff --git a/cpg-language-python/src/test/resources/python/fields.py b/cpg-language-python/src/test/resources/python/fields.py index 1ff4a16817..5e10453307 100644 --- a/cpg-language-python/src/test/resources/python/fields.py +++ b/cpg-language-python/src/test/resources/python/fields.py @@ -3,7 +3,7 @@ a = "Hello" class MyClass: - copyA = 1 + copyA = a def foo(self): self.a = 1 print(a) From f1e1a3d660d280f24ff97181e578d724d7035629 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 29 Jan 2025 23:11:45 +0100 Subject: [PATCH 3/8] Very crude hack to make imports within component work again --- .../aisec/cpg/frontends/python/PythonLanguage.kt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt index 8cf5241c7a..99e491a281 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt @@ -31,12 +31,15 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.autoType import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration +import de.fraunhofer.aisec.cpg.graph.imports import de.fraunhofer.aisec.cpg.graph.scopes.Symbol import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* +import de.fraunhofer.aisec.cpg.graph.translationUnit import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.Util.warnWithFileLocation import de.fraunhofer.aisec.cpg.passes.SymbolResolver +import de.fraunhofer.aisec.cpg.passes.updateImportedSymbols import kotlin.reflect.KClass import org.neo4j.ogm.annotation.Transient @@ -251,6 +254,13 @@ class PythonLanguage : } } + // Completely stupid, but needed, definitely needs a better solution, but we need to update + // the imports if we create variables here. Instead it would be very nice, if we could + // trigger an update from a namespace record to the import declarations that import it + // (which the import resolver could build). Then we can see whether we updated a symbol in a + // namespace and update its imports. + ref.translationUnit.imports.forEach { it.updateImportedSymbols() } + return null } From 6aa66395503a617344932a71caa92d472bd264b6 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 29 Jan 2025 23:33:32 +0100 Subject: [PATCH 4/8] Cross-component works, but with MAJOR hacks --- .../kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt | 4 ++++ .../kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt index cbb8a38637..6cd4c4b211 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt @@ -84,6 +84,10 @@ class TranslationResult( /** A free-for-use HashMap where passes can store whatever they want. */ val scratch: MutableMap = ConcurrentHashMap() + @Transient + @PopulatedByPass(ImportResolver::class) + var importDependencies = ImportDependencies(mutableListOf()) + /** * A free-for-use collection of unique nodes. Nodes stored here will be exported to Neo4j, too. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt index 3fec8cbb78..8759d299c7 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt @@ -81,7 +81,7 @@ import org.slf4j.LoggerFactory @DependsOn(TypeHierarchyResolver::class) @DependsOn(EvaluationOrderGraphPass::class) @DependsOn(ImportResolver::class) -open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { +open class SymbolResolver(ctx: TranslationContext) : TranslationResultPass(ctx) { /** Configuration for the [SymbolResolver]. */ class Configuration( @@ -122,8 +122,8 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { null } - override fun accept(component: Component) { - ctx.currentComponent = component + override fun accept(component: TranslationResult) { + // ctx.currentComponent = component walker = ScopedWalker(scopeManager) cacheTemplates(component) From a50c4f5e734dcaa4149a7a29462f886bcc052c19 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Wed, 29 Jan 2025 23:39:28 +0100 Subject: [PATCH 5/8] Python almost works - except non-local --- .../aisec/cpg/frontends/python/PythonLanguage.kt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt index 99e491a281..66f53c405a 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt @@ -241,11 +241,14 @@ class PythonLanguage : ref, setAccessValue = false, ) - } else if (ref.astParent is InitializerListExpression) { + } else if ( + ref.astParent is InitializerListExpression && + ref.astParent?.astParent is AssignExpression + ) { handleAssignmentToTarget( ref.astParent!!.astParent as AssignExpression, ref, - setAccessValue = false, + setAccessValue = true, ) } else if (ref.astParent is ForEachStatement) { val handled = handleWriteToReference(ref) From 6bb9a2c57d0f5fa1dd73465164380068d96cb6df Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Thu, 30 Jan 2025 00:07:34 +0100 Subject: [PATCH 6/8] Python tests work, broke everthing else --- .../kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt | 4 ---- .../aisec/cpg/graph/declarations/RecordDeclaration.kt | 1 - .../aisec/cpg/passes/EvaluationOrderGraphPass.kt | 8 +++++++- .../de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt | 6 +++--- .../aisec/cpg/frontends/python/StatementHandler.kt | 2 +- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt index 6cd4c4b211..cbb8a38637 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt @@ -84,10 +84,6 @@ class TranslationResult( /** A free-for-use HashMap where passes can store whatever they want. */ val scratch: MutableMap = ConcurrentHashMap() - @Transient - @PopulatedByPass(ImportResolver::class) - var importDependencies = ImportDependencies(mutableListOf()) - /** * A free-for-use collection of unique nodes. Nodes stored here will be exported to Neo4j, too. */ diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt index ac1b9086da..23440b37c2 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/declarations/RecordDeclaration.kt @@ -169,7 +169,6 @@ open class RecordDeclaration : get() { val list = mutableListOf() - // list += statements list += fields list += methods list += constructors diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt index 079e82f4fb..7dd54fd62b 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt @@ -40,6 +40,7 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.* import de.fraunhofer.aisec.cpg.graph.types.Type import de.fraunhofer.aisec.cpg.helpers.IdentitySet import de.fraunhofer.aisec.cpg.helpers.SubgraphWalker +import de.fraunhofer.aisec.cpg.helpers.identitySetOf import de.fraunhofer.aisec.cpg.tryCast import java.util.* import org.slf4j.LoggerFactory @@ -120,6 +121,8 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa */ protected val intermediateNodes = mutableListOf() + val alreadySeen = identitySetOf() + protected fun doNothing() { // Nothing to do for this node type } @@ -341,9 +344,12 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa * e.g. [LoopStatement]s or [BreakStatement]. */ protected fun handleEOG(node: Node?) { - if (node == null) { + if (node == null || alreadySeen.contains(node)) { return } + + alreadySeen.add(node) + intermediateNodes.add(node) when (node) { diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt index 8759d299c7..3fec8cbb78 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt @@ -81,7 +81,7 @@ import org.slf4j.LoggerFactory @DependsOn(TypeHierarchyResolver::class) @DependsOn(EvaluationOrderGraphPass::class) @DependsOn(ImportResolver::class) -open class SymbolResolver(ctx: TranslationContext) : TranslationResultPass(ctx) { +open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) { /** Configuration for the [SymbolResolver]. */ class Configuration( @@ -122,8 +122,8 @@ open class SymbolResolver(ctx: TranslationContext) : TranslationResultPass(ctx) null } - override fun accept(component: TranslationResult) { - // ctx.currentComponent = component + override fun accept(component: Component) { + ctx.currentComponent = component walker = ScopedWalker(scopeManager) cacheTemplates(component) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index ccd3062371..44c562d68e 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -1236,7 +1236,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : */ private fun wrapDeclarationToStatement(decl: Declaration): DeclarationStatement { val declStmt = newDeclarationStatement().codeAndLocationFrom(decl) - declStmt.addDeclaration(decl) + declStmt.declarations += decl return declStmt } From ca030ac538898810a0539a5e380e9f937afec0ea Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Fri, 31 Jan 2025 20:55:38 +0100 Subject: [PATCH 7/8] Fixed all tests --- .../cpg/passes/EvaluationOrderGraphPass.kt | 5 +- .../frontends/python/DeclarationHandler.kt | 331 ++++++++++++++++++ .../cpg/frontends/python/ExpressionHandler.kt | 2 +- .../python/PythonLanguageFrontend.kt | 9 +- .../cpg/frontends/python/StatementHandler.kt | 274 +-------------- 5 files changed, 350 insertions(+), 271 deletions(-) create mode 100644 cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DeclarationHandler.kt diff --git a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt index 7dd54fd62b..217d220609 100644 --- a/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt +++ b/cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/EvaluationOrderGraphPass.kt @@ -344,7 +344,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa * e.g. [LoopStatement]s or [BreakStatement]. */ protected fun handleEOG(node: Node?) { - if (node == null || alreadySeen.contains(node)) { + if (node == null) { return } @@ -543,8 +543,7 @@ open class EvaluationOrderGraphPass(ctx: TranslationContext) : TranslationUnitPa } else if (declaration is FunctionDeclaration) { // save the current EOG stack, because we can have a function declaration within an // existing function and the EOG handler for handling function declarations will - // reset the - // stack + // reset the stack val oldEOG = currentPredecessors.toMutableList() // analyze the defaults diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DeclarationHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DeclarationHandler.kt new file mode 100644 index 0000000000..d65d100a59 --- /dev/null +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DeclarationHandler.kt @@ -0,0 +1,331 @@ +/* + * Copyright (c) 2025, Fraunhofer AISEC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * $$$$$$\ $$$$$$$\ $$$$$$\ + * $$ __$$\ $$ __$$\ $$ __$$\ + * $$ / \__|$$ | $$ |$$ / \__| + * $$ | $$$$$$$ |$$ |$$$$\ + * $$ | $$ ____/ $$ |\_$$ | + * $$ | $$\ $$ | $$ | $$ | + * \$$$$$ |$$ | \$$$$$ | + * \______/ \__| \______/ + * + */ +package de.fraunhofer.aisec.cpg.frontends.python + +import de.fraunhofer.aisec.cpg.frontends.HasOperatorOverloading +import de.fraunhofer.aisec.cpg.frontends.isKnownOperatorName +import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage.Companion.MODIFIER_KEYWORD_ONLY_ARGUMENT +import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage.Companion.MODIFIER_POSITIONAL_ONLY_ARGUMENT +import de.fraunhofer.aisec.cpg.graph.* +import de.fraunhofer.aisec.cpg.graph.declarations.* +import de.fraunhofer.aisec.cpg.graph.scopes.FunctionScope +import de.fraunhofer.aisec.cpg.graph.scopes.RecordScope +import de.fraunhofer.aisec.cpg.graph.statements.expressions.Expression +import de.fraunhofer.aisec.cpg.graph.types.FunctionType +import de.fraunhofer.aisec.cpg.helpers.Util + +/** + * In Python, all declarations are statements. This class handles the parsing of the statements + * represented as [Declaration] nodes in our CPG. + * + * For declarations encountered directly on a namespace and classes, we directly invoke the + * [DeclarationHandler], for others, the [StatementHandler] will forward these statements to us. + */ +class DeclarationHandler(frontend: PythonLanguageFrontend) : + PythonHandler(::ProblemDeclaration, frontend) { + override fun handleNode(node: Python.AST.BaseStmt): Declaration { + return when (node) { + is Python.AST.FunctionDef -> handleFunctionDef(node) + is Python.AST.AsyncFunctionDef -> handleFunctionDef(node) + else -> { + return handleNotSupported(node, node.javaClass.simpleName) + } + } + } + + private fun handleNotSupported(node: Python.AST.BaseStmt, name: String): Declaration { + Util.errorWithFileLocation( + frontend, + node, + log, + "Parsing of type $name is not supported (yet)", + ) + + val cpgNode = this.configConstructor.get() + if (cpgNode is ProblemNode) { + cpgNode.problem = "Parsing of type $name is not supported (yet)" + } + + return cpgNode + } + + /** + * We have to consider multiple things when matching Python's FunctionDef to the CPG: + * - A [Python.AST.FunctionDef] could be one of + * - a [ConstructorDeclaration] if it appears in a record and its [name] is `__init__` + * - a [MethodeDeclaration] if it appears in a record, and it isn't a + * [ConstructorDeclaration] + * - a [FunctionDeclaration] if neither of the above apply + * + * In case of a [ConstructorDeclaration] or[MethodDeclaration]: the first argument is the + * `receiver` (most often called `self`). + */ + private fun handleFunctionDef(s: Python.AST.NormalOrAsyncFunctionDef): FunctionDeclaration { + var recordDeclaration = + (frontend.scopeManager.currentScope as? RecordScope)?.astNode as? RecordDeclaration + val language = language + val result = + if (recordDeclaration != null) { + if (s.name == "__init__") { + newConstructorDeclaration( + name = s.name, + recordDeclaration = recordDeclaration, + rawNode = s, + ) + } else if (language is HasOperatorOverloading && s.name.isKnownOperatorName) { + var decl = + newOperatorDeclaration( + name = s.name, + recordDeclaration = recordDeclaration, + operatorCode = language.operatorCodeFor(s.name) ?: "", + rawNode = s, + ) + if (decl.operatorCode == "") { + Util.warnWithFileLocation( + decl, + log, + "Could not find operator code for operator {}. This will most likely result in a failure", + s.name, + ) + } + decl + } else { + newMethodDeclaration( + name = s.name, + recordDeclaration = recordDeclaration, + isStatic = false, + rawNode = s, + ) + } + } else { + newFunctionDeclaration(name = s.name, rawNode = s) + } + frontend.scopeManager.enterScope(result) + + frontend.statementHandler.addAsyncWarning(s, result) + + // Handle decorators (which are translated into CPG "annotations") + result.annotations += frontend.statementHandler.handleAnnotations(s) + + // Handle return type and calculate function type + if (result is ConstructorDeclaration) { + // Return type of the constructor is always its record declaration type + result.returnTypes = listOf(recordDeclaration?.toType() ?: unknownType()) + } else { + result.returnTypes = listOf(frontend.typeOf(s.returns)) + } + result.type = FunctionType.computeType(result) + + handleArguments(s.args, result, recordDeclaration) + + if (s.body.isNotEmpty()) { + result.body = frontend.statementHandler.makeBlock(s.body, parentNode = s) + } + + frontend.scopeManager.leaveScope(result) + + // We want functions inside functions to be wrapped in a declaration statement, so we are + // not adding them to scope's AST + if (frontend.scopeManager.currentScope is FunctionScope) { + frontend.scopeManager.addDeclaration(result, addToAST = false) + } else { + // In any other cases, we add them to the "declarations", otherwise we will not + // correctly resolve them. + frontend.scopeManager.addDeclaration(result) + } + + return result + } + + /** Adds the arguments to [result] which might be located in a [recordDeclaration]. */ + private fun handleArguments( + args: Python.AST.arguments, + result: FunctionDeclaration, + recordDeclaration: RecordDeclaration?, + ) { + // We can merge posonlyargs and args because both are positional arguments. We do not + // enforce that posonlyargs can ONLY be used in a positional style, whereas args can be used + // both in positional and keyword style. + var positionalArguments = args.posonlyargs + args.args + + // Handle receiver if it exists and if it is not a static method + if ( + recordDeclaration != null && + result.annotations.none { it.name.localName == "staticmethod" } + ) { + handleReceiverArgument(positionalArguments, args, result, recordDeclaration) + // Skip the receiver argument for further processing + positionalArguments = positionalArguments.drop(1) + } + + // Handle remaining arguments + handlePositionalArguments(positionalArguments, args) + + args.vararg?.let { handleArgument(it, isPosOnly = false, isVariadic = true) } + args.kwarg?.let { handleArgument(it, isPosOnly = false, isVariadic = false) } + + handleKeywordOnlyArguments(args) + } + + /** + * This function creates a [newParameterDeclaration] for the argument, setting any modifiers + * (like positional-only or keyword-only) and [defaultValue] if applicable. + */ + internal fun handleArgument( + node: Python.AST.arg, + isPosOnly: Boolean = false, + isVariadic: Boolean = false, + isKwoOnly: Boolean = false, + defaultValue: Expression? = null, + ): ParameterDeclaration { + val type = frontend.typeOf(node.annotation) + val arg = + newParameterDeclaration( + name = node.arg, + type = type, + variadic = isVariadic, + rawNode = node, + ) + defaultValue?.let { arg.default = it } + if (isPosOnly) { + arg.modifiers += MODIFIER_POSITIONAL_ONLY_ARGUMENT + } + + if (isKwoOnly) { + arg.modifiers += MODIFIER_KEYWORD_ONLY_ARGUMENT + } + + frontend.scopeManager.addDeclaration(arg) + + return arg + } + + /** + * This method retrieves the first argument of the [positionalArguments], which is typically the + * receiver object. + * + * A receiver can also have a default value. However, this case is not handled and is therefore + * passed with a problem expression. + */ + private fun handleReceiverArgument( + positionalArguments: List, + args: Python.AST.arguments, + result: FunctionDeclaration, + recordDeclaration: RecordDeclaration, + ) { + // first argument is the receiver + val recvPythonNode = positionalArguments.firstOrNull() + if (recvPythonNode == null) { + result.additionalProblems += newProblemExpression("Expected a receiver", rawNode = args) + } else { + val tpe = recordDeclaration.toType() + val recvNode = + newVariableDeclaration( + name = recvPythonNode.arg, + type = tpe, + implicitInitializerAllowed = false, + rawNode = recvPythonNode, + ) + + // If the number of defaults equals the number of positional arguments, the receiver has + // a default value + if (args.defaults.size == positionalArguments.size) { + val defaultValue = + args.defaults.getOrNull(0)?.let { frontend.expressionHandler.handle(it) } + defaultValue?.let { + frontend.scopeManager.addDeclaration(recvNode) + result.additionalProblems += + newProblemExpression("Receiver with default value", rawNode = args) + } + } + + when (result) { + is ConstructorDeclaration, + is MethodDeclaration -> result.receiver = recvNode + else -> + result.additionalProblems += + newProblemExpression( + problem = + "Expected a constructor or method declaration. Got something else.", + rawNode = result, + ) + } + } + } + + /** + * This method extracts the [positionalArguments] including those that may have default values. + * + * In Python only the arguments with default values are stored in `args.defaults`. + * https://docs.python.org/3/library/ast.html#ast.arguments + * + * For example: `def my_func(a, b=1, c=2): pass` + * + * In this case, `args.defaults` contains only the defaults for `b` and `c`, while `args.args` + * includes all arguments (`a`, `b` and `c`). The number of arguments without defaults is + * determined by subtracting the size of `args.defaults` from the total number of arguments. + * This matches each default to its corresponding argument. + * + * From the Python docs: "If there are fewer defaults, they correspond to the last n arguments." + */ + private fun handlePositionalArguments( + positionalArguments: List, + args: Python.AST.arguments, + ) { + val nonDefaultArgsCount = positionalArguments.size - args.defaults.size + + for (idx in positionalArguments.indices) { + val arg = positionalArguments[idx] + val defaultIndex = idx - nonDefaultArgsCount + val defaultValue = + if (defaultIndex >= 0) { + args.defaults.getOrNull(defaultIndex)?.let { + frontend.expressionHandler.handle(it) + } + } else { + null + } + handleArgument(arg, isPosOnly = arg in args.posonlyargs, defaultValue = defaultValue) + } + } + + /** + * This method extracts the keyword-only arguments from [args] and maps them to the + * corresponding function parameters. + */ + private fun handleKeywordOnlyArguments(args: Python.AST.arguments) { + for (idx in args.kwonlyargs.indices) { + val arg = args.kwonlyargs[idx] + val default = args.kw_defaults.getOrNull(idx) + handleArgument( + arg, + isPosOnly = false, + isKwoOnly = true, + defaultValue = default?.let { frontend.expressionHandler.handle(it) }, + ) + } + } +} diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt index 3da8338b34..d90de20aea 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/ExpressionHandler.kt @@ -568,7 +568,7 @@ class ExpressionHandler(frontend: PythonLanguageFrontend) : val function = newFunctionDeclaration(name = "", rawNode = node) frontend.scopeManager.enterScope(function) for (arg in node.args.args) { - this.frontend.statementHandler.handleArgument(arg) + this.frontend.declarationHandler.handleArgument(arg) } function.body = handle(node.body) frontend.scopeManager.leaveScope(function) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt index 9046509a31..961c66d42f 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguageFrontend.kt @@ -66,8 +66,7 @@ class PythonLanguageFrontend(language: Language, ctx: Tr private val tokenTypeIndex = 0 private val jep = JepSingleton // configure Jep - // val declarationHandler = DeclarationHandler(this) - // val specificationHandler = SpecificationHandler(this) + internal val declarationHandler = DeclarationHandler(this) internal var statementHandler = StatementHandler(this) internal var expressionHandler = ExpressionHandler(this) @@ -313,7 +312,11 @@ class PythonLanguageFrontend(language: Language, ctx: Tr if (lastNamespace != null) { for (stmt in pythonASTModule.body) { - lastNamespace.statements += statementHandler.handle(stmt) + if (stmt is Python.AST.FunctionDef || stmt is Python.AST.AsyncFunctionDef) { + declarationHandler.handleNode(stmt) + } else { + lastNamespace.statements += statementHandler.handle(stmt) + } } } diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt index 44c562d68e..a0e1b32d0b 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/StatementHandler.kt @@ -25,11 +25,7 @@ */ package de.fraunhofer.aisec.cpg.frontends.python -import de.fraunhofer.aisec.cpg.frontends.HasOperatorOverloading -import de.fraunhofer.aisec.cpg.frontends.isKnownOperatorName import de.fraunhofer.aisec.cpg.frontends.python.Python.AST.IsAsync -import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage.Companion.MODIFIER_KEYWORD_ONLY_ARGUMENT -import de.fraunhofer.aisec.cpg.frontends.python.PythonLanguage.Companion.MODIFIER_POSITIONAL_ONLY_ARGUMENT import de.fraunhofer.aisec.cpg.graph.* import de.fraunhofer.aisec.cpg.graph.Annotation import de.fraunhofer.aisec.cpg.graph.declarations.* @@ -45,7 +41,6 @@ import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement import de.fraunhofer.aisec.cpg.graph.statements.Statement import de.fraunhofer.aisec.cpg.graph.statements.TryStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* -import de.fraunhofer.aisec.cpg.graph.types.FunctionType import de.fraunhofer.aisec.cpg.helpers.Util import kotlin.collections.plusAssign @@ -55,8 +50,6 @@ class StatementHandler(frontend: PythonLanguageFrontend) : override fun handleNode(node: Python.AST.BaseStmt): Statement { return when (node) { is Python.AST.ClassDef -> handleClassDef(node) - is Python.AST.FunctionDef -> handleFunctionDef(node) - is Python.AST.AsyncFunctionDef -> handleFunctionDef(node) is Python.AST.Pass -> return newEmptyStatement(rawNode = node) is Python.AST.ImportFrom -> handleImportFrom(node) is Python.AST.Assign -> handleAssign(node) @@ -85,6 +78,10 @@ class StatementHandler(frontend: PythonLanguageFrontend) : problem = "The statement of class ${node.javaClass} is not supported yet", rawNode = node, ) + is Python.AST.AsyncFunctionDef -> + wrapDeclarationToStatement(frontend.declarationHandler.handleNode(node)) + is Python.AST.FunctionDef -> + wrapDeclarationToStatement(frontend.declarationHandler.handleNode(node)) } } @@ -825,11 +822,9 @@ class StatementHandler(frontend: PythonLanguageFrontend) : for (s in stmt.body) { when (s) { - is Python.AST.FunctionDef -> { - val stmt = handleFunctionDef(s, cls) - // We need to manually set the astParent because we are not assigning it to our - // statements and therefore are not triggering our automagic parent setter - stmt.astParent = cls + is Python.AST.FunctionDef, + is Python.AST.AsyncFunctionDef -> { + frontend.declarationHandler.handleNode(s) } else -> cls.statements += handleNode(s) } @@ -841,88 +836,6 @@ class StatementHandler(frontend: PythonLanguageFrontend) : return wrapDeclarationToStatement(cls) } - /** - * We have to consider multiple things when matching Python's FunctionDef to the CPG: - * - A [Python.AST.FunctionDef] is a [Statement] from Python's point of view. The CPG sees it as - * a declaration -> we have to wrap the result in a [DeclarationStatement]. - * - A [Python.AST.FunctionDef] could be one of - * - a [ConstructorDeclaration] if it appears in a record and its [name] is `__init__` - * - a [MethodeDeclaration] if it appears in a record, and it isn't a - * [ConstructorDeclaration] - * - a [FunctionDeclaration] if neither of the above apply - * - * In case of a [ConstructorDeclaration] or[MethodDeclaration]: the first argument is the - * `receiver` (most often called `self`). - */ - private fun handleFunctionDef( - s: Python.AST.NormalOrAsyncFunctionDef, - recordDeclaration: RecordDeclaration? = null, - ): DeclarationStatement { - val language = language - val result = - if (recordDeclaration != null) { - if (s.name == "__init__") { - newConstructorDeclaration( - name = s.name, - recordDeclaration = recordDeclaration, - rawNode = s, - ) - } else if (language is HasOperatorOverloading && s.name.isKnownOperatorName) { - var decl = - newOperatorDeclaration( - name = s.name, - recordDeclaration = recordDeclaration, - operatorCode = language.operatorCodeFor(s.name) ?: "", - rawNode = s, - ) - if (decl.operatorCode == "") { - Util.warnWithFileLocation( - decl, - log, - "Could not find operator code for operator {}. This will most likely result in a failure", - s.name, - ) - } - decl - } else { - newMethodDeclaration( - name = s.name, - recordDeclaration = recordDeclaration, - isStatic = false, - rawNode = s, - ) - } - } else { - newFunctionDeclaration(name = s.name, rawNode = s) - } - frontend.scopeManager.enterScope(result) - - addAsyncWarning(s, result) - - // Handle decorators (which are translated into CPG "annotations") - result.annotations += handleAnnotations(s) - - // Handle return type and calculate function type - if (result is ConstructorDeclaration) { - // Return type of the constructor is always its record declaration type - result.returnTypes = listOf(recordDeclaration?.toType() ?: unknownType()) - } else { - result.returnTypes = listOf(frontend.typeOf(s.returns)) - } - result.type = FunctionType.computeType(result) - - handleArguments(s.args, result, recordDeclaration) - - if (s.body.isNotEmpty()) { - result.body = makeBlock(s.body, parentNode = s) - } - - frontend.scopeManager.leaveScope(result) - frontend.scopeManager.addDeclaration(result) - - return wrapDeclarationToStatement(result) - } - /** * Translates a Python [`Global`](https://docs.python.org/3/library/ast.html#ast.Global) into a * [LookupScopeStatement]. @@ -959,143 +872,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : ) } - /** Adds the arguments to [result] which might be located in a [recordDeclaration]. */ - private fun handleArguments( - args: Python.AST.arguments, - result: FunctionDeclaration, - recordDeclaration: RecordDeclaration?, - ) { - // We can merge posonlyargs and args because both are positional arguments. We do not - // enforce that posonlyargs can ONLY be used in a positional style, whereas args can be used - // both in positional and keyword style. - var positionalArguments = args.posonlyargs + args.args - - // Handle receiver if it exists and if it is not a static method - if ( - recordDeclaration != null && - result.annotations.none { it.name.localName == "staticmethod" } - ) { - handleReceiverArgument(positionalArguments, args, result, recordDeclaration) - // Skip the receiver argument for further processing - positionalArguments = positionalArguments.drop(1) - } - - // Handle remaining arguments - handlePositionalArguments(positionalArguments, args) - - args.vararg?.let { handleArgument(it, isPosOnly = false, isVariadic = true) } - args.kwarg?.let { handleArgument(it, isPosOnly = false, isVariadic = false) } - - handleKeywordOnlyArguments(args) - } - - /** - * This method retrieves the first argument of the [positionalArguments], which is typically the - * receiver object. - * - * A receiver can also have a default value. However, this case is not handled and is therefore - * passed with a problem expression. - */ - private fun handleReceiverArgument( - positionalArguments: List, - args: Python.AST.arguments, - result: FunctionDeclaration, - recordDeclaration: RecordDeclaration, - ) { - // first argument is the receiver - val recvPythonNode = positionalArguments.firstOrNull() - if (recvPythonNode == null) { - result.additionalProblems += newProblemExpression("Expected a receiver", rawNode = args) - } else { - val tpe = recordDeclaration.toType() - val recvNode = - newVariableDeclaration( - name = recvPythonNode.arg, - type = tpe, - implicitInitializerAllowed = false, - rawNode = recvPythonNode, - ) - - // If the number of defaults equals the number of positional arguments, the receiver has - // a default value - if (args.defaults.size == positionalArguments.size) { - val defaultValue = - args.defaults.getOrNull(0)?.let { frontend.expressionHandler.handle(it) } - defaultValue?.let { - frontend.scopeManager.addDeclaration(recvNode) - result.additionalProblems += - newProblemExpression("Receiver with default value", rawNode = args) - } - } - - when (result) { - is ConstructorDeclaration, - is MethodDeclaration -> result.receiver = recvNode - else -> - result.additionalProblems += - newProblemExpression( - problem = - "Expected a constructor or method declaration. Got something else.", - rawNode = result, - ) - } - } - } - - /** - * This method extracts the [positionalArguments] including those that may have default values. - * - * In Python only the arguments with default values are stored in `args.defaults`. - * https://docs.python.org/3/library/ast.html#ast.arguments - * - * For example: `def my_func(a, b=1, c=2): pass` - * - * In this case, `args.defaults` contains only the defaults for `b` and `c`, while `args.args` - * includes all arguments (`a`, `b` and `c`). The number of arguments without defaults is - * determined by subtracting the size of `args.defaults` from the total number of arguments. - * This matches each default to its corresponding argument. - * - * From the Python docs: "If there are fewer defaults, they correspond to the last n arguments." - */ - private fun handlePositionalArguments( - positionalArguments: List, - args: Python.AST.arguments, - ) { - val nonDefaultArgsCount = positionalArguments.size - args.defaults.size - - for (idx in positionalArguments.indices) { - val arg = positionalArguments[idx] - val defaultIndex = idx - nonDefaultArgsCount - val defaultValue = - if (defaultIndex >= 0) { - args.defaults.getOrNull(defaultIndex)?.let { - frontend.expressionHandler.handle(it) - } - } else { - null - } - handleArgument(arg, isPosOnly = arg in args.posonlyargs, defaultValue = defaultValue) - } - } - - /** - * This method extracts the keyword-only arguments from [args] and maps them to the - * corresponding function parameters. - */ - private fun handleKeywordOnlyArguments(args: Python.AST.arguments) { - for (idx in args.kwonlyargs.indices) { - val arg = args.kwonlyargs[idx] - val default = args.kw_defaults.getOrNull(idx) - handleArgument( - arg, - isPosOnly = false, - isKwoOnly = true, - defaultValue = default?.let { frontend.expressionHandler.handle(it) }, - ) - } - } - - private fun handleAnnotations( + internal fun handleAnnotations( node: Python.AST.NormalOrAsyncFunctionDef ): Collection { return handleDeclaratorList(node, node.decorator_list) @@ -1173,7 +950,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * itself does not have a code/location, we need to employ [codeAndLocationFromChildren] on the * [parentNode]. */ - private fun makeBlock( + internal fun makeBlock( stmts: List, parentNode: Python.AST.WithLocation, ): Block { @@ -1197,37 +974,6 @@ class StatementHandler(frontend: PythonLanguageFrontend) : return result } - /** - * This function creates a [newParameterDeclaration] for the argument, setting any modifiers - * (like positional-only or keyword-only) and [defaultValue] if applicable. - */ - internal fun handleArgument( - node: Python.AST.arg, - isPosOnly: Boolean = false, - isVariadic: Boolean = false, - isKwoOnly: Boolean = false, - defaultValue: Expression? = null, - ) { - val type = frontend.typeOf(node.annotation) - val arg = - newParameterDeclaration( - name = node.arg, - type = type, - variadic = isVariadic, - rawNode = node, - ) - defaultValue?.let { arg.default = it } - if (isPosOnly) { - arg.modifiers += MODIFIER_POSITIONAL_ONLY_ARGUMENT - } - - if (isKwoOnly) { - arg.modifiers += MODIFIER_KEYWORD_ONLY_ARGUMENT - } - - frontend.scopeManager.addDeclaration(arg) - } - /** * Wrap a declaration in a [DeclarationStatement] * @@ -1244,7 +990,7 @@ class StatementHandler(frontend: PythonLanguageFrontend) : * Checks whether [mightBeAsync] implements the [IsAsync] interface and adds a warning to the * corresponding [parentNode] stored in [Node.additionalProblems]. */ - private fun addAsyncWarning(mightBeAsync: Python.AST.AsyncOrNot, parentNode: Node) { + internal fun addAsyncWarning(mightBeAsync: Python.AST.AsyncOrNot, parentNode: Node) { if (mightBeAsync is IsAsync) { parentNode.additionalProblems += newProblemDeclaration( From 882088929b88a8996354c47e33793e82ec933979 Mon Sep 17 00:00:00 2001 From: Christian Banse Date: Thu, 6 Feb 2025 21:31:01 +0100 Subject: [PATCH 8/8] Added debug with file location --- .../cpg/frontends/python/DynamicDeclarationHelper.kt | 7 +++++-- .../aisec/cpg/frontends/python/PythonLanguage.kt | 10 ---------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt index 0314a37c20..94b98ff28e 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/DynamicDeclarationHelper.kt @@ -37,8 +37,9 @@ import de.fraunhofer.aisec.cpg.graph.statements.expressions.Reference import de.fraunhofer.aisec.cpg.graph.types.InitializerTypePropagation import de.fraunhofer.aisec.cpg.graph.types.ObjectType import de.fraunhofer.aisec.cpg.graph.types.UnknownType +import de.fraunhofer.aisec.cpg.helpers.Util.debugWithFileLocation import de.fraunhofer.aisec.cpg.passes.Pass -import de.fraunhofer.aisec.cpg.passes.Pass.Companion.log +import de.fraunhofer.aisec.cpg.passes.SymbolResolver /** * This function will create a new dynamic [VariableDeclaration] if there is a write access to the @@ -129,7 +130,9 @@ internal fun Pass<*>.handleWriteToReference(ref: Reference): VariableDeclaration decl.isImplicit = true decl.language = ref.language - log.debug( + debugWithFileLocation( + decl, + SymbolResolver.Companion.LOGGER, "Creating dynamic {} {} in {}", if (decl is FieldDeclaration) { "field" diff --git a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt index 66f53c405a..050d687258 100644 --- a/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt +++ b/cpg-language-python/src/main/kotlin/de/fraunhofer/aisec/cpg/frontends/python/PythonLanguage.kt @@ -31,15 +31,12 @@ import de.fraunhofer.aisec.cpg.graph.Node import de.fraunhofer.aisec.cpg.graph.autoType import de.fraunhofer.aisec.cpg.graph.declarations.Declaration import de.fraunhofer.aisec.cpg.graph.declarations.ParameterDeclaration -import de.fraunhofer.aisec.cpg.graph.imports import de.fraunhofer.aisec.cpg.graph.scopes.Symbol import de.fraunhofer.aisec.cpg.graph.statements.ForEachStatement import de.fraunhofer.aisec.cpg.graph.statements.expressions.* -import de.fraunhofer.aisec.cpg.graph.translationUnit import de.fraunhofer.aisec.cpg.graph.types.* import de.fraunhofer.aisec.cpg.helpers.Util.warnWithFileLocation import de.fraunhofer.aisec.cpg.passes.SymbolResolver -import de.fraunhofer.aisec.cpg.passes.updateImportedSymbols import kotlin.reflect.KClass import org.neo4j.ogm.annotation.Transient @@ -257,13 +254,6 @@ class PythonLanguage : } } - // Completely stupid, but needed, definitely needs a better solution, but we need to update - // the imports if we create variables here. Instead it would be very nice, if we could - // trigger an update from a namespace record to the import declarations that import it - // (which the import resolver could build). Then we can see whether we updated a symbol in a - // namespace and update its imports. - ref.translationUnit.imports.forEach { it.updateImportedSymbols() } - return null }