Execute ComponentPass passes according to the dependencies between components (#2010)

Author: oxisto

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/TranslationResult.kt

@@ -34,11 +34,15 @@ import de.fraunhofer.aisec.cpg.graph.edges.ast.astEdgesOf
 import de.fraunhofer.aisec.cpg.graph.edges.unwrapping
 import de.fraunhofer.aisec.cpg.helpers.MeasurementHolder
 import de.fraunhofer.aisec.cpg.helpers.StatisticsHolder
+import de.fraunhofer.aisec.cpg.passes.ImportDependencies
+import de.fraunhofer.aisec.cpg.passes.ImportResolver
 import de.fraunhofer.aisec.cpg.passes.Pass
 import de.fraunhofer.aisec.cpg.persistence.DoNotPersist
+import de.fraunhofer.aisec.cpg.processing.strategy.Strategy
 import java.util.*
 import java.util.concurrent.ConcurrentHashMap
 import org.neo4j.ogm.annotation.Relationship
+import org.neo4j.ogm.annotation.Transient
 
 /**
  * The global (intermediate) result of the translation. A [LanguageFrontend] will initially populate
@@ -62,6 +66,14 @@ class TranslationResult(
      */
     val components by unwrapping(TranslationResult::componentEdges)
 
+    /**
+     * The import dependencies of [Component] nodes of this translation result. The preferred way to
+     * access this is via [Strategy.COMPONENTS_LEAST_IMPORTS].
+     */
+    @Transient
+    @PopulatedByPass(ImportResolver::class)
+    var componentDependencies: ImportDependencies<Component>? = null
+
     /**
      * Scratch storage that can be used by passes to store additional information in this result.
      * Callers must ensure that keys are unique. It is recommended to prefix them with the class

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Component.kt

@@ -35,6 +35,7 @@ import de.fraunhofer.aisec.cpg.graph.edges.unwrapping
 import de.fraunhofer.aisec.cpg.passes.ImportDependencies
 import de.fraunhofer.aisec.cpg.passes.ImportResolver
 import de.fraunhofer.aisec.cpg.persistence.DoNotPersist
+import de.fraunhofer.aisec.cpg.processing.strategy.Strategy
 import java.io.File
 import org.neo4j.ogm.annotation.Relationship
 import org.neo4j.ogm.annotation.Transient
@@ -52,9 +53,13 @@ open class Component : Node() {
     /** All translation units belonging to this application. */
     val translationUnits by unwrapping(Component::translationUnitEdges)
 
+    /**
+     * The import dependencies of [TranslationUnitDeclaration] nodes of this component. The
+     * preferred way to access this is via [Strategy.TRANSLATION_UNITS_LEAST_IMPORTS].
+     */
     @Transient
     @PopulatedByPass(ImportResolver::class)
-    var importDependencies = ImportDependencies(translationUnits)
+    var translationUnitDependencies: ImportDependencies<TranslationUnitDeclaration>? = null
 
     @Synchronized
     fun addTranslationUnit(tu: TranslationUnitDeclaration) {

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/graph/Extensions.kt

@@ -1360,6 +1360,12 @@ val Node.translationUnit: TranslationUnitDeclaration?
         return firstParentOrNull<TranslationUnitDeclaration>()
     }
 
+/** Returns the [Component] where this node is located in. */
+val Node.component: Component?
+    get() {
+        return firstParentOrNull<Component>()
+    }
+
 /**
  * This helper function be used to find out if a particular expression (usually a [CallExpression]
  * or a [Reference]) is imported through a [ImportDeclaration].

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolver.kt

@@ -42,6 +42,8 @@ import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLast
 import de.fraunhofer.aisec.cpg.passes.configuration.ExecuteLate
 import de.fraunhofer.aisec.cpg.passes.configuration.RequiredFrontend
 import de.fraunhofer.aisec.cpg.passes.configuration.RequiresLanguageTrait
+import de.fraunhofer.aisec.cpg.processing.strategy.Strategy
+import de.fraunhofer.aisec.cpg.processing.strategy.Strategy.TRANSLATION_UNITS_LEAST_IMPORTS
 import java.util.concurrent.CompletableFuture
 import java.util.function.Consumer
 import kotlin.reflect.KClass
@@ -272,14 +274,19 @@ fun executePass(
             consumeTargets(
                 (prototype as ComponentPass)::class,
                 ctx,
-                result.components,
+                // Execute them in the "sorted" order (if available)
+                (Strategy::COMPONENTS_LEAST_IMPORTS)(result).asSequence().toList(),
                 executedFrontends,
             )
         is TranslationUnitPass ->
             consumeTargets(
                 (prototype as TranslationUnitPass)::class,
                 ctx,
-                result.components.flatMap { it.translationUnits },
+                // Execute them in the "sorted" order (if available)
+                (Strategy::COMPONENTS_LEAST_IMPORTS)(result)
+                    .asSequence()
+                    .flatMap { (Strategy::TRANSLATION_UNITS_LEAST_IMPORTS)(it).asSequence() }
+                    .toList(),
                 executedFrontends,
             )
         is EOGStarterPass -> {

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/Pass.kt

@@ -139,12 +139,12 @@ open class SymbolResolver(ctx: TranslationContext) : ComponentPass(ctx) {
 
         // Resolve symbols in our translation units in the order depending on their import
         // dependencies
-        component.importDependencies.sortedTranslationUnits.forEach {
-            log.debug("Resolving symbols of translation unit {}", it.name)
+        for (tu in (Strategy::TRANSLATION_UNITS_LEAST_IMPORTS)(component)) {
+            log.debug("Resolving symbols of translation unit {}", tu.name)
 
             // Gather all resolution EOG starters; and make sure they really do not have a
             // predecessor, otherwise we might analyze a node multiple times
-            val nodes = it.allEOGStarters.filter { it.prevEOGEdges.isEmpty() }
+            val nodes = tu.allEOGStarters.filter { it.prevEOGEdges.isEmpty() }
 
             for (node in nodes) {
                 walker.iterate(node)

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/passes/SymbolResolver.kt

@@ -25,11 +25,19 @@
  */
 package de.fraunhofer.aisec.cpg.processing.strategy
 
+import de.fraunhofer.aisec.cpg.TranslationResult
+import de.fraunhofer.aisec.cpg.graph.Component
 import de.fraunhofer.aisec.cpg.graph.Node
+import de.fraunhofer.aisec.cpg.graph.declarations.TranslationUnitDeclaration
 import java.util.*
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
 
 /** Strategies (iterators) for traversing graphs to be used by visitors. */
 object Strategy {
+
+    val log: Logger = LoggerFactory.getLogger(Strategy::class.java)
+
     /**
      * Do not traverse any nodes.
      *
@@ -55,6 +63,26 @@ object Strategy {
         return x.nextEOGEdges.filter { !it.unreachable }.map { it.end }.iterator()
     }
 
+    fun COMPONENTS_LEAST_IMPORTS(x: TranslationResult): Iterator<Component> {
+        return x.componentDependencies?.sorted?.iterator()
+            ?: x.components.iterator().also {
+                log.warn(
+                    "Strategy for components with least import dependencies was requested, but no import dependency information is available."
+                )
+                log.warn("Please make sure that the ImportResolver pass was run successfully.")
+            }
+    }
+
+    fun TRANSLATION_UNITS_LEAST_IMPORTS(x: Component): Iterator<TranslationUnitDeclaration> {
+        return x.translationUnitDependencies?.sorted?.iterator()
+            ?: x.translationUnits.iterator().also {
+                log.warn(
+                    "Strategy for translation units with least import dependencies was requested, but no import dependency information is available."
+                )
+                log.warn("Please make sure that the ImportResolver pass was run successfully.")
+            }
+    }
+
     /**
      * Traverse Evaluation Order Graph in backward direction.
      *

cpg-core/src/main/kotlin/de/fraunhofer/aisec/cpg/processing/strategy/Strategy.kt

@@ -176,7 +176,7 @@ class FluentTest {
         var app = result.components.firstOrNull()
         assertNotNull(app)
 
-        ImportResolver(result.finalCtx).accept(app)
+        ImportResolver(result.finalCtx).accept(result)
         EvaluationOrderGraphPass(result.finalCtx).accept(app.translationUnits.first())
         SymbolResolver(result.finalCtx).accept(app)
 

cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/graph/FluentTest.kt

@@ -113,25 +113,25 @@ class ImportResolverTest {
 
         // a has 0 dependencies
         var a =
-            app.importDependencies.entries
-                .filter { it.key.name.toString() == "file.a" }
-                .firstOrNull()
+            app.translationUnitDependencies?.entries?.firstOrNull {
+                it.key.name.toString() == "file.a"
+            }
         assertNotNull(a)
         assertEquals(0, a.value.size)
 
         // c has 0 dependencies
         var c =
-            app.importDependencies.entries
-                .filter { it.key.name.toString() == "file.c" }
-                .firstOrNull()
+            app.translationUnitDependencies?.entries?.firstOrNull {
+                it.key.name.toString() == "file.c"
+            }
         assertNotNull(c)
         assertEquals(0, c.value.size)
 
         // b has two dependencies (a, c)
         var b =
-            app.importDependencies.entries
-                .filter { it.key.name.toString() == "file.b" }
-                .firstOrNull()
+            app.translationUnitDependencies?.entries?.firstOrNull {
+                it.key.name.toString() == "file.b"
+            }
         assertNotNull(b)
         assertEquals(2, b.value.size)
         assertEquals(setOf(a.key, c.key), b.value)

cpg-core/src/test/kotlin/de/fraunhofer/aisec/cpg/passes/ImportResolverTest.kt

@@ -26,12 +26,19 @@
 package de.fraunhofer.aisec.cpg.processing
 
 import de.fraunhofer.aisec.cpg.GraphExamples
+import de.fraunhofer.aisec.cpg.ScopeManager
+import de.fraunhofer.aisec.cpg.TranslationConfiguration
+import de.fraunhofer.aisec.cpg.TranslationContext
+import de.fraunhofer.aisec.cpg.TranslationManager
+import de.fraunhofer.aisec.cpg.TranslationResult
+import de.fraunhofer.aisec.cpg.TypeManager
 import de.fraunhofer.aisec.cpg.frontends.TranslationException
 import de.fraunhofer.aisec.cpg.graph.*
 import de.fraunhofer.aisec.cpg.graph.Node
 import de.fraunhofer.aisec.cpg.graph.declarations.*
 import de.fraunhofer.aisec.cpg.graph.statements
 import de.fraunhofer.aisec.cpg.graph.statements.ReturnStatement
+import de.fraunhofer.aisec.cpg.passes.ImportDependencies
 import de.fraunhofer.aisec.cpg.processing.strategy.Strategy
 import de.fraunhofer.aisec.cpg.test.*
 import java.util.concurrent.ExecutionException
@@ -130,6 +137,66 @@ class VisitorTest : BaseTest() {
         assertEquals(2, returnStatements.size)
     }
 
+    @Test
+    fun testFallbackComponentLeastImported() {
+        val component1 = Component().also { it.name = Name("component1") }
+        val component2 = Component().also { it.name = Name("component2") }
+
+        val tr =
+            TranslationResult(
+                translationManager = TranslationManager.builder().build(),
+                finalCtx =
+                    TranslationContext(
+                        config = TranslationConfiguration.builder().build(),
+                        scopeManager = ScopeManager(),
+                        typeManager = TypeManager(),
+                    ),
+            )
+        tr.components += component1
+        tr.components += component2
+
+        // will trigger fallback as we have no dependency information
+        val fallback = Strategy.COMPONENTS_LEAST_IMPORTS(tr).asSequence().toList()
+        assertEquals(listOf(component1, component2), fallback)
+
+        tr.componentDependencies =
+            ImportDependencies<Component>(tr.components).also {
+                it.add(component1, component2)
+                it
+            }
+
+        // will use sorted
+        val sorted = Strategy.COMPONENTS_LEAST_IMPORTS(tr).asSequence().toList()
+        assertEquals(listOf(component2, component1), sorted)
+        assertEquals(tr.componentDependencies?.sorted, sorted)
+    }
+
+    @Test
+    fun testFallbackTULeastImported() {
+        val component = Component()
+
+        val tr1 = TranslationUnitDeclaration().also { it.name = Name("tr1") }
+        val tr2 = TranslationUnitDeclaration().also { it.name = Name("tr2") }
+
+        component.translationUnits += tr1
+        component.translationUnits += tr2
+
+        // will trigger fallback as we have no dependency information
+        val fallback = Strategy.TRANSLATION_UNITS_LEAST_IMPORTS(component).asSequence().toList()
+        assertEquals(listOf(tr1, tr2), fallback)
+
+        component.translationUnitDependencies =
+            ImportDependencies<TranslationUnitDeclaration>(component.translationUnits).also {
+                it.add(tr1, tr2)
+                it
+            }
+
+        // will use sorted
+        val sorted = Strategy.TRANSLATION_UNITS_LEAST_IMPORTS(component).asSequence().toList()
+        assertEquals(listOf(tr2, tr1), sorted)
+        assertEquals(component.translationUnitDependencies?.sorted, sorted)
+    }
+
     companion object {
         private var recordDeclaration: RecordDeclaration? = null