Yarn - Refactor the process of building impact graph

build.gradle

@@ -54,7 +54,11 @@ def idePluginsCommonVersion = '2.3.1'
 dependencies {
     implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.15.2'
     implementation group: 'org.jfrog.buildinfo', name: 'build-info-extractor', version: buildInfoVersion
-    implementation group: 'com.jfrog.ide', name: 'ide-plugins-common', version: idePluginsCommonVersion
+    implementation('com.jfrog.ide:ide-plugins-common') {
+        version {
+            branch = 'yarn1-hoisting'
+        }
+    }
     implementation group: 'org.jfrog.buildinfo', name: 'build-info-client', version: buildInfoVersion
     implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.15.2'
     implementation group: 'org.jfrog.buildinfo', name: 'build-info-api', version: buildInfoVersion

settings.gradle

@@ -7,4 +7,10 @@ pluginManagement {
     }
 }
 
+sourceControl {
+    gitRepository("https://github.com/noyshabtay/ide-plugins-common.git") {
+        producesModule("com.jfrog.ide:ide-plugins-common")
+    }
+}
+
 rootProject.name = 'jfrog-idea-plugin'

src/main/java/com/jfrog/ide/idea/scan/ScannerBase.java

@@ -61,8 +61,6 @@
  * Created by romang on 4/26/17.
  */
 public abstract class ScannerBase {
-    public static final int IMPACT_PATHS_LIMIT = 50;
-
     private final ServerConfig serverConfig;
     private final ComponentPrefix prefix;
     @Getter
@@ -185,7 +183,7 @@ private void scanAndUpdate(ProgressIndicator indicator) {
      * @param vulnerableDependencies a map of component IDs and the DependencyNode object matching each of them.
      * @param depTree                the project's dependency tree to walk through.
      */
-    private List<FileTreeNode> walkDepTree(Map<String, DependencyNode> vulnerableDependencies, DepTree depTree) {
+    protected List<FileTreeNode> walkDepTree(Map<String, DependencyNode> vulnerableDependencies, DepTree depTree) throws IOException {
         Map<String, DescriptorFileTreeNode> descriptorNodes = new HashMap<>();
         visitDepTreeNode(vulnerableDependencies, depTree, Collections.singletonList(depTree.getRootId()), descriptorNodes, new ArrayList<>(), new HashMap<>());
         return new CopyOnWriteArrayList<>(descriptorNodes.values());
@@ -255,13 +253,12 @@ private void visitDepTreeNode(Map<String, DependencyNode> vulnerableDependencies
         }
     }
 
-    private void addImpactPathToDependencyNode(DependencyNode dependencyNode, List<String> path) {
+    protected void addImpactPathToDependencyNode(DependencyNode dependencyNode, List<String> path) {
         if (dependencyNode.getImpactTree() == null) {
             dependencyNode.setImpactTree(new ImpactTree(new ImpactTreeNode(path.get(0))));
         }
         ImpactTree impactTree = dependencyNode.getImpactTree();
-        impactTree.incImpactPathsCount();
-        if (impactTree.getImpactPathsCount() > IMPACT_PATHS_LIMIT) {
+        if (impactTree.getImpactPathsCount() > ImpactTree.IMPACT_PATHS_LIMIT) {
             return;
         }
         ImpactTreeNode parentImpactTreeNode = impactTree.getRoot();
@@ -272,6 +269,10 @@ private void addImpactPathToDependencyNode(DependencyNode dependencyNode, List<S
             if (currImpactTreeNode == null) {
                 currImpactTreeNode = new ImpactTreeNode(currPathNode);
                 parentImpactTreeNode.getChildren().add(currImpactTreeNode);
+                if (pathNodeIndex == path.size() - 1) {
+                    // If a new leaf was added, thus a new impact path was added (impact paths don't collide after they split)
+                    impactTree.incImpactPathsCount();
+                }
             }
             parentImpactTreeNode = currImpactTreeNode;
         }

src/main/java/com/jfrog/ide/idea/scan/SingleDescriptorScanner.java

@@ -1,12 +1,18 @@
 package com.jfrog.ide.idea.scan;
 
 import com.intellij.openapi.project.Project;
+import com.jfrog.ide.common.deptree.DepTree;
+import com.jfrog.ide.common.nodes.DependencyNode;
+import com.jfrog.ide.common.nodes.FileTreeNode;
 import com.jfrog.ide.common.scan.ComponentPrefix;
 import com.jfrog.ide.common.scan.ScanLogic;
 import com.jfrog.ide.idea.ui.ComponentsTree;
 import com.jfrog.ide.idea.ui.menus.filtermanager.ConsistentFilterManager;
 import org.jetbrains.annotations.NotNull;
 
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ExecutorService;
 
 public abstract class SingleDescriptorScanner extends ScannerBase {
@@ -39,4 +45,8 @@ public abstract class SingleDescriptorScanner extends ScannerBase {
                             ScanLogic scanLogic) {
         this(project, basePath, prefix, executor, "", scanLogic);
     }
+
+    protected List<FileTreeNode> walkDepTree(Map<String, DependencyNode> vulnerableDependencies, DepTree depTree) throws IOException {
+        return super.walkDepTree(vulnerableDependencies, depTree);
+    }
 }

src/main/java/com/jfrog/ide/idea/scan/YarnScanner.java

@@ -7,6 +7,9 @@
 import com.intellij.psi.PsiManager;
 import com.intellij.util.EnvironmentUtil;
 import com.jfrog.ide.common.deptree.DepTree;
+import com.jfrog.ide.common.nodes.DependencyNode;
+import com.jfrog.ide.common.nodes.DescriptorFileTreeNode;
+import com.jfrog.ide.common.nodes.FileTreeNode;
 import com.jfrog.ide.common.scan.ComponentPrefix;
 import com.jfrog.ide.common.scan.ScanLogic;
 import com.jfrog.ide.common.yarn.YarnTreeBuilder;
@@ -15,9 +18,13 @@
 import com.jfrog.ide.idea.scan.data.PackageManagerType;
 import com.jfrog.ide.idea.ui.ComponentsTree;
 import com.jfrog.ide.idea.ui.menus.filtermanager.ConsistentFilterManager;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.io.IOException;
 import java.nio.file.Paths;
+import java.util.*;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutorService;
 
 /**
@@ -26,6 +33,7 @@
 public class YarnScanner extends SingleDescriptorScanner {
 
     private final YarnTreeBuilder yarnTreeBuilder;
+
     /**
      * @param project   currently opened IntelliJ project. We'll use this project to retrieve project based services
      *                  like {@link ConsistentFilterManager} and {@link ComponentsTree}.
@@ -63,5 +71,65 @@ protected AbstractInspection getInspectionTool() {
     protected PackageManagerType getPackageManagerType() {
         return PackageManagerType.YARN;
     }
+
+    /**
+     * Builds a map of package name to versions out of a set of <package-name>:<version> Strings.
+     *
+     * @param packages - A set of packages in the format of 'package-name:version'.
+     * @return - A map of package name to a set of versions.
+     */
+    Map<String, Set<String>> getPackageNameToVersionsMap(Set<String> packages) {
+        Map<String, Set<String>> packageNameToVersions = new HashMap<>();
+        for (String fullNamePackage : CollectionUtils.emptyIfNull(packages)) {
+            String[] packageSplit = StringUtils.split(fullNamePackage, ":");
+            String packageName = packageSplit[0];
+            String packageVersion = packageSplit[1];
+            packageNameToVersions.putIfAbsent(packageName, new HashSet<>());
+            packageNameToVersions.get(packageName).add(packageVersion);
+        }
+        return packageNameToVersions;
+    }
+
+    /**
+     * Builds the impact graph for each given vulnerable dependencies.
+     * The impact graph is built by running 'yarn why <package>' command, making it different from other package managers.
+     *
+     * @param vulnerableDependencies - The vulnerable dependencies to build the impact graph for.
+     *                               The key is the package name and version, and the value is the dependency node.
+     * @param depTree                - The whole dependency tree (not just vulnerable dependencies) that was generated earlier.
+     * @return - The impact graph attached to package.json DescriptorFileTreeNode
+     */
+    @Override
+    protected List<FileTreeNode> walkDepTree(Map<String, DependencyNode> vulnerableDependencies, DepTree depTree) throws IOException {
+        DescriptorFileTreeNode descriptorNode = new DescriptorFileTreeNode(depTree.getRootNode().getDescriptorFilePath());
+        // Build a map of package name to versions, to avoid running 'yarn why' multiple times for the same package.
+        Map<String, Set<String>> packageNameToVersions = this.getPackageNameToVersionsMap(vulnerableDependencies.keySet());
+
+        for (Map.Entry<String, Set<String>> entry : packageNameToVersions.entrySet()) {
+            String packageName = entry.getKey();
+            Set<String> packageVersions = entry.getValue();
+            // find the impact paths for each package for all its vulnerable versions
+            Map<String, List<List<String>>> packageVersionsImpactPaths = yarnTreeBuilder.findDependencyImpactPaths(getLog(), depTree.getRootId(), packageName, packageVersions);
+            for (Map.Entry<String, List<List<String>>> aPackageVersionImpactPaths : packageVersionsImpactPaths.entrySet()) {
+                String packageFullName = aPackageVersionImpactPaths.getKey();
+                List<List<String>> impactPaths = aPackageVersionImpactPaths.getValue();
+                DependencyNode dependencyNode = vulnerableDependencies.get(packageFullName);
+                boolean indirect = true;
+                // build the impact graph for each vulnerable dependency out of its impact paths
+                for (List<String> impactPath : impactPaths) {
+                    this.addImpactPathToDependencyNode(dependencyNode, impactPath);
+                    if (impactPath.size() == 2) {
+                        indirect = false; // If the impact path is of length 2 (root -> dependency), this dependency is direct
+                    }
+                }
+                dependencyNode.setIndirect(indirect);
+                descriptorNode.addDependency(dependencyNode);
+            }
+        }
+
+        // Return a list of one element - the descriptor node for package.json
+        // COW list is used to avoid ConcurrentModificationException in SourceCodeScannerManager
+        return new CopyOnWriteArrayList<>(Collections.singletonList(descriptorNode));
+    }
 }
 

src/main/java/com/jfrog/ide/idea/ui/webview/WebviewObjectConverter.java

@@ -3,7 +3,6 @@
 import com.jfrog.ide.common.nodes.*;
 import com.jfrog.ide.common.nodes.subentities.*;
 import com.jfrog.ide.common.scan.ComponentPrefix;
-import com.jfrog.ide.idea.scan.ScannerBase;
 import com.jfrog.ide.idea.ui.webview.model.Cve;
 import com.jfrog.ide.idea.ui.webview.model.Evidence;
 import com.jfrog.ide.idea.ui.webview.model.License;
@@ -143,7 +142,7 @@ public static DependencyPage convertLicenseToDepPage(LicenseViolationNode licens
     }
 
     private static ImpactGraph convertImpactGraph(ImpactTree impactTree) {
-        return new ImpactGraph(convertImpactGraphNode(impactTree.getRoot()), impactTree.getImpactPathsCount(), ScannerBase.IMPACT_PATHS_LIMIT);
+        return new ImpactGraph(convertImpactGraphNode(impactTree.getRoot()), impactTree.getImpactPathsCount(), ImpactTree.IMPACT_PATHS_LIMIT);
     }
 
     private static ImpactGraphNode convertImpactGraphNode(ImpactTreeNode impactTreeNode) {

src/test/java/com/jfrog/ide/idea/scan/YarnScannerTest.java

@@ -0,0 +1,113 @@
+package com.jfrog.ide.idea.scan;
+
+import com.intellij.testFramework.HeavyPlatformTestCase;
+import com.intellij.util.ConcurrencyUtil;
+import com.jfrog.ide.common.deptree.DepTree;
+import com.jfrog.ide.common.nodes.DependencyNode;
+import com.jfrog.ide.common.nodes.FileTreeNode;
+import com.jfrog.ide.common.scan.GraphScanLogic;
+import org.apache.commons.io.FileUtils;
+import org.jetbrains.plugins.gradle.service.project.open.GradleProjectImportUtil;
+import org.jfrog.build.api.util.NullLog;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+
+public class YarnScannerTest extends HeavyPlatformTestCase {
+    private static final Path YARN_ROOT = Paths.get(".").toAbsolutePath().normalize().resolve(Paths.get("src", "test", "resources", "yarn"));
+    private ExecutorService executorService;
+    private String tempProjectDir;
+
+    @Override
+    protected void setUp() throws Exception {
+        super.setUp();
+        tempProjectDir = createTempProjectDir("exampleYarnPackage");
+        executorService = ConcurrencyUtil.newSameThreadExecutorService();
+    }
+
+    @Override
+    protected void tearDown() throws Exception {
+        try {
+            executorService.shutdown();
+        } finally {
+            // Ensure that tearDown gets executed even if an exception is thrown
+            super.tearDown();
+        }
+    }
+
+    private String createTempProjectDir(String projectName) throws IOException {
+        // Using a virtual directory allows each test to have its own isolated workspace, preventing interference between tests.
+        String tempProjectDir = getTempDir().createVirtualDir(projectName).toNioPath().toString();
+        FileUtils.copyDirectory(YARN_ROOT.resolve(projectName).toFile(), FileUtils.getFile(tempProjectDir));
+        return tempProjectDir;
+    }
+
+    public void testGetPackageNameToVersionsMap() {
+        YarnScanner yarnScanner = new YarnScanner(getProject(), tempProjectDir, executorService, new GraphScanLogic(new NullLog()));
+        Set<String> packages = new HashSet<>(Arrays.asList("package1:1.0.123", "package2:2.0.9", "package1:1.0.123", "package2:2.1.7"));
+        Map<String, Set<String>> packageNameToVersions = yarnScanner.getPackageNameToVersionsMap(packages);
+
+        assertEquals(2, packageNameToVersions.size());
+        assertTrue(packageNameToVersions.containsKey("package1"));
+        assertTrue(packageNameToVersions.containsKey("package2"));
+        assertFalse(packageNameToVersions.containsKey("package3"));
+
+        assertEquals(new HashSet<>(List.of("1.0.123")), packageNameToVersions.get("package1"));
+        assertEquals(new HashSet<>(Arrays.asList("2.0.9", "2.1.7")), packageNameToVersions.get("package2"));
+    }
+
+    public void testWalkDepTree() throws IOException {
+        GradleProjectImportUtil.linkAndRefreshGradleProject(tempProjectDir, getProject());
+        YarnScanner yarnScanner = new YarnScanner(getProject(), tempProjectDir, executorService, new GraphScanLogic(new NullLog()));
+
+        // Sanity check - make sure the dependency tree is generated.
+        DepTree depTree = yarnScanner.buildTree();
+        assertNotNull(depTree);
+        assertEquals(depTree.getRootId(), "example-yarn-package:1.0.0");
+
+        // Test the walkDepTree method:
+        // Init params
+        String[] vulnerableDependencies = {"lodash:4.16.2", "tough-cookie:2.3.1"};
+        Map<String, DependencyNode> vulnerableDependenciesMap = new HashMap<>();
+        for (String vulnerableDependency : vulnerableDependencies) {
+            DependencyNode dependencyNode = new DependencyNode();
+            dependencyNode.componentId(vulnerableDependency);
+            // Insert a dummy node and remove it to init the children vector.
+            dependencyNode.insert(new DependencyNode(), 0);
+            dependencyNode.remove(0);
+            vulnerableDependenciesMap.put(vulnerableDependency, dependencyNode);
+        }
+        // Run the method
+        List<FileTreeNode> fileTreeNodes = yarnScanner.walkDepTree(vulnerableDependenciesMap, depTree);
+        // Check there is only one file tree node and it's package.json
+        assertEquals(1, fileTreeNodes.size());
+        assertEquals("package.json", fileTreeNodes.get(0).getTitle());
+
+        // Check the impact graphs is attached to the package.json node.
+        FileTreeNode packageJsonNode = fileTreeNodes.get(0);
+        assertEquals(2, packageJsonNode.getChildren().size());
+
+        // Check the impact graph correctness.
+        DependencyNode toughCookieDep = (DependencyNode) packageJsonNode.getChildren().get(0);
+        DependencyNode lodashDep = (DependencyNode) packageJsonNode.getChildren().get(1);
+        if (lodashDep.getComponentId().equals("tough-cookie:2.3.1")) {
+            DependencyNode temp = lodashDep;
+            lodashDep = toughCookieDep;
+            toughCookieDep = temp;
+        }
+
+        // lodash:4.16.2 tests
+        assertEquals("lodash:4.16.2", lodashDep.getComponentId());
+        assertFalse(lodashDep.isIndirect());
+        assertEquals(7, lodashDep.getImpactTree().getImpactPathsCount());
+
+        // tough-cookie:2.3.1 tests
+        assertEquals("tough-cookie:2.3.1", toughCookieDep.getComponentId());
+        assertTrue(toughCookieDep.isIndirect());
+        assertEquals(2, toughCookieDep.getImpactTree().getImpactPathsCount());
+
+    }
+}

src/test/resources/yarn/exampleYarnPackage/.gitignore

@@ -0,0 +1 @@
+node_modules/

src/test/resources/yarn/exampleYarnPackage/LICENSE

@@ -0,0 +1,26 @@
+BSD License
+
+For Yarn software
+
+Copyright (c) 2016-present, Yarn Contributors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+ * Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.