CM-40909 - Add lock file restore for Golang, NPM and Nuget (#266)

Author: naftalicy

cycode/cli/files_collector/sca/base_restore_dependencies.py

@@ -13,19 +13,27 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str:
     return join_paths(get_file_dir(path), generated_file_name)
 
 
-def execute_command(
-    command: List[str],
+def execute_commands(
+    commands: List[List[str]],
     file_name: str,
     command_timeout: int,
     dependencies_file_name: Optional[str] = None,
     working_directory: Optional[str] = None,
 ) -> Optional[str]:
     try:
-        dependencies = shell(command=command, timeout=command_timeout, working_directory=working_directory)
-        # Write stdout output to the file if output_file_path is provided
+        all_dependencies = []
+
+        # Run all commands and collect outputs
+        for command in commands:
+            dependencies = shell(command=command, timeout=command_timeout, working_directory=working_directory)
+            all_dependencies.append(dependencies)  # Collect each command's output
+
+        dependencies = '\n'.join(all_dependencies)
+
+        # Write all collected outputs to the file if dependencies_file_name is provided
         if dependencies_file_name:
-            with open(dependencies_file_name, 'w') as output_file:
-                output_file.write(dependencies)
+            with open(dependencies_file_name, 'w') as output_file:  # Open once in 'w' mode to start fresh
+                output_file.writelines(dependencies)
     except Exception as e:
         logger.debug('Failed to restore dependencies via shell command, %s', {'filename': file_name}, exc_info=e)
         return None
@@ -62,8 +70,8 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]:
             restore_file_content = get_file_content(restore_file_path)
         else:
             output_file_path = restore_file_path if self.create_output_file_manually else None
-            execute_command(
-                self.get_command(manifest_file_path),
+            execute_commands(
+                self.get_commands(manifest_file_path),
                 manifest_file_path,
                 self.command_timeout,
                 output_file_path,
@@ -85,7 +93,7 @@ def is_project(self, document: Document) -> bool:
         pass
 
     @abstractmethod
-    def get_command(self, manifest_file_path: str) -> List[str]:
+    def get_commands(self, manifest_file_path: str) -> List[List[str]]:
         pass
 
     @abstractmethod

cycode/cli/files_collector/sca/go/restore_go_dependencies.py

@@ -1,31 +1,51 @@
+import logging
 import os
-from typing import List
+from typing import List, Optional
 
 import click
 
 from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies
 from cycode.cli.models import Document
 
-GO_PROJECT_FILE_EXTENSIONS = ['.mod']
-GO_RESTORE_FILE_NAME = 'go.sum'
+GO_PROJECT_FILE_EXTENSIONS = ['.mod', '.sum']
+GO_RESTORE_FILE_NAME = 'go.mod.graph'
 BUILD_GO_FILE_NAME = 'go.mod'
+BUILD_GO_LOCK_FILE_NAME = 'go.sum'
 
 
 class RestoreGoDependencies(BaseRestoreDependencies):
     def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None:
         super().__init__(context, is_git_diff, command_timeout, create_output_file_manually=True)
 
+    def try_restore_dependencies(self, document: Document) -> Optional[Document]:
+        manifest_exists = os.path.isfile(self.get_working_directory(document) + os.sep + BUILD_GO_FILE_NAME)
+        lock_exists = os.path.isfile(self.get_working_directory(document) + os.sep + BUILD_GO_LOCK_FILE_NAME)
+
+        if not manifest_exists or not lock_exists:
+            logging.info('No manifest go.mod file found' if not manifest_exists else 'No manifest go.sum file found')
+
+        manifest_files_exists = manifest_exists & lock_exists
+
+        if not manifest_files_exists:
+            return None
+
+        return super().try_restore_dependencies(document)
+
     def is_project(self, document: Document) -> bool:
         return any(document.path.endswith(ext) for ext in GO_PROJECT_FILE_EXTENSIONS)
 
-    def get_command(self, manifest_file_path: str) -> List[str]:
-        return ['cd', self.prepare_tree_file_path_for_command(manifest_file_path), '&&', 'go', 'list', '-m', '-json']
+    def get_commands(self, manifest_file_path: str) -> List[List[str]]:
+        return [
+            ['go', 'list', '-m', '-json', 'all'],
+            ['echo', '------------------------------------------------------'],
+            ['go', 'mod', 'graph'],
+        ]
 
     def get_lock_file_name(self) -> str:
         return GO_RESTORE_FILE_NAME
 
     def verify_restore_file_already_exist(self, restore_file_path: str) -> bool:
         return os.path.isfile(restore_file_path)
 
-    def prepare_tree_file_path_for_command(self, manifest_file_path: str) -> str:
-        return manifest_file_path.replace(os.sep + BUILD_GO_FILE_NAME, '')
+    def get_working_directory(self, document: Document) -> Optional[str]:
+        return os.path.dirname(document.absolute_path)

cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py

@@ -18,8 +18,8 @@ def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: i
     def is_project(self, document: Document) -> bool:
         return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME)
 
-    def get_command(self, manifest_file_path: str) -> List[str]:
-        return ['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']
+    def get_commands(self, manifest_file_path: str) -> List[List[str]]:
+        return [['gradle', 'dependencies', '-b', manifest_file_path, '-q', '--console', 'plain']]
 
     def get_lock_file_name(self) -> str:
         return BUILD_GRADLE_DEP_TREE_FILE_NAME

cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py

@@ -7,7 +7,7 @@
 from cycode.cli.files_collector.sca.base_restore_dependencies import (
     BaseRestoreDependencies,
     build_dep_tree_path,
-    execute_command,
+    execute_commands,
 )
 from cycode.cli.models import Document
 from cycode.cli.utils.path_utils import get_file_content, get_file_dir, join_paths
@@ -24,8 +24,8 @@ def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: i
     def is_project(self, document: Document) -> bool:
         return path.basename(document.path).split('/')[-1] == BUILD_MAVEN_FILE_NAME
 
-    def get_command(self, manifest_file_path: str) -> List[str]:
-        return ['mvn', 'org.cyclonedx:cyclonedx-maven-plugin:2.7.4:makeAggregateBom', '-f', manifest_file_path]
+    def get_commands(self, manifest_file_path: str) -> List[List[str]]:
+        return [['mvn', 'org.cyclonedx:cyclonedx-maven-plugin:2.7.4:makeAggregateBom', '-f', manifest_file_path]]
 
     def get_lock_file_name(self) -> str:
         return join_paths('target', MAVEN_CYCLONE_DEP_TREE_FILE_NAME)
@@ -52,7 +52,7 @@ def restore_from_secondary_command(
     ) -> Optional[Document]:
         # TODO(MarshalX): does it even work? Ignored restore_dependencies_document arg
         secondary_restore_command = create_secondary_restore_command(manifest_file_path)
-        backup_restore_content = execute_command(secondary_restore_command, manifest_file_path, self.command_timeout)
+        backup_restore_content = execute_commands(secondary_restore_command, manifest_file_path, self.command_timeout)
         restore_dependencies_document = Document(
             build_dep_tree_path(document.path, MAVEN_DEP_TREE_FILE_NAME), backup_restore_content, self.is_git_diff
         )

cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py

@@ -18,15 +18,17 @@ def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: i
     def is_project(self, document: Document) -> bool:
         return any(document.path.endswith(ext) for ext in NPM_PROJECT_FILE_EXTENSIONS)
 
-    def get_command(self, manifest_file_path: str) -> List[str]:
+    def get_commands(self, manifest_file_path: str) -> List[List[str]]:
         return [
-            'npm',
-            'install',
-            '--prefix',
-            self.prepare_manifest_file_path_for_command(manifest_file_path),
-            '--package-lock-only',
-            '--ignore-scripts',
-            '--no-audit',
+            [
+                'npm',
+                'install',
+                '--prefix',
+                self.prepare_manifest_file_path_for_command(manifest_file_path),
+                '--package-lock-only',
+                '--ignore-scripts',
+                '--no-audit',
+            ]
         ]
 
     def get_lock_file_name(self) -> str:

cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py

@@ -17,8 +17,8 @@ def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: i
     def is_project(self, document: Document) -> bool:
         return any(document.path.endswith(ext) for ext in NUGET_PROJECT_FILE_EXTENSIONS)
 
-    def get_command(self, manifest_file_path: str) -> List[str]:
-        return ['dotnet', 'restore', manifest_file_path, '--use-lock-file', '--verbosity', 'quiet']
+    def get_commands(self, manifest_file_path: str) -> List[List[str]]:
+        return [['dotnet', 'restore', manifest_file_path, '--use-lock-file', '--verbosity', 'quiet']]
 
     def get_lock_file_name(self) -> str:
         return NUGET_LOCK_FILE_NAME

cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py

@@ -12,8 +12,8 @@ class RestoreSbtDependencies(BaseRestoreDependencies):
     def is_project(self, document: Document) -> bool:
         return any(document.path.endswith(ext) for ext in SBT_PROJECT_FILE_EXTENSIONS)
 
-    def get_command(self, manifest_file_path: str) -> List[str]:
-        return ['sbt', 'dependencyLockWrite', '--verbose']
+    def get_commands(self, manifest_file_path: str) -> List[List[str]]:
+        return [['sbt', 'dependencyLockWrite', '--verbose']]
 
     def get_lock_file_name(self) -> str:
         return SBT_LOCK_FILE_NAME

cycode/cli/files_collector/sca/sca_code_scanner.py

@@ -5,8 +5,11 @@
 
 from cycode.cli import consts
 from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies
+from cycode.cli.files_collector.sca.go.restore_go_dependencies import RestoreGoDependencies
 from cycode.cli.files_collector.sca.maven.restore_gradle_dependencies import RestoreGradleDependencies
 from cycode.cli.files_collector.sca.maven.restore_maven_dependencies import RestoreMavenDependencies
+from cycode.cli.files_collector.sca.npm.restore_npm_dependencies import RestoreNpmDependencies
+from cycode.cli.files_collector.sca.nuget.restore_nuget_dependencies import RestoreNugetDependencies
 from cycode.cli.files_collector.sca.sbt.restore_sbt_dependencies import RestoreSbtDependencies
 from cycode.cli.models import Document
 from cycode.cli.utils.git_proxy import git_proxy
@@ -132,6 +135,9 @@ def restore_handlers(context: click.Context, is_git_diff: bool) -> List[BaseRest
         RestoreGradleDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT),
         RestoreMavenDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT),
         RestoreSbtDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT),
+        RestoreGoDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT),
+        RestoreNugetDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT),
+        RestoreNpmDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT),
     ]