Merge pull request #174 from microsoft/revert-ps-module-extraction

PS: Revert extraction of code found via `PSModulePath`

Author: MathiasVP

misc/scripts/prepare-db-upgrade.sh

@@ -33,7 +33,7 @@ EOF
 # default for prev_hash: the main branch of the remote for 'github/codeql'.
 # This works out as a dynamic lookup of the hash of the file in the main branch
 # of the repo.
-prev_hash=$(git remote -v | grep 'github/codeql\.git (fetch)$' | cut -f1)/main
+prev_hash=$(git remote -v | grep 'microsoft/codeql\.git (fetch)$' | cut -f1)/main
 
 while [ $# -gt 0 ]; do
   case "$1" in

powershell/codeql-extractor.yml

@@ -10,10 +10,4 @@ file_types:
     display_name: powershellscripts
     extensions:
       - .ps1
-      - .psd1
-options:
-  skip_psmodulepath_files:
-    title: Skip PSModulePath files.
-    description: Whether to avoid extracting source files in paths specified by the PSModulePath environment variable.
-    type: string
-    pattern: "^(false|true)$"
+      - .psd1

powershell/downgrades/802d5b9f407fb0dac894df1c0b4584f2215e1512/semmlecode.powershell.dbscheme

@@ -41,6 +41,10 @@ containerparent(
   unique int child: @container ref
 );
 
+is_in_psmodule_path(
+  int file: @file ref
+);
+
 /* Comments */
 comment_entity(
   unique int id: @comment_entity,
@@ -82,7 +86,7 @@ parent(
 // https://learn.microsoft.com/en-us/dotnet/api/system.management.automation.language.commandbaseast?view=powershellsdk-7.3.0
 @command_base = @command | @command_expression;
 // https://learn.microsoft.com/en-us/dotnet/api/system.management.automation.language.chainableast?view=powershellsdk-7.3.0
-@chainable = @pipeline | @pipeline_chain;
+@chainable = @command_base | @pipeline | @pipeline_chain;
 //https://learn.microsoft.com/en-us/dotnet/api/system.management.automation.language.pipelinebaseast?view=powershellsdk-7.3.0
 @pipeline_base = @chainable | @error_statement | @assignment_statement;
 // https://learn.microsoft.com/en-us/dotnet/api/system.management.automation.language.statementast?view=powershellsdk-7.3.0

powershell/downgrades/802d5b9f407fb0dac894df1c0b4584f2215e1512/upgrade.properties

@@ -1,2 +1,2 @@
-description: Remove Pipelines with one element.
+description: Remove psmodule file extraction
 compatibility: partial

powershell/extractor/Semmle.Extraction.PowerShell.Standalone/Options.cs

@@ -1,4 +1,3 @@
-using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
@@ -25,9 +24,6 @@ public override bool HandleFlag(string key, bool value)
                 case "dry-run":
                     SkipExtraction = value;
                     return true;
-                case "skip-psmodulepath-files":
-                    SkipPSModulePathFiles = value;
-                    return true;
                 default:
                     return base.HandleFlag(key, value);
             }
@@ -78,7 +74,7 @@ public override void InvalidArgument(string argument)
         /// <summary>
         /// List of extensions to include.
         /// </summary>
-        public IList<string> Extensions { get; } = new List<string>() { ".ps1", ".psd1" };
+        public IList<string> Extensions { get; } = new List<string>() { ".ps1" };
 
         /// <summary>
         /// Files/patterns to exclude.
@@ -121,24 +117,6 @@ private static FileInfo[] GetDefaultFiles()
             }
         }
 
-        /// <summary>
-        /// Returns true if the extractor should skip files in the PSModulePath because the
-        /// environment variable CODEQL_EXTRACTOR_POWERSHELL_OPTION_SKIP_PSMODULEPATH_FILES
-        /// is set to a truthy value.
-        /// </summary>
-        private static bool GetDefaultSkipPSModulePathFiles()
-        {
-            var skip = System.Environment.GetEnvironmentVariable(
-                "CODEQL_EXTRACTOR_POWERSHELL_OPTION_SKIP_PSMODULEPATH_FILES"
-            );
-            bool b = skip != null && skip.ToLower() != "false";
-            if (b)
-            {
-                System.Console.WriteLine("Skipping files in PSModulePath");
-            }
-            return b;
-        }
-
         /// <summary>
         /// The directory or file containing the source code;
         /// </summary>
@@ -149,12 +127,6 @@ private static bool GetDefaultSkipPSModulePathFiles()
         /// </summary>
         public bool SkipExtraction { get; private set; } = false;
 
-        /// <summary>
-        /// Whether to extract files in the paths found in the `PSModulePath`
-        /// environment variable.
-        /// </summary>
-        public bool SkipPSModulePathFiles { get; private set; } = GetDefaultSkipPSModulePathFiles();
-
         /// <summary>
         /// Whether errors were encountered parsing the arguments.
         /// </summary>
@@ -186,18 +158,13 @@ public static void ShowHelp(System.IO.TextWriter output)
                 "PowerShell# standalone extractor\n\nExtracts PowerShell scripts in the current directory.\n"
             );
             output.WriteLine("Additional options:\n");
-            output.WriteLine("    <path>                       Use the provided path instead.");
-            output.WriteLine(
-                "    --exclude:xxx                Exclude a file or directory (can be specified multiple times)"
-            );
-            output.WriteLine("    --dry-run                    Stop before extraction");
-            output.WriteLine(
-                "    --threads:nnn                Specify number of threads (default=CPU cores)"
-            );
-            output.WriteLine("    --verbose                    Produce more output");
+            output.WriteLine("    <path>           Use the provided path instead.");
             output.WriteLine(
-                "    --skip-psmodulepath-files    Avoid extracting source files in paths specified by the PSModulePath environment variable."
+                "    --exclude:xxx    Exclude a file or directory (can be specified multiple times)"
             );
+            output.WriteLine("    --dry-run        Stop before extraction");
+            output.WriteLine("    --threads:nnn    Specify number of threads (default=CPU cores)");
+            output.WriteLine("    --verbose        Produce more output");
         }
 
         private Options() { }

powershell/extractor/Semmle.Extraction.PowerShell.Standalone/Program.cs

@@ -45,8 +45,7 @@ public static int Main(string[] args)
 
             output.Log(Severity.Info, "Running PowerShell standalone extractor");
             var sourceFiles = options
-                .Files.Concat(GetPSModuleFiles(options))
-                .Where(d =>
+                .Files.Where(d =>
                     options.Extensions.Contains(
                         d.Extension,
                         StringComparer.InvariantCultureIgnoreCase
@@ -88,30 +87,6 @@ public static int Main(string[] args)
             return 0;
         }
 
-        private static String[] GetPSModulePaths()
-        {
-            return Environment.GetEnvironmentVariable("PSModulePath")?.Split(Path.PathSeparator)
-                ?? Array.Empty<string>();
-        }
-
-        private static IEnumerable<FileInfo> GetPSModuleFiles(Options options)
-        {
-            if(options.SkipPSModulePathFiles)
-            {
-                return Array.Empty<FileInfo>();
-            }
-            
-            return GetPSModulePaths()
-                .Where(d => Directory.Exists(d))
-                .SelectMany(d =>
-                {
-                    var di = new DirectoryInfo(d);
-                    return di.Exists
-                        ? di.GetFiles("*.*", SearchOption.AllDirectories)
-                        : new FileInfo[] { new(d) };
-                });
-        }
-
         private class ExtractionProgress : IProgressMonitor
         {
             public ExtractionProgress(ILogger output)

powershell/extractor/Semmle.Extraction.PowerShell/Entities/Base/File.cs

@@ -2,7 +2,6 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
 
 namespace Semmle.Extraction.PowerShell.Entities
 {
@@ -15,18 +14,6 @@ protected File(PowerShellContext cx, string path)
         {
         }
 
-        private static string[] GetPSModulePaths()
-        {
-            return Environment.GetEnvironmentVariable("PSModulePath")?.Split(Path.PathSeparator)
-                ?? Array.Empty<string>();
-        }
-
-        private bool PathIsInPSModulePath()
-        {
-            // Check if f's path is inside one of the paths in $Env:PSModulePath
-            return GetPSModulePaths().Any(originalPath.StartsWith);
-        }
-
         public override void Populate(TextWriter trapFile)
         {
             trapFile.files(this, TransformedPath.Value);
@@ -36,11 +23,6 @@ public override void Populate(TextWriter trapFile)
                 trapFile.containerparent(Extraction.Entities.Folder.Create(PowerShellContext, dir), this);
             }
 
-            if(PathIsInPSModulePath())
-            {
-                trapFile.is_in_psmodule_path(this);
-            }
-
             try
             {
                 System.Text.Encoding encoding;

powershell/extractor/Semmle.Extraction/Tuples.cs

@@ -32,9 +32,5 @@ public static void locations_default(this System.IO.TextWriter trapFile, SourceL
         {
             trapFile.WriteTuple("locations_default", label, file, startLine, startCol, endLine, endCol);
         }
-
-        public static void is_in_psmodule_path(this System.IO.TextWriter trapFile, Entities.File file) {
-            trapFile.WriteTuple("is_in_psmodule_path", file);
-        }
     }
 }

powershell/ql/lib/semmle/code/powershell/File.qll

@@ -209,10 +209,6 @@ class File extends Container, @file {
     not this.isStub()
   }
 
-  predicate isInPSModulePath() {
-    is_in_psmodule_path(this)
-  }
-
   /** Holds if this file is a library. */
   predicate fromLibrary() {
     not this.getBaseName() = "" and

powershell/ql/lib/semmlecode.powershell.dbscheme

@@ -41,10 +41,6 @@ containerparent(
   unique int child: @container ref
 );
 
-is_in_psmodule_path(
-  int file: @file ref
-);
-
 /* Comments */
 comment_entity(
   unique int id: @comment_entity,