Merge pull request #43 from nexB/filter-duplicated-deps

Filter duplicated deps
aboutcode-org · Apr 19, 2023 · 121dfcb · 121dfcb
2 parents 2fa4516 + 50ab0a4
commit 121dfcb
Show file tree

Hide file tree

Showing 112 changed files with 79,407 additions and 252,413 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,14 @@
 Changelog
 =========
 
+v0.9.12
+--------
+
+This is a minor bug fix release with this fix:
+
+* Improve filtering of duplicated dependency names.
+
+
 v0.9.11
 --------
 

diff --git a/Dockerfile b/Dockerfile
@@ -31,7 +31,7 @@ RUN mkdir -p $NI_DOTNET_HOME \
     && curl --location https://aka.ms/dotnet/6.0/dotnet-sdk-linux-x64.tar.gz \
     | tar -C $NI_DOTNET_HOME -xz
 
-ARG NI_VERSION=0.9.11
+ARG NI_VERSION=0.9.12
 RUN mkdir -p $NI_HOME \
     && curl -L https://github.com/nexB/nuget-inspector/releases/download/v${NI_VERSION}/nuget-inspector-v${NI_VERSION}-linux-x64.tar.gz \
     | tar --strip-components=1 -C $NI_HOME -xz

diff --git a/Dockerfile.ubuntu18 b/Dockerfile.ubuntu18
@@ -28,7 +28,7 @@ RUN mkdir -p $NI_DOTNET_HOME \
     && curl --location https://aka.ms/dotnet/6.0/dotnet-sdk-linux-x64.tar.gz \
     | tar -C $NI_DOTNET_HOME -xz
 
-# ARG NI_VERSION=0.9.11
+# ARG NI_VERSION=0.9.12
 # RUN mkdir -p $NI_HOME \
 #    && curl -L https://github.com/nexB/nuget-inspector/releases/download/v${NI_VERSION}/nuget-inspector-v${NI_VERSION}-linux-x64.tar.gz \
 #    | tar --strip-components=1 -C $NI_HOME -xz

diff --git a/build.sh b/build.sh
@@ -17,6 +17,6 @@ dotnet publish \
   --runtime linux-x64 \
   --self-contained true \
   --configuration Release \
-  -p:Version=0.9.11 \
+  -p:Version=0.9.12 \
   --output build \
   src/nuget-inspector/nuget-inspector.csproj
diff --git a/release.sh b/release.sh
@@ -16,7 +16,7 @@
 rm -rf release/
 mkdir release
 
-VERSION=0.9.11
+VERSION=0.9.12
 
 TARGET_BASE=nuget-inspector-$(git describe)
 

diff --git a/setup.cfg b/setup.cfg
@@ -1,7 +1,7 @@
 [bumpversion]
 commit = False
 tag = False
-current_version = 0.9.11
+current_version = 0.9.12
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+))?
 serialize = 
 	{major}.{minor}.{patch}-{release}

diff --git a/src/nuget-inspector/Config.cs b/src/nuget-inspector/Config.cs
@@ -9,6 +9,6 @@ public static class Config
     public static bool TRACE_DEEP = false;
     public static bool TRACE_META = false;
     public static bool TRACE_OUTPUT = false;
-    public const string NUGET_INSPECTOR_VERSION = "0.9.11";
+    public const string NUGET_INSPECTOR_VERSION = "0.9.12";
     #pragma warning restore CA2211
 }
diff --git a/src/nuget-inspector/Models.cs b/src/nuget-inspector/Models.cs
@@ -14,14 +14,23 @@ public class Dependency
         public string? name;
         public NuGetFramework? framework;
         public VersionRange? version_range;
-       public bool is_direct;
+        public bool is_direct;
 
-        public Dependency(string? name, VersionRange? version_range, NuGetFramework? framework = null, bool is_direct = false)
+        //True only for legacy packages.config-based projects only when set there
+        public bool is_development_dependency = false;
+
+        public Dependency(
+            string? name,
+            VersionRange? version_range,
+            NuGetFramework? framework = null,
+            bool is_direct = false,
+            bool is_development_dependency = false)
         {
             this.framework = framework;
             this.name = name;
             this.version_range = version_range;
             this.is_direct = is_direct;
+            this.is_development_dependency = is_development_dependency;
         }
         /// <summary>
         /// Return a new empty BasePackageWithDeps using this package.

diff --git a/src/nuget-inspector/NugetApi.cs b/src/nuget-inspector/NugetApi.cs
@@ -121,11 +121,11 @@ private List<PackageSearchMetadataRegistration> FindPackageVersionsThroughCache(
     public PackageSearchMetadataRegistration? FindPackageVersion(PackageIdentity pid)
     {
         if (Config.TRACE)
-            Console.WriteLine($"Fetching package metadata for: {pid}");
+            Console.WriteLine($"      Fetching package metadata for: {pid}");
 
         if (psmr_by_identity.TryGetValue(key: pid, out PackageSearchMetadataRegistration? psmr))
         {
-            if (Config.TRACE)
+            if (Config.TRACE_META)
                 Console.WriteLine($"  Metadata Cache hit for '{pid}'");
             return psmr;
         }
@@ -145,7 +145,7 @@ private List<PackageSearchMetadataRegistration> FindPackageVersionsThroughCache(
 
                 if (psmr != null)
                 {
-                    if (Config.TRACE)
+                    if (Config.TRACE_META)
                         Console.WriteLine($"  Found metadata for '{pid}' from: {metadata_resource}");
                     psmr_by_identity[pid] = psmr;
                     return psmr;
@@ -400,7 +400,7 @@ public IEnumerable<PackageDependency> GetPackageDependenciesForPackage(PackageId
             return spdi;
         }
 
-        if (Config.TRACE)
+        if (Config.TRACE_META)
             Console.WriteLine($"  GetPackageInfo: {identity} framework: {framework}");
 
         foreach (var dir in dependency_info_resources)
@@ -416,7 +416,7 @@ public IEnumerable<PackageDependency> GetPackageDependenciesForPackage(PackageId
 
                 spdi = infoTask.Result;
 
-                if (Config.TRACE && spdi != null)
+                if (Config.TRACE_META && spdi != null)
                     Console.WriteLine($"    Found download URL: {spdi.DownloadUri} hash: {spdi.PackageHash}");
 
                 if (spdi != null)
@@ -506,8 +506,8 @@ public IEnumerable<PackageDependency> GetPackageDependenciesForPackage(PackageId
             var spdi = GetResolvedSourcePackageDependencyInfo(
                 identity: identity,
                 framework: project_framework);
-            if (Config.TRACE)
-                Console.WriteLine($"        Info available for package '{spdi}'");
+            if (Config.TRACE_DEEP)
+                Console.WriteLine($"      Info available for package '{spdi}'");
 
             if (spdi != null)
             {
@@ -720,8 +720,9 @@ public HashSet<SourcePackageDependencyInfo> ResolveDependenciesForPackageReferen
         var walk_context = new RemoteWalkContext(
             cacheContext: source_cache_context,
             packageSourceMapping: psm,
-            logger: nuget_logger);
-
+            logger: nuget_logger){
+                IsMsBuildBased = true
+            };
         var packages = new List<PackageId>();
         foreach (var targetref in target_references)
         {
@@ -750,8 +751,8 @@ public HashSet<SourcePackageDependencyInfo> ResolveDependenciesForPackageReferen
             walk_context.RemoteLibraryProviders.Add(provider);
         }
 
-        // We need a fake root lib as there is only one allowed input
-        // This represents the project
+        // We need a fake root lib as there is only one allowed input to walk
+        // the dependencies: This represents the project
         var rootlib = new LibraryRange(
             name: "root_project",
             versionRange: VersionRange.Parse("1.0.0"),
@@ -762,23 +763,71 @@ public HashSet<SourcePackageDependencyInfo> ResolveDependenciesForPackageReferen
         var results = walker.WalkAsync(
             library: rootlib,
             framework: project_framework,
+            // TODO: add eventual support for runtime id
+            // https://learn.microsoft.com/en-us/dotnet/core/rid-catalog
             runtimeIdentifier: null,
-            runtimeGraph: RuntimeGraph.Empty,
+            runtimeGraph: null, //RuntimeGraph.Empty,
             recursive: true);
-        var resolved_graph = results.Result;
+        GraphNode<RemoteResolveResult> resolved_graph = results.Result;
         CheckGraphForErrors(resolved_graph);
 
+        RestoreTargetGraph rg = RestoreTargetGraph.Create(
+            graphs: new List<GraphNode<RemoteResolveResult>>() { resolved_graph },
+            context: walk_context,
+            logger: nuget_logger,
+            framework: project_framework
+        );
+
         var resolved_package_info_by_package_id = new Dictionary<PackageId, ResolvedPackageInfo>();
 
-        // we iterate only inner nodes, because we have only one outer node: the "fake" root project 
-        foreach (GraphNode<RemoteResolveResult> inner in resolved_graph.InnerNodes)
+        if (Config.TRACE)
+            Console.WriteLine("      RestoreTargetGraph");
+
+        if (rg.Flattened != null)
         {
-            if (Config.TRACE_DEEP)
-                Console.WriteLine($"    Resolved direct dependency: {inner.Item.Key.Name}@{inner.Item.Key.Version}");
+            var flats = new List<GraphItem<RemoteResolveResult>>(rg.Flattened);
+            flats.Sort((x,y) => x.Data.Match.Library.CompareTo(y.Data.Match.Library));
+
+            foreach (var item in flats)
+            {
+                LibraryIdentity lib = item.Key;
+                if (lib.Type != "package")
+                    continue;
+
+
+                string name = lib.Name;
+                string version = lib.Version.ToNormalizedString();
+                bool is_prerelease = lib.Version.IsPrerelease;
 
-            FlattenGraph(inner, resolved_package_info_by_package_id);
+                RemoteMatch remote_match= item.Data.Match;
+
+                var deps = item.Data.Dependencies;
+                var pid = new PackageId(id: name, version: version,  allow_prerelease_versions: is_prerelease);
+                var rpi = new ResolvedPackageInfo() {
+                    package_id= pid,
+                    remote_match= remote_match
+                };
+                resolved_package_info_by_package_id[pid] = rpi;
+                if (Config.TRACE)
+                {
+                    Console.WriteLine($"      {lib}");
+                    foreach (var dep in deps)
+                    {
+                        Console.WriteLine($"           {lib.Type}/{dep.Name}@{dep.LibraryRange} autoref: {dep.AutoReferenced}");
+                    }
+                }
+            }
         }
 
+        // we iterate only inner nodes, because we have only one outer node: the "fake" root project 
+        // foreach (GraphNode<RemoteResolveResult> inner in resolved_graph.InnerNodes)
+        // {
+        //     if (Config.TRACE_DEEP)
+        //         Console.WriteLine($"\n    Resolved direct dependency: {inner.Item.Key.Name}@{inner.Item.Key.Version}");
+
+        //     FlattenGraph(inner, resolved_package_info_by_package_id);
+        // }
+
         HashSet<SourcePackageDependencyInfo> flat_dependencies = new();
         foreach (KeyValuePair<PackageId, ResolvedPackageInfo> item in resolved_package_info_by_package_id)
         {
@@ -803,7 +852,7 @@ public HashSet<SourcePackageDependencyInfo> ResolveDependenciesForPackageReferen
     }
 
     /// <summary>
-    /// Flatten the graph and populate the result a mapping recursively
+    /// Flatten the graph and populate the result mapping recursively
     /// </summary>
     public static void FlattenGraph(
         GraphNode<RemoteResolveResult> node,
@@ -831,8 +880,18 @@ public static void FlattenGraph(
             string name = key.Name;
             string version = key.Version.ToNormalizedString();
             bool isPrerelease = key.Version.IsPrerelease;
+
+            RemoteMatch remote_match = item.Data.Match;
+
             if (Config.TRACE_DEEP)
-                Console.WriteLine($"      FlattenGraph: node.Item {node.Item} LibraryId: {key}");
+            {
+                Console.WriteLine($"\n     FlattenGraph: node.Item {node.Item} type: {node.GetType()} LibraryId: {key} LibraryIdType: {key.Type}");
+                Console.WriteLine($"          remote_match: {remote_match} path: {remote_match.Path} type: {remote_match.Library}");
+                foreach (var idd in item.Data.Dependencies)
+                {
+                    Console.WriteLine($"             Dependency: {idd} AutoReferenced: {idd.AutoReferenced} ");
+                }
+            }
 
             var pid = new PackageId(
                 id: name,
@@ -842,11 +901,11 @@ public static void FlattenGraph(
             var resolved_package_info = new ResolvedPackageInfo
             {
                 package_id = pid,
-                remote_match = (RemoteMatch?)item.Data.Match
+                remote_match = item.Data.Match
             };
 
             if (Config.TRACE_DEEP)
-                Console.WriteLine($"        FlattenGraph: {pid} Library: {item.Data.Match.Library}");
+                Console.WriteLine($"          FlattenGraph: {pid} Library: {item.Data.Match.Library}");
 
             if (!resolved_package_info_by_package_id.ContainsKey(resolved_package_info.package_id))
                 resolved_package_info_by_package_id.Add(resolved_package_info.package_id, resolved_package_info);

diff --git a/src/nuget-inspector/NugetResolverHelper.cs b/src/nuget-inspector/NugetResolverHelper.cs
@@ -74,8 +74,8 @@ public void ResolveOne(Dependency dependency)
             var resolved_version = package_tree.GetResolvedVersion(name: pkg.Id, range: pkg.VersionRange);
             if (resolved_version != null)
             {
-                var id = new BasePackage(name: pkg.Id, version: resolved_version);
-                dependencies.Add(item: id);
+                var base_pkg = new BasePackage(name: pkg.Id, version: resolved_version);
+                dependencies.Add(item: base_pkg);
                 if (Config.TRACE)
                     Console.WriteLine($"        dependencies.Add name: {pkg.Id}, version: {resolved_version}");
             }

diff --git a/src/nuget-inspector/PackagesConfigHelper.cs b/src/nuget-inspector/PackagesConfigHelper.cs
@@ -39,7 +39,11 @@ public List<BasePackage> ProcessAll(List<Dependency> dependencies)
     {
         foreach (var dependency in dependencies)
         {
-            Add(id: dependency.name!, name: dependency.name, range: dependency.version_range, framework: dependency.framework);
+            Add(
+                id: dependency.name!,
+                name: dependency.name,
+                range: dependency.version_range,
+                framework: dependency.framework);
         }
 
         var builder = new PackageTree();
@@ -72,7 +76,11 @@ public List<BasePackage> ProcessAll(List<Dependency> dependencies)
     public void Add(string id, string? name, VersionRange? range, NuGetFramework? framework)
     {
         id = id.ToLower();
-        Resolve(id: id, name: name, project_target_framework: framework, overrideRange: range);
+        Resolve(
+            id: id,
+            name: name,
+            project_target_framework: framework,
+            overrideRange: range);
     }
 
     private void Resolve(

diff --git a/src/nuget-inspector/PackagesConfigProcessor.cs b/src/nuget-inspector/PackagesConfigProcessor.cs
@@ -103,7 +103,8 @@ private List<Dependency> GetDependencies()
                 name: name,
                 version_range: range,
                 framework: package_framework,
-                is_direct: true);
+                is_direct: true,
+                is_development_dependency: package.IsDevelopmentDependency);
             dependencies.Add(item: dep);
         }
 
@@ -131,7 +132,7 @@ private List<BasePackage> CreateBasePackage(List<Dependency> dependencies)
             catch (Exception treeex)
             {
                 if (Config.TRACE)
-                    Console.WriteLine($"PackagesConfigHandler.CreateBasePackage: TFailed processing packages.config as a tree: {treeex.Message}");
+                    Console.WriteLine($"PackagesConfigHandler.CreateBasePackage: Failed processing packages.config as a tree: {treeex.Message}");
                 var packages =
                     new List<BasePackage>(
                         collection: dependencies.Select(selector: dependency => dependency.CreateEmptyBasePackage()));

diff --git a/src/nuget-inspector/ProjectFileProcessor.cs b/src/nuget-inspector/ProjectFileProcessor.cs
@@ -142,6 +142,7 @@ public virtual List<PackageReference> GetPackageReferences()
 
             if (Config.TRACE_DEEP)
             {
+                Console.WriteLine($"    Project reference: {reference}");
                 Console.WriteLine($"    Project reference: name: {name}");
                 foreach (var meta in reference.Metadata)
                     Console.WriteLine($"        Metadata: name: '{meta.Name}' value: '{meta.EvaluatedValue}'");

diff --git a/src/nuget-inspector/nuget-inspector.csproj b/src/nuget-inspector/nuget-inspector.csproj
@@ -24,9 +24,9 @@
         <Product>nuget-inspector</Product>
         <AssemblyName>nuget-inspector</AssemblyName>
 
-        <Version>0.9.11</Version>
-        <AssemblyVersion>0.9.11.0</AssemblyVersion>
-        <FileVersion>0.9.11.0</FileVersion>
+        <Version>0.9.12</Version>
+        <AssemblyVersion>0.9.12.0</AssemblyVersion>
+        <FileVersion>0.9.12.0</FileVersion>
 
         <Authors>nexB Inc.</Authors>
         <Company>nexB Inc</Company>