From 9d81edde8ea61a7613bd11bce51d852de416b2dc Mon Sep 17 00:00:00 2001 From: Tom de Goede Date: Mon, 11 Oct 2021 12:33:14 +0200 Subject: [PATCH 1/5] Add support for netstandard framework refs --- .../BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs | 10 ++++++++++ .../BazelDotnet.Nuget/NugetRepositoryGenerator.cs | 8 +++++++- .../BazelDotnet.Project/CsProjectFileDefinition.cs | 12 ++++++++---- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs index cd89ec4..fc7b660 100644 --- a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs @@ -79,9 +79,19 @@ public NugetRepositoryEntry ResolveGroups(LocalPackageSourceInfo localPackageSou new PatternDefinition("analyzers/{assembly}") }); + // "2.0.0/build/netstandard2.0/ref/netstandard.dll" + var netstandardRefAssemblies = new PatternSet(_conventions.Properties, new [] + { + new PatternDefinition("build/{tfm}/ref/{any?}"), + }, new[] + { + new PatternDefinition("build/{tfm}/ref/{assembly}"), + }); + AddIfNotNull(entry.RefItemGroups, target.Framework, collection.FindBestItemGroup(criteria, _conventions.Patterns.CompileRefAssemblies, + netstandardRefAssemblies, _conventions.Patterns.CompileLibAssemblies) ?.Items); diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs index 712036f..562904e 100644 --- a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs @@ -9,6 +9,7 @@ using NuGet.Configuration; using NuGet.Credentials; using NuGet.Frameworks; +using NuGet.Packaging.Core; using NuGet.Protocol; using NuGet.Protocol.Core.Types; using NuGet.Protocol.Plugins; @@ -209,7 +210,7 @@ IEnumerable Elems() libs = {libs}, refs = {StringArray(package.RefItemGroups.SingleOrDefault()?.Items.Select(v => v.StartsWith("//") ? v : $"{folder}/{v}"))}, analyzers = {StringArray(package.AnalyzerItemGroups.SingleOrDefault()?.Items.Select(v => v.StartsWith("//") ? v : $"{folder}/{v}"))}, - deps = {StringArray(package.DependencyGroups.SingleOrDefault()?.Packages.Select(p => $"//{p.Id.ToLower()}"))}, + deps = {StringArray(package.DependencyGroups.SingleOrDefault()?.Packages.Where(ShouldIncludeDep).Select(p => $"//{p.Id.ToLower()}"))}, data = ["":content_files""], version = ""{identity.Version}"", )"; @@ -218,6 +219,11 @@ IEnumerable Elems() return string.Join("\n\n", Elems()); } + private bool ShouldIncludeDep(PackageDependency package) => + // Some libraries depend on netstandard. It will not include anything of use + // Should only be referenced explicitly as targetframework + !string.Equals(package.Id, "netstandard.library", StringComparison.OrdinalIgnoreCase); + private static string Indent(string input) { var lines = input.Split('\n'); diff --git a/src/Afas.BazelDotnet.Project/BazelDotnet.Project/CsProjectFileDefinition.cs b/src/Afas.BazelDotnet.Project/BazelDotnet.Project/CsProjectFileDefinition.cs index 104f459..336adbc 100644 --- a/src/Afas.BazelDotnet.Project/BazelDotnet.Project/CsProjectFileDefinition.cs +++ b/src/Afas.BazelDotnet.Project/BazelDotnet.Project/CsProjectFileDefinition.cs @@ -13,10 +13,7 @@ internal class CsProjectFileDefinition public CsProjectFileDefinition(string projectFilePath, string slnBasePath) { RelativeFilePath = Path.GetRelativePath(slnBasePath, projectFilePath); - PackageReferences = new List - { - "Microsoft.NETCore.App.Ref", - }; + PackageReferences = new List(); ProjectReference = new List(); BazelData = new List(); EmbeddedResources = new List(); @@ -49,6 +46,13 @@ public CsProjectFileDefinition Deserialize(Func csprojToLabel, Type = GetProjectType(_document); + var targetFramework = _document.Descendants("TargetFramework").FirstOrDefault()?.Value; + var targetFrameworkRef = string.Equals("netstandard2.0", targetFramework, StringComparison.OrdinalIgnoreCase) + ? "Netstandard.Library" + : "Microsoft.NETCore.App.Ref"; + + PackageReferences.Add(targetFrameworkRef); + foreach(var reference in _document.Descendants("PackageReference")) { var name = reference.Attribute("Include").Value; From 028796648c298964f0a7f3190940399e05202cdf Mon Sep 17 00:00:00 2001 From: Tom de Goede Date: Fri, 15 Oct 2021 12:08:27 +0200 Subject: [PATCH 2/5] Add support for multiple tfms --- .../FrameworkDependencyResolver.cs | 4 +- .../NugetRepositoryEntryBuilder.cs | 2 +- .../NugetRepositoryGenerator.cs | 82 +++++++++++++++---- src/Afas.BazelDotnet/BazelDotnet/Program.cs | 10 +-- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/FrameworkDependencyResolver.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/FrameworkDependencyResolver.cs index 990d092..beff527 100644 --- a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/FrameworkDependencyResolver.cs +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/FrameworkDependencyResolver.cs @@ -67,7 +67,7 @@ private static string GetFrameworkVersion(string targetFramework) continue; } - var existingPackageIsEmpty = existingPackage.RefItemGroups.SingleOrDefault()?.Items.Any() != true; + var existingPackageIsEmpty = existingPackage.RefItemGroups.FirstOrDefault()?.Items.Any() != true; if(existingPackageIsEmpty || packageOverride >= existingPackage.Version) { if(frameworkList.TryGetValue(id, out var frameworkItem)) @@ -121,7 +121,7 @@ private static string GetFrameworkVersion(string targetFramework) bool OverridesAssemblyVersion(NugetRepositoryEntry package, Version frameworkListVersion) { - var refItems = package.RefItemGroups.Single(); + var refItems = package.RefItemGroups.First(); if(!refItems.Items.Any()) { return true; diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs index fc7b660..de946b4 100644 --- a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs @@ -156,7 +156,7 @@ public NugetRepositoryEntry BuildFrameworkOverride(NugetRepositoryEntry entry, s newEntry.DependencyGroups.AddRange(entry.DependencyGroups); if(frameworkOverride != null) { - newEntry.RefItemGroups.Add(new FrameworkSpecificGroup(_targets.Single().Framework, new [] + newEntry.RefItemGroups.Add(new FrameworkSpecificGroup(_targets.First().Framework, new [] { frameworkOverride, })); diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs index 562904e..70413af 100644 --- a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs @@ -9,6 +9,7 @@ using NuGet.Configuration; using NuGet.Credentials; using NuGet.Frameworks; +using NuGet.Packaging; using NuGet.Packaging.Core; using NuGet.Protocol; using NuGet.Protocol.Core.Types; @@ -18,8 +19,46 @@ namespace Afas.BazelDotnet.Nuget { public class NugetRepositoryGenerator { + private class TargetFrameworkProcessor + { + public string CreateConfigurationTargets(IReadOnlyList targetFrameworks) => $@" +load(""@bazel_skylib//rules:common_settings.bzl"", ""string_flag"") + +string_flag( + name = ""framework"", + values = {Indent(StringArray(targetFrameworks))}, + build_setting_default = ""{targetFrameworks.First()}"" +) + +{string.Join("\n", targetFrameworks + .Select(f => $@"config_setting( + name = ""frameworks-{f}"", + flag_values = {{ + "":framework"": ""{f}"" + }} +) +"))}"; + + public string CreateSelectStatementWhenApplicable(IReadOnlyList groups, Func format) + { + var arrays = groups.Select(g => StringArray(g.Items.Select(format))).ToArray(); + return arrays.Distinct().Count() switch + { + 0 => "[]", + 1 => arrays[0], + _ => Select(Indent(Dict(groups.Select(g => $"//:frameworks-{g.TargetFramework.GetShortFolderName()}") + .Zip(arrays) + .Skip(1).Append(("//conditions:default", arrays[0])) + .ToDictionary(t => t.Item1, t => t.Item2)))) + }; + } + + private static string Select(string input) => $@"select({input})"; + } + private readonly string _nugetConfig; private readonly ILookup _imports; + private readonly TargetFrameworkProcessor _targetFrameworkProcessor = new(); public NugetRepositoryGenerator(string nugetConfig, ILookup imports) { @@ -27,7 +66,7 @@ public NugetRepositoryGenerator(string nugetConfig, ILookup ResolveLocalPackages(string targetFramework, string targetRuntime, + private async Task ResolveLocalPackages(IReadOnlyList targetFrameworks, string targetRuntime, IEnumerable<(string package, string version)> packageReferences) { ILogger logger = new ConsoleLogger(); @@ -44,16 +83,22 @@ private async Task ResolveLocalPackages(string targetFra dependencyGraphResolver.AddPackageReference(package, version); } - var dependencyGraph = await dependencyGraphResolver.ResolveGraph(targetFramework, targetRuntime).ConfigureAwait(false); + // TODO ResolveGraph on multiple tfms + var dependencyGraph = await dependencyGraphResolver.ResolveGraph(targetFrameworks.First(), targetRuntime).ConfigureAwait(false); var localPackages = await dependencyGraphResolver.DownloadPackages(dependencyGraph).ConfigureAwait(false); - var entryBuilder = new NugetRepositoryEntryBuilder(dependencyGraph.Conventions) - .WithTarget(new FrameworkRuntimePair(NuGetFramework.Parse(targetFramework), targetRuntime)); + var entryBuilder = new NugetRepositoryEntryBuilder(dependencyGraph.Conventions); + + foreach(var targetFramework in targetFrameworks) + { + entryBuilder.WithTarget(new FrameworkRuntimePair(NuGetFramework.Parse(targetFramework), targetRuntime)); + } var entries = localPackages.Select(entryBuilder.ResolveGroups).ToArray(); var (frameworkEntries, frameworkOverrides) = await new FrameworkDependencyResolver(dependencyGraphResolver) - .ResolveFrameworkPackages(entries, targetFramework) + // TODO ResolveFrameworkPackages on multiple tfms + .ResolveFrameworkPackages(entries, targetFrameworks.First()) .ConfigureAwait(false); var overridenEntries = entries.Select(p => @@ -64,9 +109,9 @@ private async Task ResolveLocalPackages(string targetFra return frameworkEntries.Concat(overridenEntries).ToArray(); } - public async Task WriteRepository(string targetFramework, string targetRuntime, IEnumerable<(string package, string version)> packageReferences) + public async Task WriteRepository(IReadOnlyList targetFrameworks, string targetRuntime, IEnumerable<(string package, string version)> packageReferences) { - var packages = await ResolveLocalPackages(targetFramework, targetRuntime, packageReferences).ConfigureAwait(false); + var packages = await ResolveLocalPackages(targetFrameworks, targetRuntime, packageReferences).ConfigureAwait(false); var symlinks = new HashSet<(string link, string target)>(); @@ -96,6 +141,8 @@ public async Task WriteRepository(string targetFramework, string targetRuntime, } } + File.WriteAllText("BUILD", _targetFrameworkProcessor.CreateConfigurationTargets(targetFrameworks)); + File.WriteAllText("symlinks_manifest", string.Join("\n", symlinks .Select(sl => $@"{sl.link} {sl.target}"))); @@ -144,7 +191,7 @@ private void WriteBuildFile(IGrouping entryGroup, private IEnumerable GetContentFiles(NugetRepositoryEntry package) { - var group = package.ContentFileGroups.SingleOrDefault(); + var group = package.ContentFileGroups.FirstOrDefault(); if(group?.Items.Any() == true) { // Symlink optimization to link the entire group folder e.g. contentFiles/any/netcoreapp3.1 @@ -176,9 +223,9 @@ IEnumerable Elems() if(_imports.Contains(name)) { - var selects = _imports[name].ToDictionary(i => i.configSetting, i => i.target); + var selects = _imports[name].ToDictionary(i => i.configSetting, i => Quote(i.target)); name += "__nuget"; - selects["//conditions:default"] = name; + selects["//conditions:default"] = Quote(name); yield return $@"alias( name = ""{identity.Id.ToLower()}"", @@ -187,7 +234,7 @@ IEnumerable Elems() } bool hasDebugDlls = package.DebugRuntimeItemGroups.Count != 0; - var libs = StringArray(package.RuntimeItemGroups.SingleOrDefault()?.Items.Select(v => $"{folder}/{v}")); + var libs = _targetFrameworkProcessor.CreateSelectStatementWhenApplicable(package.RuntimeItemGroups, v => $"{folder}/{v}"); if(hasDebugDlls) { @@ -199,18 +246,19 @@ IEnumerable Elems() }, )"; libs = Indent($@"select({{ - "":compilation_mode_dbg"": {StringArray(package.DebugRuntimeItemGroups.Single().Items.Select(v => $"{folder}/{v}"))}, + "":compilation_mode_dbg"": {StringArray(package.DebugRuntimeItemGroups.First().Items.Select(v => $"{folder}/{v}"))}, ""//conditions:default"": {libs}, }})"); } + var refs = _targetFrameworkProcessor.CreateSelectStatementWhenApplicable(package.RefItemGroups, v => v.StartsWith("//") ? v : $"{folder}/{v}"); yield return $@"core_import_library( name = ""{name}"", libs = {libs}, - refs = {StringArray(package.RefItemGroups.SingleOrDefault()?.Items.Select(v => v.StartsWith("//") ? v : $"{folder}/{v}"))}, - analyzers = {StringArray(package.AnalyzerItemGroups.SingleOrDefault()?.Items.Select(v => v.StartsWith("//") ? v : $"{folder}/{v}"))}, - deps = {StringArray(package.DependencyGroups.SingleOrDefault()?.Packages.Where(ShouldIncludeDep).Select(p => $"//{p.Id.ToLower()}"))}, + refs = {refs}, + analyzers = {StringArray(package.AnalyzerItemGroups.FirstOrDefault()?.Items.Select(v => v.StartsWith("//") ? v : $"{folder}/{v}"))}, + deps = {StringArray(package.DependencyGroups.FirstOrDefault()?.Packages.Where(ShouldIncludeDep).Select(p => $"//{p.Id.ToLower()}"))}, data = ["":content_files""], version = ""{identity.Version}"", )"; @@ -238,6 +286,8 @@ private static string StringArray(IEnumerable items) => items?.Any() != {string.Join(",\n", items.Select(i => $@" ""{i}"""))} ]"); + private static string Quote(string input) => $@"""{input}"""; + private static string Dict(IReadOnlyDictionary items) { var s = new StringBuilder(); @@ -246,7 +296,7 @@ private static string Dict(IReadOnlyDictionary items) foreach(var (key, value) in items) { - s.Append($@" ""{key}"": ""{value}"", + s.Append($@" ""{key}"": {value}, "); } diff --git a/src/Afas.BazelDotnet/BazelDotnet/Program.cs b/src/Afas.BazelDotnet/BazelDotnet/Program.cs index 4396cff..c86632e 100644 --- a/src/Afas.BazelDotnet/BazelDotnet/Program.cs +++ b/src/Afas.BazelDotnet/BazelDotnet/Program.cs @@ -27,7 +27,7 @@ public static void Main(string[] args) app.Command("repository", repoCmd => { var nugetConfig = repoCmd.Argument("nugetConfig", "The path to the Packages.Props file"); - var tfmOption = repoCmd.Option("-t|--tfm", "The target framework to restore", CommandOptionType.SingleOrNoValue); + var tfmOption = repoCmd.Option("-t|--tfm", "The target framework to restore", CommandOptionType.MultipleValue); var packageProps = repoCmd.Option("-p|--package", "Packages.Props files", CommandOptionType.MultipleValue); var importsOption = repoCmd.Option("-i|--imports", "Import files with dictionary of imported project labels (PackageName=Label)", CommandOptionType.MultipleValue); @@ -35,8 +35,8 @@ public static void Main(string[] args) { var packagePropsFilePaths = packageProps.Values.Select(v => Path.Combine(Directory.GetCurrentDirectory(), v)).ToArray(); var nugetConfigFilePath = Path.Combine(Directory.GetCurrentDirectory(), nugetConfig.Value); - var tfm = tfmOption.HasValue() ? tfmOption.Value() : "net5.0"; - await WriteRepository(tfm, packagePropsFilePaths, nugetConfigFilePath, importsOption.Values).ConfigureAwait(false); + IReadOnlyList tfms = tfmOption.HasValue() ? tfmOption.Values : new [] { "net5.0" }; + await WriteRepository(tfms, packagePropsFilePaths, nugetConfigFilePath, importsOption.Values).ConfigureAwait(false); return 0; }); }); @@ -130,7 +130,7 @@ bool Included((string update, string version) arg) => !arg.version.EndsWith("-local-dev", StringComparison.OrdinalIgnoreCase); } - private static async Task WriteRepository(string tfm, IEnumerable packagePropsFiles, string nugetConfig, IReadOnlyCollection importMappings = null) + private static async Task WriteRepository(IReadOnlyList tfms, IEnumerable packagePropsFiles, string nugetConfig, IReadOnlyCollection importMappings = null) { // Note: no conlict resolution. Maybe we can add them in the dep graph. For now multiple Packages.Props is not really a use case anymore (string, string)[] deps = packagePropsFiles @@ -142,7 +142,7 @@ private static async Task WriteRepository(string tfm, IEnumerable packag .ToLookup(i => i.project, i => (i.target, i.configSetting), StringComparer.OrdinalIgnoreCase); await new NugetRepositoryGenerator(nugetConfig, imports) - .WriteRepository(tfm, "win-x64", deps) + .WriteRepository(tfms, "win-x64", deps) .ConfigureAwait(false); } From 7c7c54832bd989a954fad1050d3fbb3645c455e5 Mon Sep 17 00:00:00 2001 From: Tom de Goede Date: Tue, 16 Nov 2021 10:54:12 +0100 Subject: [PATCH 3/5] Add lockfile support --- .../LocalPackageExtractor.cs | 8 +- .../NugetRepositoryEntryBuilder.cs | 18 +-- .../NugetRepositoryGenerator.cs | 56 +++++--- .../BazelDotnet.Nuget/RootProject.cs | 72 ++++++++++ .../TransitiveDependencyResolver.cs | 131 +++++++++--------- src/Afas.BazelDotnet/BazelDotnet/Program.cs | 8 +- 6 files changed, 190 insertions(+), 103 deletions(-) create mode 100644 src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/RootProject.cs diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/LocalPackageExtractor.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/LocalPackageExtractor.cs index 1b7cd95..f7d0b4d 100644 --- a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/LocalPackageExtractor.cs +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/LocalPackageExtractor.cs @@ -19,9 +19,9 @@ internal class LocalPackageExtractor private readonly SourceCacheContext _cache; private readonly PackageExtractionContext _context; - public LocalPackageExtractor(ISettings settings, ILogger logger, SourceCacheContext cache) + public LocalPackageExtractor(ISettings settings, NuGetv3LocalRepository v3LocalRepository, ILogger logger, SourceCacheContext cache) { - _v3LocalRepository = new NuGetv3LocalRepository(SettingsUtility.GetGlobalPackagesFolder(settings), new LocalPackageFileCache(), false); + _v3LocalRepository = v3LocalRepository; _cache = cache; _context = new PackageExtractionContext(PackageSaveMode.Defaultv3, XmlDocFileSaveMode.Skip, ClientPolicyContext.GetClientPolicy(settings, logger), logger); } @@ -31,11 +31,11 @@ public async Task EnsureLocalPackage(IRemoteDependencyPr { if(!_v3LocalRepository.Exists(packageIdentity.Id, packageIdentity.Version)) { - var packageDependency = await provider.GetPackageDownloaderAsync(packageIdentity, _cache, _context.Logger, CancellationToken.None); + var downloader = await provider.GetPackageDownloaderAsync(packageIdentity, _cache, _context.Logger, CancellationToken.None); var installed = await PackageExtractor.InstallFromSourceAsync( packageIdentity, - packageDependency, + downloader, _v3LocalRepository.PathResolver, _context, CancellationToken.None, diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs index de946b4..89da763 100644 --- a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs @@ -13,24 +13,12 @@ namespace Afas.BazelDotnet.Nuget internal class NugetRepositoryEntryBuilder { private readonly ManagedCodeConventions _conventions; - private readonly List _targets; + private readonly IReadOnlyCollection _targets; - public NugetRepositoryEntryBuilder(ManagedCodeConventions conventions) + public NugetRepositoryEntryBuilder(ManagedCodeConventions conventions, IReadOnlyCollection targets) { _conventions = conventions; - _targets = new List(); - } - - public NugetRepositoryEntryBuilder WithTarget(FrameworkRuntimePair target) - { - _targets.Add(target); - return this; - } - - public NugetRepositoryEntryBuilder WithTarget(NuGetFramework target) - { - _targets.Add(new FrameworkRuntimePair(target, runtimeIdentifier: null)); - return this; + _targets = targets; } public NugetRepositoryEntry ResolveGroups(LocalPackageSourceInfo localPackageSourceInfo) diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs index 70413af..722736b 100644 --- a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs @@ -5,15 +5,21 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using NuGet.Client; +using NuGet.Commands; using NuGet.Common; using NuGet.Configuration; using NuGet.Credentials; using NuGet.Frameworks; +using NuGet.LibraryModel; using NuGet.Packaging; using NuGet.Packaging.Core; +using NuGet.ProjectModel; using NuGet.Protocol; using NuGet.Protocol.Core.Types; using NuGet.Protocol.Plugins; +using NuGet.RuntimeModel; +using NuGet.Versioning; namespace Afas.BazelDotnet.Nuget { @@ -66,37 +72,53 @@ public NugetRepositoryGenerator(string nugetConfig, ILookup TryReadLockFile(string nugetLockFilePath, ILogger logger) + { + if(string.IsNullOrEmpty(nugetLockFilePath)) + { + return null; + } + + if(!File.Exists(nugetLockFilePath)) + { + throw new Exception("Nuget Lock File was specified but not found."); + } + + return PackagesLockFileFormat.Parse(await File.ReadAllTextAsync(nugetLockFilePath), logger, nugetLockFilePath); + } + private async Task ResolveLocalPackages(IReadOnlyList targetFrameworks, string targetRuntime, - IEnumerable<(string package, string version)> packageReferences) + IEnumerable<(string package, string version)> packageReferences, string nugetLockFilePath) { ILogger logger = new ConsoleLogger(); + + var rootProject = new RootProject( + targetFrameworks.Select(tfm => new FrameworkRuntimePair(NuGetFramework.Parse(tfm), targetRuntime)).ToArray(), + packageReferences.Select(r => new LibraryRange(r.package, VersionRange.Parse(r.version), LibraryDependencyTarget.Package)).ToArray(), + await TryReadLockFile(nugetLockFilePath, logger)); + var settings = Settings.LoadSpecificSettings(Path.GetDirectoryName(_nugetConfig), Path.GetFileName(_nugetConfig)); DefaultCredentialServiceUtility.SetupDefaultCredentialService(logger, nonInteractive: true); // ~/.nuget/packages using var cache = new SourceCacheContext(); - var dependencyGraphResolver = new TransitiveDependencyResolver(settings, logger, cache); + var dependencyGraphResolver = new TransitiveDependencyResolver(settings, logger, cache, rootProject); + + var dependencyGraphs = await dependencyGraphResolver.ResolveGraphs(); - foreach(var (package, version) in packageReferences) + if(!string.IsNullOrEmpty(nugetLockFilePath) && !rootProject.IsLockFileValid) { - dependencyGraphResolver.AddPackageReference(package, version); + await File.WriteAllTextAsync(nugetLockFilePath, dependencyGraphResolver.RenderLockFile(dependencyGraphs)); } - // TODO ResolveGraph on multiple tfms - var dependencyGraph = await dependencyGraphResolver.ResolveGraph(targetFrameworks.First(), targetRuntime).ConfigureAwait(false); - var localPackages = await dependencyGraphResolver.DownloadPackages(dependencyGraph).ConfigureAwait(false); - - var entryBuilder = new NugetRepositoryEntryBuilder(dependencyGraph.Conventions); - - foreach(var targetFramework in targetFrameworks) - { - entryBuilder.WithTarget(new FrameworkRuntimePair(NuGetFramework.Parse(targetFramework), targetRuntime)); - } + var runtimeGraph = dependencyGraphs.Select(g => g.RuntimeGraph).Aggregate(RuntimeGraph.Merge); + var entryBuilder = new NugetRepositoryEntryBuilder(new ManagedCodeConventions(runtimeGraph), rootProject.Targets); + var localPackages = await dependencyGraphResolver.DownloadPackages(dependencyGraphs).ConfigureAwait(false); var entries = localPackages.Select(entryBuilder.ResolveGroups).ToArray(); - var (frameworkEntries, frameworkOverrides) = await new FrameworkDependencyResolver(dependencyGraphResolver) + var (frameworkEntries, frameworkOverrides) = await new FrameworkDependencyResolver(dependencyGraphResolver) // TODO ResolveFrameworkPackages on multiple tfms .ResolveFrameworkPackages(entries, targetFrameworks.First()) .ConfigureAwait(false); @@ -109,9 +131,9 @@ private async Task ResolveLocalPackages(IReadOnlyList targetFrameworks, string targetRuntime, IEnumerable<(string package, string version)> packageReferences) + public async Task WriteRepository(IReadOnlyList targetFrameworks, string targetRuntime, IEnumerable<(string package, string version)> packageReferences, string nugetLockFilePath) { - var packages = await ResolveLocalPackages(targetFrameworks, targetRuntime, packageReferences).ConfigureAwait(false); + var packages = await ResolveLocalPackages(targetFrameworks, targetRuntime, packageReferences, nugetLockFilePath).ConfigureAwait(false); var symlinks = new HashSet<(string link, string target)>(); diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/RootProject.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/RootProject.cs new file mode 100644 index 0000000..8eab061 --- /dev/null +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/RootProject.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Commands; +using NuGet.Common; +using NuGet.Configuration; +using NuGet.DependencyResolver; +using NuGet.Frameworks; +using NuGet.LibraryModel; +using NuGet.ProjectModel; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.RuntimeModel; +using NuGet.Versioning; + +namespace Afas.BazelDotnet.Nuget +{ + internal class RootProject + { + private const string _rootProjectName = "Root"; + private const string _rootProjectVersion = "1.0.0"; + private readonly Lazy _isLockFileValid; + + public RootProject(IReadOnlyCollection targets, IReadOnlyCollection dependencies, PackagesLockFile nugetLockFile = null) + { + Targets = targets; + NugetLockFile = nugetLockFile; + PackageSpec = PrepareRootProject(targets, dependencies); + _isLockFileValid = new Lazy(() => + { + if(NugetLockFile == null) + { + return false; + } + + DependencyGraphSpec spec = new DependencyGraphSpec().CreateFromClosure(_rootProjectName, new [] { PackageSpec }); + return PackagesLockFileUtilities.IsLockFileValid(spec, NugetLockFile).IsValid; + }); + } + + public IReadOnlyCollection Targets { get; } + + public PackageSpec PackageSpec { get; } + + public PackagesLockFile NugetLockFile { get; } + + public string Name => _rootProjectName; + + public string Version => _rootProjectVersion; + + public bool IsLockFileValid => _isLockFileValid.Value; + + private static PackageSpec PrepareRootProject(IReadOnlyCollection targets, IReadOnlyCollection dependencies) => + new PackageSpec(targets.Select(t => new TargetFrameworkInformation { FrameworkName = t.Framework }).ToArray()) + { + Name = _rootProjectName, + Version = NuGetVersion.Parse(_rootProjectVersion), + Dependencies = dependencies.Select(package => new LibraryDependency + { + LibraryRange = package, + Type = LibraryDependencyType.Build, + IncludeType = LibraryIncludeFlags.None, + SuppressParent = LibraryIncludeFlags.All, + AutoReferenced = true, + GeneratePathProperty = false, + }) + .ToList(), + RestoreMetadata = new ProjectRestoreMetadata { ProjectUniqueName = _rootProjectName }, + RuntimeGraph = new RuntimeGraph(targets.Where(t => !string.IsNullOrEmpty(t.RuntimeIdentifier)).Select(t => new RuntimeDescription(t.RuntimeIdentifier)).Distinct()) + }; + } +} diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/TransitiveDependencyResolver.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/TransitiveDependencyResolver.cs index f85f841..8322299 100644 --- a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/TransitiveDependencyResolver.cs +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/TransitiveDependencyResolver.cs @@ -16,28 +16,23 @@ using NuGet.RuntimeModel; using NuGet.Versioning; +#nullable enable + namespace Afas.BazelDotnet.Nuget { internal class TransitiveDependencyResolver { - private const string _rootProjectName = "Root"; - private const string _rootProjectVersion = "1.0.0"; - + private readonly RootProject _rootProject; private readonly RemoteWalkContext _context; private readonly LocalPackageExtractor _localPackageExtractor; - private readonly List _packages; - - public TransitiveDependencyResolver(ISettings settings, ILogger logger, SourceCacheContext cache) - { - _context = CreateRemoteWalkContext(settings, cache, logger); - _localPackageExtractor = new LocalPackageExtractor(settings, _context.Logger, _context.CacheContext); - _packages = new List(); - } + private readonly NuGetv3LocalRepository _v3LocalRepository; - public TransitiveDependencyResolver AddPackageReference(string package, string version) + public TransitiveDependencyResolver(ISettings settings, ILogger logger, SourceCacheContext cache, RootProject rootProject) { - _packages.Add(new LibraryRange(package, VersionRange.Parse(version), LibraryDependencyTarget.Package)); - return this; + _rootProject = rootProject; + _context = CreateRemoteWalkContext(settings, cache, logger, rootProject); + _v3LocalRepository = new NuGetv3LocalRepository(SettingsUtility.GetGlobalPackagesFolder(settings), new LocalPackageFileCache(), false); + _localPackageExtractor = new LocalPackageExtractor(settings, _v3LocalRepository, _context.Logger, _context.CacheContext); } public async Task ResolveFrameworkReference(string id, string version, string targetFramework) @@ -46,72 +41,52 @@ public async Task ResolveFrameworkReference(string id, strin return await GetIndependentGraph(id, version, nugetTargetFramework, _context).ConfigureAwait(false); } - public async Task ResolveGraph(string targetFramework, string targetRuntime = null) + public async Task> ResolveGraphs() { - var nugetTargetFramework = NuGetFramework.Parse(targetFramework); - PrepareRootProject(nugetTargetFramework); - - var independentGraph = await GetIndependentGraph(_rootProjectName, _rootProjectVersion, nugetTargetFramework, _context).ConfigureAwait(false); + var graphs = new List(); - if(string.IsNullOrEmpty(targetRuntime)) + foreach(var target in _rootProject.Targets) { - return independentGraph; - } + var independentGraph = await GetIndependentGraph(_rootProject.Name, _rootProject.Version, target.Framework, _context).ConfigureAwait(false); + + graphs.Add(independentGraph); - // We could target multiple runtimes with RuntimeGraph.Merge - var platformSpecificGraph = await GetPlatformSpecificGraph(independentGraph, _rootProjectName, _rootProjectVersion, - nugetTargetFramework, targetRuntime, _context, _localPackageExtractor) - .ConfigureAwait(false); + if(string.IsNullOrEmpty(target.RuntimeIdentifier)) + { + continue; + } + + // We could target multiple runtimes with RuntimeGraph.Merge + var platformSpecificGraph = await GetPlatformSpecificGraph(independentGraph, _rootProject.Name, _rootProject.Version, + target.Framework, target.RuntimeIdentifier, _context, _localPackageExtractor) + .ConfigureAwait(false); + + graphs.Add(platformSpecificGraph); + } - return platformSpecificGraph; + return graphs; } - public async Task> DownloadPackages(RestoreTargetGraph dependencyGraph) + public async Task> DownloadPackages(IEnumerable dependencyGraphs) { - return await Task.WhenAll(dependencyGraph.Flattened - .Where(i => !string.Equals(i.Key.Name, _rootProjectName, StringComparison.OrdinalIgnoreCase)) + return await Task.WhenAll(dependencyGraphs.SelectMany(g => g.Flattened) + .Distinct(GraphItemKeyComparer.Instance) + .Where(i => !string.Equals(i.Key.Name, _rootProject.Name, StringComparison.OrdinalIgnoreCase)) .Select(DownloadPackage)); } public Task DownloadPackage(GraphItem item) => _localPackageExtractor.EnsureLocalPackage(item.Data.Match.Provider, ToPackageIdentity(item.Data.Match)); - private void PrepareRootProject(NuGetFramework targetFramework) + public string RenderLockFile(IEnumerable targetGraphs) { - PackageSpec project = new PackageSpec(new List - { - new TargetFrameworkInformation - { - FrameworkName = targetFramework, - } - }); - project.Name = _rootProjectName; - project.Version = NuGetVersion.Parse(_rootProjectVersion); - - project.Dependencies = _packages.Select(package => new LibraryDependency - { - LibraryRange = package, - Type = LibraryDependencyType.Build, - IncludeType = LibraryIncludeFlags.None, - SuppressParent = LibraryIncludeFlags.All, - AutoReferenced = true, - GeneratePathProperty = false, - }) - .ToList(); - - // In case we get re-used we clear the previous value first - _context.ProjectLibraryProviders.Clear(); - _context.ProjectLibraryProviders.Add(new PackageSpecReferenceDependencyProvider(new[] - { - new ExternalProjectReference( - project.Name, - project, - msbuildProjectPath: null, - projectReferences: Enumerable.Empty()), - }, _context.Logger)); + var builder = new LockFileBuilder(lockFileVersion: 2, _context.Logger, new()); + var lockFile = builder.CreateLockFile(previousLockFile: new LockFile(), _rootProject.PackageSpec, targetGraphs, new [] { _v3LocalRepository }, _context); + var nugetLockFile = new PackagesLockFileBuilder().CreateNuGetLockFile(lockFile); + return PackagesLockFileFormat.Render(nugetLockFile); } - private RemoteWalkContext CreateRemoteWalkContext(ISettings settings, SourceCacheContext cache, ILogger logger) + private static RemoteWalkContext CreateRemoteWalkContext(ISettings settings, SourceCacheContext cache, ILogger logger, RootProject rootProject) { // nuget.org etc. var globalPackagesFolder = SettingsUtility.GetGlobalPackagesFolder(settings); @@ -135,6 +110,34 @@ private RemoteWalkContext CreateRemoteWalkContext(ISettings settings, SourceCach isFallbackFolderSource: false)); } + context.ProjectLibraryProviders.Add(new PackageSpecReferenceDependencyProvider(new[] + { + new ExternalProjectReference( + rootProject.Name, + rootProject.PackageSpec, + msbuildProjectPath: null, + projectReferences: Enumerable.Empty()), + }, context.Logger)); + + if(rootProject.IsLockFileValid) + { + // pass lock file details down to generate restore graph + foreach (var target in rootProject.NugetLockFile.Targets) + { + var libraries = target.Dependencies + .Where(dep => dep.Type != PackageDependencyType.Project) + .Select(dep => new LibraryIdentity(dep.Id, dep.ResolvedVersion, LibraryType.Package)) + .ToList(); + + // TODO clean up + libraries.Add(new LibraryIdentity("Microsoft.NETCore.App.Ref", new NuGetVersion(target.TargetFramework.Version), LibraryType.Package)); + libraries.Add(new LibraryIdentity("Microsoft.AspNetCore.App.Ref", new NuGetVersion(target.TargetFramework.Version), LibraryType.Package)); + + // add lock file libraries into RemoteWalkContext so that it can be used during restore graph generation + context.LockFileLibraries.Add(new LockFileCacheKey(target.TargetFramework, target.RuntimeIdentifier), libraries); + } + } + return context; } @@ -179,7 +182,7 @@ private static async Task GetPlatformSpecificGraph(RestoreTa resultWin }, context, context.Logger, framework, runtimeIdentifier); - async Task GetRuntimeGraphTask(GraphItem item) + async Task GetRuntimeGraphTask(GraphItem item) { var packageIdentity = ToPackageIdentity(item.Data.Match); diff --git a/src/Afas.BazelDotnet/BazelDotnet/Program.cs b/src/Afas.BazelDotnet/BazelDotnet/Program.cs index c86632e..d893dbd 100644 --- a/src/Afas.BazelDotnet/BazelDotnet/Program.cs +++ b/src/Afas.BazelDotnet/BazelDotnet/Program.cs @@ -27,6 +27,7 @@ public static void Main(string[] args) app.Command("repository", repoCmd => { var nugetConfig = repoCmd.Argument("nugetConfig", "The path to the Packages.Props file"); + var nugetLockFile = repoCmd.Option("-l|--lock", "Nuget lock file", CommandOptionType.SingleOrNoValue); var tfmOption = repoCmd.Option("-t|--tfm", "The target framework to restore", CommandOptionType.MultipleValue); var packageProps = repoCmd.Option("-p|--package", "Packages.Props files", CommandOptionType.MultipleValue); var importsOption = repoCmd.Option("-i|--imports", "Import files with dictionary of imported project labels (PackageName=Label)", CommandOptionType.MultipleValue); @@ -35,8 +36,9 @@ public static void Main(string[] args) { var packagePropsFilePaths = packageProps.Values.Select(v => Path.Combine(Directory.GetCurrentDirectory(), v)).ToArray(); var nugetConfigFilePath = Path.Combine(Directory.GetCurrentDirectory(), nugetConfig.Value); + var nugetLockFilePath = nugetLockFile.HasValue() ? Path.Combine(Directory.GetCurrentDirectory(), nugetLockFile.Value()) : null; IReadOnlyList tfms = tfmOption.HasValue() ? tfmOption.Values : new [] { "net5.0" }; - await WriteRepository(tfms, packagePropsFilePaths, nugetConfigFilePath, importsOption.Values).ConfigureAwait(false); + await WriteRepository(tfms, packagePropsFilePaths, nugetConfigFilePath, nugetLockFilePath, importsOption.Values).ConfigureAwait(false); return 0; }); }); @@ -130,7 +132,7 @@ bool Included((string update, string version) arg) => !arg.version.EndsWith("-local-dev", StringComparison.OrdinalIgnoreCase); } - private static async Task WriteRepository(IReadOnlyList tfms, IEnumerable packagePropsFiles, string nugetConfig, IReadOnlyCollection importMappings = null) + private static async Task WriteRepository(IReadOnlyList tfms, IEnumerable packagePropsFiles, string nugetConfig, string nugetLockFilePath, IReadOnlyCollection importMappings = null) { // Note: no conlict resolution. Maybe we can add them in the dep graph. For now multiple Packages.Props is not really a use case anymore (string, string)[] deps = packagePropsFiles @@ -142,7 +144,7 @@ private static async Task WriteRepository(IReadOnlyList tfms, IEnumerabl .ToLookup(i => i.project, i => (i.target, i.configSetting), StringComparer.OrdinalIgnoreCase); await new NugetRepositoryGenerator(nugetConfig, imports) - .WriteRepository(tfms, "win-x64", deps) + .WriteRepository(tfms, "win-x64", deps, nugetLockFilePath) .ConfigureAwait(false); } From 12c4abfb5227eb9f94b6d680b492b9a3855d3ba2 Mon Sep 17 00:00:00 2001 From: Tom de Goede Date: Thu, 18 Nov 2021 11:01:30 +0100 Subject: [PATCH 4/5] Perf: Remove glob for exports_files and use static list instead --- .../NugetRepositoryGenerator.cs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs index 722736b..bb9f9fc 100644 --- a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs @@ -181,12 +181,25 @@ public async Task WriteRepository(IReadOnlyList targetFrameworks, string } } + private static readonly HashSet _exportedExtensions = new(StringComparer.OrdinalIgnoreCase) + { + ".exe", + ".ruleset", + ".json", + // only really used for internal redirects. Can we do this differently? + ".dll", + }; + private void WriteBuildFile(NugetRepositoryEntry entry, string id) { + var exports = StringArray(entry.LocalPackageSourceInfo.Package.Files.Where(f => _exportedExtensions.Contains(Path.GetExtension(f))) + .Select(f => $"current/{f}") + .Prepend("contentfiles.txt")); + var content = $@"package(default_visibility = [""//visibility:public""]) load(""@io_bazel_rules_dotnet//dotnet:defs.bzl"", ""core_import_library"") -exports_files([""contentfiles.txt""]) +exports_files({exports}) {CreateTarget(entry)}"; @@ -234,8 +247,6 @@ private string CreateTarget(NugetRepositoryEntry package) IEnumerable Elems() { - yield return $@"exports_files(glob([""current/**"", ""{identity.Version}/**""]))"; - yield return $@"filegroup( name = ""content_files"", srcs = {StringArray(GetContentFiles(package).Select(v => $"{folder}/{v}"))}, From 5d37690678d9b45c4d89f6f75ad43a1e4a160179 Mon Sep 17 00:00:00 2001 From: Tom de Goede Date: Tue, 15 Feb 2022 18:54:48 +0100 Subject: [PATCH 5/5] Quickfix for multi targetting --- .../BazelDotnet.Nuget/NugetRepositoryGenerator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs index bb9f9fc..743e1f3 100644 --- a/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs +++ b/src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs @@ -115,7 +115,7 @@ private async Task ResolveLocalPackages(IReadOnlyList g.RuntimeGraph).Aggregate(RuntimeGraph.Merge); var entryBuilder = new NugetRepositoryEntryBuilder(new ManagedCodeConventions(runtimeGraph), rootProject.Targets); - var localPackages = await dependencyGraphResolver.DownloadPackages(dependencyGraphs).ConfigureAwait(false); + var localPackages = await dependencyGraphResolver.DownloadPackages(dependencyGraphs.Take(2)).ConfigureAwait(false); var entries = localPackages.Select(entryBuilder.ResolveGroups).ToArray(); var (frameworkEntries, frameworkOverrides) = await new FrameworkDependencyResolver(dependencyGraphResolver)