Skip to content
This repository was archived by the owner on Oct 31, 2024. It is now read-only.

Add support for multiple tfms and a packages.lock.json #4

Open
tomdegoede wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Add support for multiple tfms and a packages.lock.json #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9d81edd
Add support for netstandard framework refs
tomdegoede Oct 11, 2021
0287966
Add support for multiple tfms
tomdegoede Oct 15, 2021
7c7c548
Add lockfile support
tomdegoede Nov 16, 2021
12c4abf
Perf: Remove glob for exports_files and use static list instead
tomdegoede Nov 18, 2021
5d37690
Quickfix for multi targetting
tomdegoede Feb 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/FrameworkDependencyResolver.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ private static string GetFrameworkVersion(string targetFramework)
continue;
}

var existingPackageIsEmpty = existingPackage.RefItemGroups.SingleOrDefault()?.Items.Any() != true;
var existingPackageIsEmpty = existingPackage.RefItemGroups.FirstOrDefault()?.Items.Any() != true;
if(existingPackageIsEmpty || packageOverride >= existingPackage.Version)
{
if(frameworkList.TryGetValue(id, out var frameworkItem))
Expand Down Expand Up @@ -121,7 +121,7 @@ private static string GetFrameworkVersion(string targetFramework)

bool OverridesAssemblyVersion(NugetRepositoryEntry package, Version frameworkListVersion)
{
var refItems = package.RefItemGroups.Single();
var refItems = package.RefItemGroups.First();
if(!refItems.Items.Any())
{
return true;
Expand Down
8 changes: 4 additions & 4 deletions src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/LocalPackageExtractor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ internal class LocalPackageExtractor
private readonly SourceCacheContext _cache;
private readonly PackageExtractionContext _context;

public LocalPackageExtractor(ISettings settings, ILogger logger, SourceCacheContext cache)
public LocalPackageExtractor(ISettings settings, NuGetv3LocalRepository v3LocalRepository, ILogger logger, SourceCacheContext cache)
{
_v3LocalRepository = new NuGetv3LocalRepository(SettingsUtility.GetGlobalPackagesFolder(settings), new LocalPackageFileCache(), false);
_v3LocalRepository = v3LocalRepository;
_cache = cache;
_context = new PackageExtractionContext(PackageSaveMode.Defaultv3, XmlDocFileSaveMode.Skip, ClientPolicyContext.GetClientPolicy(settings, logger), logger);
}
Expand All @@ -31,11 +31,11 @@ public async Task<LocalPackageSourceInfo> EnsureLocalPackage(IRemoteDependencyPr
{
if(!_v3LocalRepository.Exists(packageIdentity.Id, packageIdentity.Version))
{
var packageDependency = await provider.GetPackageDownloaderAsync(packageIdentity, _cache, _context.Logger, CancellationToken.None);
var downloader = await provider.GetPackageDownloaderAsync(packageIdentity, _cache, _context.Logger, CancellationToken.None);

var installed = await PackageExtractor.InstallFromSourceAsync(
packageIdentity,
packageDependency,
downloader,
_v3LocalRepository.PathResolver,
_context,
CancellationToken.None,
Expand Down
30 changes: 14 additions & 16 deletions src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryEntryBuilder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,12 @@ namespace Afas.BazelDotnet.Nuget
internal class NugetRepositoryEntryBuilder
{
private readonly ManagedCodeConventions _conventions;
private readonly List<FrameworkRuntimePair> _targets;
private readonly IReadOnlyCollection<FrameworkRuntimePair> _targets;

public NugetRepositoryEntryBuilder(ManagedCodeConventions conventions)
public NugetRepositoryEntryBuilder(ManagedCodeConventions conventions, IReadOnlyCollection<FrameworkRuntimePair> targets)
{
_conventions = conventions;
_targets = new List<FrameworkRuntimePair>();
}

public NugetRepositoryEntryBuilder WithTarget(FrameworkRuntimePair target)
{
_targets.Add(target);
return this;
}

public NugetRepositoryEntryBuilder WithTarget(NuGetFramework target)
{
_targets.Add(new FrameworkRuntimePair(target, runtimeIdentifier: null));
return this;
_targets = targets;
}

public NugetRepositoryEntry ResolveGroups(LocalPackageSourceInfo localPackageSourceInfo)
Expand Down Expand Up @@ -79,9 +67,19 @@ public NugetRepositoryEntry ResolveGroups(LocalPackageSourceInfo localPackageSou
new PatternDefinition("analyzers/{assembly}")
});

// "2.0.0/build/netstandard2.0/ref/netstandard.dll"
var netstandardRefAssemblies = new PatternSet(_conventions.Properties, new []
{
new PatternDefinition("build/{tfm}/ref/{any?}"),
}, new[]
{
new PatternDefinition("build/{tfm}/ref/{assembly}"),
});

AddIfNotNull(entry.RefItemGroups, target.Framework,
collection.FindBestItemGroup(criteria,
_conventions.Patterns.CompileRefAssemblies,
netstandardRefAssemblies,
_conventions.Patterns.CompileLibAssemblies)
?.Items);

Expand Down Expand Up @@ -146,7 +144,7 @@ public NugetRepositoryEntry BuildFrameworkOverride(NugetRepositoryEntry entry, s
newEntry.DependencyGroups.AddRange(entry.DependencyGroups);
if(frameworkOverride != null)
{
newEntry.RefItemGroups.Add(new FrameworkSpecificGroup(_targets.Single().Framework, new []
newEntry.RefItemGroups.Add(new FrameworkSpecificGroup(_targets.First().Framework, new []
{
frameworkOverride,
}));
Expand Down
141 changes: 115 additions & 26 deletions src/Afas.BazelDotnet.Nuget/BazelDotnet.Nuget/NugetRepositoryGenerator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,54 +5,122 @@
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using NuGet.Client;
using NuGet.Commands;
using NuGet.Common;
using NuGet.Configuration;
using NuGet.Credentials;
using NuGet.Frameworks;
using NuGet.LibraryModel;
using NuGet.Packaging;
using NuGet.Packaging.Core;
using NuGet.ProjectModel;
using NuGet.Protocol;
using NuGet.Protocol.Core.Types;
using NuGet.Protocol.Plugins;
using NuGet.RuntimeModel;
using NuGet.Versioning;

namespace Afas.BazelDotnet.Nuget
{
public class NugetRepositoryGenerator
{
private class TargetFrameworkProcessor
{
public string CreateConfigurationTargets(IReadOnlyList<string> targetFrameworks) => $@"
load(""@bazel_skylib//rules:common_settings.bzl"", ""string_flag"")

string_flag(
name = ""framework"",
values = {Indent(StringArray(targetFrameworks))},
build_setting_default = ""{targetFrameworks.First()}""
)

{string.Join("\n", targetFrameworks
.Select(f => $@"config_setting(
name = ""frameworks-{f}"",
flag_values = {{
"":framework"": ""{f}""
}}
)
"))}";

public string CreateSelectStatementWhenApplicable(IReadOnlyList<FrameworkSpecificGroup> groups, Func<string, string> format)
{
var arrays = groups.Select(g => StringArray(g.Items.Select(format))).ToArray();
return arrays.Distinct().Count() switch
{
0 => "[]",
1 => arrays[0],
_ => Select(Indent(Dict(groups.Select(g => $"//:frameworks-{g.TargetFramework.GetShortFolderName()}")
.Zip(arrays)
.Skip(1).Append(("//conditions:default", arrays[0]))
.ToDictionary(t => t.Item1, t => t.Item2))))
};
}

private static string Select(string input) => $@"select({input})";
}

private readonly string _nugetConfig;
private readonly ILookup<string, (string target, string configSetting)> _imports;
private readonly TargetFrameworkProcessor _targetFrameworkProcessor = new();

public NugetRepositoryGenerator(string nugetConfig, ILookup<string, (string target, string configSetting)> imports)
{
_nugetConfig = nugetConfig;
_imports = imports;
}

private async Task<NugetRepositoryEntry[]> ResolveLocalPackages(string targetFramework, string targetRuntime,
IEnumerable<(string package, string version)> packageReferences)
private async Task<PackagesLockFile> TryReadLockFile(string nugetLockFilePath, ILogger logger)
{
if(string.IsNullOrEmpty(nugetLockFilePath))
{
return null;
}

if(!File.Exists(nugetLockFilePath))
{
throw new Exception("Nuget Lock File was specified but not found.");
}

return PackagesLockFileFormat.Parse(await File.ReadAllTextAsync(nugetLockFilePath), logger, nugetLockFilePath);
}

private async Task<NugetRepositoryEntry[]> ResolveLocalPackages(IReadOnlyList<string> targetFrameworks, string targetRuntime,
IEnumerable<(string package, string version)> packageReferences, string nugetLockFilePath)
{
ILogger logger = new ConsoleLogger();

var rootProject = new RootProject(
targetFrameworks.Select(tfm => new FrameworkRuntimePair(NuGetFramework.Parse(tfm), targetRuntime)).ToArray(),
packageReferences.Select(r => new LibraryRange(r.package, VersionRange.Parse(r.version), LibraryDependencyTarget.Package)).ToArray(),
await TryReadLockFile(nugetLockFilePath, logger));

var settings = Settings.LoadSpecificSettings(Path.GetDirectoryName(_nugetConfig), Path.GetFileName(_nugetConfig));
DefaultCredentialServiceUtility.SetupDefaultCredentialService(logger, nonInteractive: true);

// ~/.nuget/packages
using var cache = new SourceCacheContext();

var dependencyGraphResolver = new TransitiveDependencyResolver(settings, logger, cache);
var dependencyGraphResolver = new TransitiveDependencyResolver(settings, logger, cache, rootProject);

foreach(var (package, version) in packageReferences)
var dependencyGraphs = await dependencyGraphResolver.ResolveGraphs();

if(!string.IsNullOrEmpty(nugetLockFilePath) && !rootProject.IsLockFileValid)
{
dependencyGraphResolver.AddPackageReference(package, version);
await File.WriteAllTextAsync(nugetLockFilePath, dependencyGraphResolver.RenderLockFile(dependencyGraphs));
}

var dependencyGraph = await dependencyGraphResolver.ResolveGraph(targetFramework, targetRuntime).ConfigureAwait(false);
var localPackages = await dependencyGraphResolver.DownloadPackages(dependencyGraph).ConfigureAwait(false);

var entryBuilder = new NugetRepositoryEntryBuilder(dependencyGraph.Conventions)
.WithTarget(new FrameworkRuntimePair(NuGetFramework.Parse(targetFramework), targetRuntime));
var runtimeGraph = dependencyGraphs.Select(g => g.RuntimeGraph).Aggregate(RuntimeGraph.Merge);
var entryBuilder = new NugetRepositoryEntryBuilder(new ManagedCodeConventions(runtimeGraph), rootProject.Targets);

var localPackages = await dependencyGraphResolver.DownloadPackages(dependencyGraphs.Take(2)).ConfigureAwait(false);
var entries = localPackages.Select(entryBuilder.ResolveGroups).ToArray();

var (frameworkEntries, frameworkOverrides) = await new FrameworkDependencyResolver(dependencyGraphResolver)
.ResolveFrameworkPackages(entries, targetFramework)
var (frameworkEntries, frameworkOverrides) = await new FrameworkDependencyResolver(dependencyGraphResolver)
// TODO ResolveFrameworkPackages on multiple tfms
.ResolveFrameworkPackages(entries, targetFrameworks.First())
.ConfigureAwait(false);

var overridenEntries = entries.Select(p =>
Expand All @@ -63,9 +131,9 @@ private async Task<NugetRepositoryEntry[]> ResolveLocalPackages(string targetFra
return frameworkEntries.Concat(overridenEntries).ToArray();
}

public async Task WriteRepository(string targetFramework, string targetRuntime, IEnumerable<(string package, string version)> packageReferences)
public async Task WriteRepository(IReadOnlyList<string> targetFrameworks, string targetRuntime, IEnumerable<(string package, string version)> packageReferences, string nugetLockFilePath)
{
var packages = await ResolveLocalPackages(targetFramework, targetRuntime, packageReferences).ConfigureAwait(false);
var packages = await ResolveLocalPackages(targetFrameworks, targetRuntime, packageReferences, nugetLockFilePath).ConfigureAwait(false);

var symlinks = new HashSet<(string link, string target)>();

Expand Down Expand Up @@ -95,6 +163,8 @@ public async Task WriteRepository(string targetFramework, string targetRuntime,
}
}

File.WriteAllText("BUILD", _targetFrameworkProcessor.CreateConfigurationTargets(targetFrameworks));

File.WriteAllText("symlinks_manifest", string.Join("\n", symlinks
.Select(sl => $@"{sl.link} {sl.target}")));

Expand All @@ -111,12 +181,25 @@ public async Task WriteRepository(string targetFramework, string targetRuntime,
}
}

private static readonly HashSet<string> _exportedExtensions = new(StringComparer.OrdinalIgnoreCase)
{
".exe",
".ruleset",
".json",
// only really used for internal redirects. Can we do this differently?
".dll",
};

private void WriteBuildFile(NugetRepositoryEntry entry, string id)
{
var exports = StringArray(entry.LocalPackageSourceInfo.Package.Files.Where(f => _exportedExtensions.Contains(Path.GetExtension(f)))
.Select(f => $"current/{f}")
.Prepend("contentfiles.txt"));

var content = $@"package(default_visibility = [""//visibility:public""])
load(""@io_bazel_rules_dotnet//dotnet:defs.bzl"", ""core_import_library"")

exports_files([""contentfiles.txt""])
exports_files({exports})

{CreateTarget(entry)}";

Expand All @@ -143,7 +226,7 @@ private void WriteBuildFile(IGrouping<string, NugetRepositoryEntry> entryGroup,

private IEnumerable<string> GetContentFiles(NugetRepositoryEntry package)
{
var group = package.ContentFileGroups.SingleOrDefault();
var group = package.ContentFileGroups.FirstOrDefault();
if(group?.Items.Any() == true)
{
// Symlink optimization to link the entire group folder e.g. contentFiles/any/netcoreapp3.1
Expand All @@ -164,8 +247,6 @@ private string CreateTarget(NugetRepositoryEntry package)

IEnumerable<string> Elems()
{
yield return $@"exports_files(glob([""current/**"", ""{identity.Version}/**""]))";

yield return $@"filegroup(
name = ""content_files"",
srcs = {StringArray(GetContentFiles(package).Select(v => $"{folder}/{v}"))},
Expand All @@ -175,9 +256,9 @@ IEnumerable<string> Elems()

if(_imports.Contains(name))
{
var selects = _imports[name].ToDictionary(i => i.configSetting, i => i.target);
var selects = _imports[name].ToDictionary(i => i.configSetting, i => Quote(i.target));
name += "__nuget";
selects["//conditions:default"] = name;
selects["//conditions:default"] = Quote(name);

yield return $@"alias(
name = ""{identity.Id.ToLower()}"",
Expand All @@ -186,7 +267,7 @@ IEnumerable<string> Elems()
}

bool hasDebugDlls = package.DebugRuntimeItemGroups.Count != 0;
var libs = StringArray(package.RuntimeItemGroups.SingleOrDefault()?.Items.Select(v => $"{folder}/{v}"));
var libs = _targetFrameworkProcessor.CreateSelectStatementWhenApplicable(package.RuntimeItemGroups, v => $"{folder}/{v}");

if(hasDebugDlls)
{
Expand All @@ -198,18 +279,19 @@ IEnumerable<string> Elems()
},
)";
libs = Indent($@"select({{
"":compilation_mode_dbg"": {StringArray(package.DebugRuntimeItemGroups.Single().Items.Select(v => $"{folder}/{v}"))},
"":compilation_mode_dbg"": {StringArray(package.DebugRuntimeItemGroups.First().Items.Select(v => $"{folder}/{v}"))},
""//conditions:default"": {libs},
}})");
}

var refs = _targetFrameworkProcessor.CreateSelectStatementWhenApplicable(package.RefItemGroups, v => v.StartsWith("//") ? v : $"{folder}/{v}");

yield return $@"core_import_library(
name = ""{name}"",
libs = {libs},
refs = {StringArray(package.RefItemGroups.SingleOrDefault()?.Items.Select(v => v.StartsWith("//") ? v : $"{folder}/{v}"))},
analyzers = {StringArray(package.AnalyzerItemGroups.SingleOrDefault()?.Items.Select(v => v.StartsWith("//") ? v : $"{folder}/{v}"))},
deps = {StringArray(package.DependencyGroups.SingleOrDefault()?.Packages.Select(p => $"//{p.Id.ToLower()}"))},
refs = {refs},
analyzers = {StringArray(package.AnalyzerItemGroups.FirstOrDefault()?.Items.Select(v => v.StartsWith("//") ? v : $"{folder}/{v}"))},
deps = {StringArray(package.DependencyGroups.FirstOrDefault()?.Packages.Where(ShouldIncludeDep).Select(p => $"//{p.Id.ToLower()}"))},
data = ["":content_files""],
version = ""{identity.Version}"",
)";
Expand All @@ -218,6 +300,11 @@ IEnumerable<string> Elems()
return string.Join("\n\n", Elems());
}

private bool ShouldIncludeDep(PackageDependency package) =>
// Some libraries depend on netstandard. It will not include anything of use
// Should only be referenced explicitly as targetframework
!string.Equals(package.Id, "netstandard.library", StringComparison.OrdinalIgnoreCase);

private static string Indent(string input)
{
var lines = input.Split('\n');
Expand All @@ -232,6 +319,8 @@ private static string StringArray(IEnumerable<string> items) => items?.Any() !=
{string.Join(",\n", items.Select(i => $@" ""{i}"""))}
]");

private static string Quote(string input) => $@"""{input}""";

private static string Dict(IReadOnlyDictionary<string, string> items)
{
var s = new StringBuilder();
Expand All @@ -240,7 +329,7 @@ private static string Dict(IReadOnlyDictionary<string, string> items)

foreach(var (key, value) in items)
{
s.Append($@" ""{key}"": ""{value}"",
s.Append($@" ""{key}"": {value},
");
}

Expand Down
Loading