diff --git a/.vault-config/maestroint.yaml b/.vault-config/maestroint.yaml deleted file mode 100644 index 2c2a222d12..0000000000 --- a/.vault-config/maestroint.yaml +++ /dev/null @@ -1,26 +0,0 @@ -storageLocation: - type: azure-key-vault - parameters: - subscription: cab65fc3-d077-467d-931f-3932eabf36d3 - name: maestroint - -references: - helixkv: - type: azure-key-vault - parameters: - subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1 - name: helixkv - -keys: - data-protection-encryption-key: - type: RSA - size: 2048 - -importSecretsFrom: shared/maestro-secrets.yaml - -secrets: - # PCS is using the same auth as Maestro so tokens created for Maestro are valid for PCS - product-construction-service-int-token: - type: maestro-access-token - parameters: - environment: https://maestro.int-dot.net/ diff --git a/.vault-config/maestrolocal.yaml b/.vault-config/maestrolocal.yaml deleted file mode 100644 index f87f74e309..0000000000 --- a/.vault-config/maestrolocal.yaml +++ /dev/null @@ -1,20 +0,0 @@ -storageLocation: - type: azure-key-vault - parameters: - name: maestrolocal - subscription: cab65fc3-d077-467d-931f-3932eabf36d3 - -references: - helixkv: - type: azure-key-vault - parameters: - subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1 - name: helixkv - -secrets: - github: - type: github-app-secret - parameters: - hasPrivateKey: true - hasWebhookSecret: false - hasOAuthSecret: true diff --git a/.vault-config/maestroprod.yaml b/.vault-config/maestroprod.yaml deleted file mode 100644 index a71c1895b2..0000000000 --- a/.vault-config/maestroprod.yaml +++ /dev/null @@ -1,27 +0,0 @@ -storageLocation: - type: azure-key-vault - parameters: - subscription: 68672ab8-de0c-40f1-8d1b-ffb20bd62c0f - name: maestroprod - -references: - helixkv: - type: azure-key-vault - parameters: - subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1 - name: helixkv - - engkeyvault: - type: azure-key-vault - parameters: - subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1 - name: engkeyvault - -keys: - data-protection-encryption-key: - type: RSA - size: 2048 - -importSecretsFrom: shared/maestro-secrets.yaml - -secrets: {} diff --git a/.vault-config/product-construction-dev.yaml b/.vault-config/product-construction-dev.yaml index 6521a799a5..baa45e2e58 100644 --- a/.vault-config/product-construction-dev.yaml +++ b/.vault-config/product-construction-dev.yaml @@ -4,19 +4,6 @@ storageLocation: subscription: e6b5f9f5-0ca4-4351-879b-014d78400ec2 name: ProductConstructionDev -references: - helixkv: - type: azure-key-vault - parameters: - subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1 - name: helixkv - - engkeyvault: - type: azure-key-vault - parameters: - subscription: a4fc5514-21a9-4296-bfaf-5c7ee7fa35d1 - name: engkeyvault - secrets: github: diff --git a/.vault-config/shared/maestro-secrets.yaml b/.vault-config/shared/maestro-secrets.yaml deleted file mode 100644 index 38dbe367aa..0000000000 --- a/.vault-config/shared/maestro-secrets.yaml +++ /dev/null @@ -1,6 +0,0 @@ -github: - type: github-app-secret - parameters: - hasPrivateKey: true - hasWebhookSecret: true - hasOAuthSecret: true diff --git a/docs/DevGuide.md b/docs/DevGuide.md index df1906cdd0..21e19a3e21 100644 --- a/docs/DevGuide.md +++ b/docs/DevGuide.md @@ -4,6 +4,11 @@ - Be sure to install the `Azure Development => .NET Aspire SDK (Preview)` optional workload in the VS installer - Be sure to install the `ASP.NET and web development` => `.NET 8.0/9.0 WebAssembly Build Tools` 1. Install Docker Desktop: https://www.docker.com/products/docker-desktop +1. Configure git to support long paths: + ```ps1 + git config --system core.longpaths true # you will need elevated shell for this one + git config --global core.longpaths true + ``` 1. Install SQL Server Express: https://www.microsoft.com/en-us/sql-server/sql-server-downloads 1. Install Entity Framework Core CLI by running `dotnet tool install --global dotnet-ef` 1. Build the `src\Maestro\Maestro.Data\Maestro.Data.csproj` project (either from console or from IDE) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a2b8c7b785..faf2a31062 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -59,29 +59,29 @@ - + https://github.com/dotnet/arcade - c255aae7f2b128fa20a4441f0e192c3c53561621 + 4db725213dccb0d1102427bce1c39ba3117da7f7 - + https://github.com/dotnet/arcade - c255aae7f2b128fa20a4441f0e192c3c53561621 + 4db725213dccb0d1102427bce1c39ba3117da7f7 - + https://github.com/dotnet/arcade - c255aae7f2b128fa20a4441f0e192c3c53561621 + 4db725213dccb0d1102427bce1c39ba3117da7f7 - + https://github.com/dotnet/arcade - c255aae7f2b128fa20a4441f0e192c3c53561621 + 4db725213dccb0d1102427bce1c39ba3117da7f7 - + https://github.com/dotnet/arcade - c255aae7f2b128fa20a4441f0e192c3c53561621 + 4db725213dccb0d1102427bce1c39ba3117da7f7 - + https://github.com/dotnet/arcade - c255aae7f2b128fa20a4441f0e192c3c53561621 + 4db725213dccb0d1102427bce1c39ba3117da7f7 https://github.com/dotnet/dnceng diff --git a/eng/Versions.props b/eng/Versions.props index 527eb110be..05d3b935b2 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -9,11 +9,11 @@ true 1.0.0-preview.1 - 8.0.0-beta.25060.1 - 8.0.0-beta.25060.1 - 8.0.0-beta.25060.1 - 8.0.0-beta.25060.1 - 8.0.0-beta.25060.1 + 8.0.0-beta.25066.6 + 8.0.0-beta.25066.6 + 8.0.0-beta.25066.6 + 8.0.0-beta.25066.6 + 8.0.0-beta.25066.6 17.4.1 1.1.0-beta.25053.1 1.1.0-beta.25053.1 diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 73828dd30d..4f0546dce1 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.12.0" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 60352ede19..a00577ed17 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -384,8 +384,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.1-2 - $defaultXCopyMSBuildVersion = '17.8.1-2' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.12.0 + $defaultXCopyMSBuildVersion = '17.12.0' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { diff --git a/global.json b/global.json index 89a412a3ca..4c3f1ed9a5 100644 --- a/global.json +++ b/global.json @@ -15,6 +15,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25060.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25066.6" } } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs index ec4724e993..dcf516d6a6 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs @@ -480,7 +480,8 @@ await client.UpdatePullRequestAsync( catch (Exception ex) when ( ex.Message.StartsWith("The pull request needs a minimum number of approvals") || ex.Message == "Proof of presence is required" || - ex.Message == "Failure while attempting to queue Build.") + ex.Message == "Failure while attempting to queue Build." || + ex.Message.Contains("Please re-approve the most recent pull request iteration")) { throw new PullRequestNotMergeableException(ex.Message); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Conflicts/BackFlowConflictResolver.cs b/src/Microsoft.DotNet.Darc/DarcLib/Conflicts/BackFlowConflictResolver.cs new file mode 100644 index 0000000000..9472afa281 --- /dev/null +++ b/src/Microsoft.DotNet.Darc/DarcLib/Conflicts/BackFlowConflictResolver.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.Extensions.Logging; + +#nullable enable +namespace Microsoft.DotNet.DarcLib.Conflicts; + +public interface IBackFlowConflictResolver +{ + Task TryMergingRepoBranch( + ILocalGitRepo repo, + string baseBranch, + string targetBranch); +} + +/// +/// This class is responsible for resolving well-known conflicts that can occur during a backflow operation. +/// The conflicts can happen when backward and forward flow PRs get merged out of order. +/// This can be shown on the following schema (the order of events is numbered): +/// +/// repo VMR +/// O────────────────────►O +/// │ 2. │ +/// 1.O────────────────O │ +/// │ 4. │ │ +/// │ O───────────┼────O 3. +/// │ │ │ │ +/// │ │ │ │ +/// 6.O◄───┘ └───►O 5. +/// │ 7. │ +/// │ O───────────────| +/// 8.O◄────┘ │ +/// │ │ +/// +/// The conflict arises in step 8. and is caused by the fact that: +/// - When the backflow PR branch is being opened in 7., the last sync (from the point of view of 5.) is from 1. +/// - This means that the PR branch will be based on 1. (the real PR branch will be a commit on top of 1.) +/// - This means that when 6. merged, Version.Details.xml got updated with the SHA of the 3. +/// - So the Source tag in Version.Details.xml in 6. contains the SHA of 3. +/// - The backflow PR branch contains the SHA of 5. +/// - So the Version.Details.xml file conflicts on the SHA (3. vs 5.) +/// - There's also a similar conflict in the package versions that got updated in those commits. +/// - However, if only the version files are in conflict, we can try merging 6. into 7. and resolve the conflict. +/// - This is because basically we know we want to set the version files to point at 5. +/// +public class BackFlowConflictResolver : CodeFlowConflictResolver, IBackFlowConflictResolver +{ + private readonly ILogger _logger; + + public BackFlowConflictResolver(ILogger logger) + : base(logger) + { + _logger = logger; + } + + public async Task TryMergingRepoBranch( + ILocalGitRepo repo, + string targetBranch, + string branchToMerge) + { + return await TryMergingBranch(repo, targetBranch, branchToMerge); + } + + protected override async Task TryResolvingConflicts(ILocalGitRepo repo, IEnumerable conflictedFiles) + { + foreach (var filePath in conflictedFiles) + { + // Known conflict in eng/Version.Details.xml + if (string.Equals(filePath, VersionFiles.VersionDetailsXml, StringComparison.InvariantCultureIgnoreCase)) + { + await Task.CompletedTask; + return false; + + // TODO https://github.com/dotnet/arcade-services/issues/4196: Resolve conflicts in eng/Version.Details.xml + // return true; + } + + // Known conflict in eng/Versions.props + if (string.Equals(filePath, VersionFiles.VersionProps, StringComparison.InvariantCultureIgnoreCase)) + { + await Task.CompletedTask; + return false; + + // TODO https://github.com/dotnet/arcade-services/issues/4196: Resolve conflicts in eng/Version.Details.xml + // return true; + } + + _logger.LogInformation("Unable to resolve conflicts in {file}", filePath); + return false; + } + + return true; + } +} diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Conflicts/CodeFlowConflictResolver.cs b/src/Microsoft.DotNet.Darc/DarcLib/Conflicts/CodeFlowConflictResolver.cs new file mode 100644 index 0000000000..a71854320f --- /dev/null +++ b/src/Microsoft.DotNet.Darc/DarcLib/Conflicts/CodeFlowConflictResolver.cs @@ -0,0 +1,76 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.Extensions.Logging; + +#nullable enable +namespace Microsoft.DotNet.DarcLib.Conflicts; + +/// +/// This class is responsible for resolving well-known conflicts that can occur during codeflow operations. +/// The conflicts usually happen when backward a forward flow PRs get merged out of order. +/// +public abstract class CodeFlowConflictResolver +{ + private readonly ILogger _logger; + + public CodeFlowConflictResolver(ILogger logger) + { + _logger = logger; + } + + protected async Task TryMergingBranch( + ILocalGitRepo repo, + string targetBranch, + string branchToMerge) + { + _logger.LogInformation("Trying to merge target branch {targetBranch} into {baseBranch}", branchToMerge, targetBranch); + + await repo.CheckoutAsync(targetBranch); + var result = await repo.RunGitCommandAsync(["merge", "--no-commit", "--no-ff", branchToMerge]); + if (result.Succeeded) + { + _logger.LogInformation("Successfully merged the branch {targetBranch} into {headBranch} in {repoPath}", + branchToMerge, + targetBranch, + repo.Path); + await repo.CommitAsync($"Merging {branchToMerge} into {targetBranch}", allowEmpty: true); + return true; + } + + result = await repo.RunGitCommandAsync(["diff", "--name-only", "--diff-filter=U", "--relative"]); + if (!result.Succeeded) + { + _logger.LogInformation("Failed to merge the branch {targetBranch} into {headBranch} in {repoPath}", + branchToMerge, + targetBranch, + repo.Path); + result = await repo.RunGitCommandAsync(["merge", "--abort"]); + return false; + } + + var conflictedFiles = result.StandardOutput + .Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries) + .Select(line => new UnixPath(line.Trim())); + + if (!await TryResolvingConflicts(repo, conflictedFiles)) + { + result = await repo.RunGitCommandAsync(["merge", "--abort"]); + return false; + } + + _logger.LogInformation("Successfully resolved version file conflicts between branches {targetBranch} and {headBranch} in {repoPath}", + branchToMerge, + targetBranch, + repo.Path); + await repo.CommitAsync($"Merge branch {branchToMerge} into {targetBranch}", allowEmpty: false); + return true; + } + + protected abstract Task TryResolvingConflicts(ILocalGitRepo repo, IEnumerable conflictedFiles); +} diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Conflicts/ForwardFlowConflictResolver.cs b/src/Microsoft.DotNet.Darc/DarcLib/Conflicts/ForwardFlowConflictResolver.cs new file mode 100644 index 0000000000..fa2a14194d --- /dev/null +++ b/src/Microsoft.DotNet.Darc/DarcLib/Conflicts/ForwardFlowConflictResolver.cs @@ -0,0 +1,145 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; +using Microsoft.Extensions.Logging; + +#nullable enable +namespace Microsoft.DotNet.DarcLib.Conflicts; + +public interface IForwardFlowConflictResolver +{ + Task TryMergingBranch( + ILocalGitRepo vmr, + string mappingName, + string baseBranch, + string targetBranch); +} + +/// +/// This class is responsible for resolving well-known conflicts that can occur during a forward flow operation. +/// The conflicts can happen when backward a forward flow PRs get merged out of order. +/// This can be shown on the following schema (the order of events is numbered): +/// +/// repo VMR +/// O────────────────────►O +/// │ 2. │ 1. +/// │ O◄────────────────O- - ┐ +/// │ │ 4. │ +/// 3.O───┼────────────►O │ │ +/// │ │ │ │ +/// │ ┌─┘ │ │ │ +/// │ │ │ │ +/// 5.O◄┘ └──►O 6. │ +/// │ 7. │ O (actual branch for 7. is based on top of 1.) +/// |────────────────►O │ +/// │ └──►O 8. +/// │ │ +/// +/// The conflict arises in step 8. and is caused by the fact that: +/// - When the forward flow PR branch is being opened in 7., the last sync (from the point of view of 5.) is from 1. +/// - This means that the PR branch will be based on 1. (the real PR branch is the "actual 7.") +/// - This means that when 6. merged, VMR's source-manifest.json got updated with the SHA of the 3. +/// - So the source-manifest in 6. contains the SHA of 3. +/// - The forward flow PR branch contains the SHA of 5. +/// - So the source-manifest file conflicts on the SHA (3. vs 5.) +/// - There's also a similar conflict in the git-info files. +/// - However, if only the version files are in conflict, we can try merging 6. into 7. and resolve the conflict. +/// - This is because basically we know we want to set the version files to point at 5. +/// +public class ForwardFlowConflictResolver : CodeFlowConflictResolver, IForwardFlowConflictResolver +{ + private readonly IVmrInfo _vmrInfo; + private readonly ISourceManifest _sourceManifest; + private readonly IFileSystem _fileSystem; + private readonly ILogger _logger; + + public ForwardFlowConflictResolver( + IVmrInfo vmrInfo, + ISourceManifest sourceManifest, + IFileSystem fileSystem, + ILogger logger) + : base(logger) + { + _vmrInfo = vmrInfo; + _sourceManifest = sourceManifest; + _fileSystem = fileSystem; + _logger = logger; + } + + private string? _mappingName; + + public async Task TryMergingBranch( + ILocalGitRepo vmr, + string mappingName, + string targetBranch, + string branchToMerge) + { + _mappingName = mappingName; + return await TryMergingBranch(vmr, targetBranch, branchToMerge); + } + + protected override async Task TryResolvingConflicts(ILocalGitRepo repo, IEnumerable conflictedFiles) + { + var gitInfoFile = $"{VmrInfo.GitInfoSourcesDir}/{_mappingName}.props"; + foreach (var filePath in conflictedFiles) + { + // Known conflict in source-manifest.json + if (string.Equals(filePath, VmrInfo.DefaultRelativeSourceManifestPath, StringComparison.OrdinalIgnoreCase)) + { + await TryResolvingSourceManifestConflict(repo, _mappingName!); + continue; + } + + // Known conflict in a git-info props file - we just use our version as we expect it to be newer + // TODO https://github.com/dotnet/arcade-services/issues/3378: For batched subscriptions, we need to handle all git-info files + if (string.Equals(filePath, gitInfoFile, StringComparison.OrdinalIgnoreCase)) + { + _logger.LogInformation("Auto-resolving conflict in {file}", gitInfoFile); + await repo.RunGitCommandAsync(["checkout", "--ours", filePath]); + await repo.StageAsync([filePath]); + continue; + } + + _logger.LogInformation("Unable to resolve conflicts in {file}", filePath); + return false; + } + + return true; + } + + // TODO https://github.com/dotnet/arcade-services/issues/3378: This won't work for batched subscriptions + private async Task TryResolvingSourceManifestConflict(ILocalGitRepo vmr, string mappingName) + { + _logger.LogInformation("Auto-resolving conflict in {file}", VmrInfo.DefaultRelativeSourceManifestPath); + + // We load the source manifest from the target branch and replace the current mapping (and its submodules) with our branches' information + var result = await vmr.RunGitCommandAsync(["show", "MERGE_HEAD:" + VmrInfo.DefaultRelativeSourceManifestPath]); + + var theirSourceManifest = SourceManifest.FromJson(result.StandardOutput); + var ourSourceManifest = _sourceManifest; + var updatedMapping = ourSourceManifest.Repositories.First(r => r.Path == mappingName); + + theirSourceManifest.UpdateVersion(mappingName, updatedMapping.RemoteUri, updatedMapping.CommitSha, updatedMapping.PackageVersion, updatedMapping.BarId); + + foreach (var submodule in theirSourceManifest.Submodules.Where(s => s.Path.StartsWith(mappingName + "/"))) + { + theirSourceManifest.RemoveSubmodule(submodule); + } + + foreach (var submodule in _sourceManifest.Submodules.Where(s => s.Path.StartsWith(mappingName + "/"))) + { + theirSourceManifest.UpdateSubmodule(submodule); + } + + _fileSystem.WriteToFile(_vmrInfo.SourceManifestPath, theirSourceManifest.ToJson()); + _sourceManifest.Refresh(_vmrInfo.SourceManifestPath); + await vmr.StageAsync([_vmrInfo.SourceManifestPath]); + } +} diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/DependencyFileManager.cs b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/DependencyFileManager.cs index 20f3524c11..3dfff2bf36 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/DependencyFileManager.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/DependencyFileManager.cs @@ -11,6 +11,7 @@ using System.Xml; using Microsoft.DotNet.DarcLib.Models; using Microsoft.DotNet.DarcLib.Models.Darc; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; using NuGet.Versioning; @@ -95,24 +96,36 @@ public async Task ReadVersionPropsAsync(string repoUri, string bran return await ReadXmlFileAsync(VersionFiles.VersionProps, repoUri, branch); } - public async Task ReadGlobalJsonAsync(string repoUri, string branch) + public async Task ReadGlobalJsonAsync(string repoUri, string branch, bool repoIsVmr) { - _logger.LogInformation( - $"Reading '{VersionFiles.GlobalJson}' in repo '{repoUri}' and branch '{branch}'..."); + var path = repoIsVmr ? + VmrInfo.ArcadeRepoDir / VersionFiles.GlobalJson : + VersionFiles.GlobalJson; + + _logger.LogInformation("Reading '{filePath}' in repo '{repoUri}' and branch '{branch}'...", + path, + repoUri, + branch); - var fileContent = await GetGitClient(repoUri).GetFileContentsAsync(VersionFiles.GlobalJson, repoUri, branch); + var fileContent = await GetGitClient(repoUri).GetFileContentsAsync(path, repoUri, branch); return JObject.Parse(fileContent); } - public async Task ReadDotNetToolsConfigJsonAsync(string repoUri, string branch) + public async Task ReadDotNetToolsConfigJsonAsync(string repoUri, string branch, bool repoIsVmr) { - _logger.LogInformation( - $"Reading '{VersionFiles.DotnetToolsConfigJson}' in repo '{repoUri}' and branch '{branch}'..."); + var path = repoIsVmr ? + VmrInfo.ArcadeRepoDir / VersionFiles.DotnetToolsConfigJson : + VersionFiles.DotnetToolsConfigJson; + + _logger.LogInformation("Reading '{filePath}' in repo '{repoUri}' and branch '{branch}'...", + path, + repoUri, + branch); try { - var fileContent = await GetGitClient(repoUri).GetFileContentsAsync(VersionFiles.DotnetToolsConfigJson, repoUri, branch); + var fileContent = await GetGitClient(repoUri).GetFileContentsAsync(path, repoUri, branch); return JObject.Parse(fileContent); } catch (DependencyFileNotFoundException) @@ -128,9 +141,9 @@ public async Task ReadDotNetToolsConfigJsonAsync(string repoUri, string /// /// repo to get the version from /// commit sha to query - public async Task ReadToolsDotnetVersionAsync(string repoUri, string commit) + public async Task ReadToolsDotnetVersionAsync(string repoUri, string commit, bool repoIsVmr) { - JObject globalJson = await ReadGlobalJsonAsync(repoUri, commit); + JObject globalJson = await ReadGlobalJsonAsync(repoUri, commit, repoIsVmr); JToken dotnet = globalJson.SelectToken("tools.dotnet", true); _logger.LogInformation("Reading dotnet version from global.json succeeded!"); @@ -171,6 +184,8 @@ public async Task AddDependencyAsync( string repoUri, string branch) { + // The Add Dependency operation doesn't support adding dependencies to VMR src/... folders + bool repoIsVmr = false; var versionDetails = await ParseVersionDetailsXmlAsync(repoUri, branch); var existingDependencies = versionDetails.Dependencies; if (existingDependencies.Any(dep => dep.Name.Equals(dependency.Name, StringComparison.OrdinalIgnoreCase))) @@ -186,7 +201,7 @@ public async Task AddDependencyAsync( throw new Exception($"Dependency '{dependency.Name}' has no parent mapping defined."); } - await AddDependencyToGlobalJson(repoUri, branch, parent, dependency); + await AddDependencyToGlobalJson(repoUri, branch, parent, dependency, repoIsVmr); } else { @@ -289,10 +304,13 @@ public async Task UpdateDependencyFiles( IEnumerable oldDependencies, SemanticVersion incomingDotNetSdkVersion) { + // When updating version files, we always want to look in the base folder, even when we're updating it in the VMR + // src/arcade version files only get updated during arcade forward flows + bool repoIsVmr = false; XmlDocument versionDetails = await ReadVersionDetailsXmlAsync(repoUri, branch); XmlDocument versionProps = await ReadVersionPropsAsync(repoUri, branch); - JObject globalJson = await ReadGlobalJsonAsync(repoUri, branch); - JObject toolsConfigurationJson = await ReadDotNetToolsConfigJsonAsync(repoUri, branch); + JObject globalJson = await ReadGlobalJsonAsync(repoUri, branch, repoIsVmr); + JObject toolsConfigurationJson = await ReadDotNetToolsConfigJsonAsync(repoUri, branch, repoIsVmr); XmlDocument nugetConfig = await ReadNugetConfigAsync(repoUri, branch); foreach (DependencyDetail itemToUpdate in itemsToUpdate) @@ -876,10 +894,11 @@ private async Task AddDependencyToGlobalJson( string repoUri, string branch, string parentField, - DependencyDetail dependency) + DependencyDetail dependency, + bool repoIsVmr = false) { JToken versionProperty = new JProperty(dependency.Name, dependency.Version); - JObject globalJson = await ReadGlobalJsonAsync(repoUri, branch); + JObject globalJson = await ReadGlobalJsonAsync(repoUri, branch, repoIsVmr); JToken parent = globalJson[parentField]; if (parent != null) @@ -891,7 +910,8 @@ private async Task AddDependencyToGlobalJson( globalJson.Add(new JProperty(parentField, new JObject(versionProperty))); } - var file = new GitFile(VersionFiles.GlobalJson, globalJson); + var globalJsonPath = repoIsVmr ? VmrInfo.ArcadeRepoDir / VersionFiles.GlobalJson: VersionFiles.GlobalJson; + var file = new GitFile(globalJsonPath, globalJson); await GetGitClient(repoUri).CommitFilesAsync( [file], repoUri, @@ -1049,6 +1069,8 @@ public async Task Verify(string repo, string branch) Task versionProps; Task globalJson; Task dotnetToolsJson; + // This operation doesn't support VMR verification + bool repoIsVmr = false; try { @@ -1072,7 +1094,7 @@ public async Task Verify(string repo, string branch) try { - globalJson = ReadGlobalJsonAsync(repo, branch); + globalJson = ReadGlobalJsonAsync(repo, branch, repoIsVmr); } catch (Exception e) { @@ -1082,7 +1104,7 @@ public async Task Verify(string repo, string branch) try { - dotnetToolsJson = ReadDotNetToolsConfigJsonAsync(repo, branch); + dotnetToolsJson = ReadDotNetToolsConfigJsonAsync(repo, branch, repoIsVmr); } catch (Exception e) { diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/IDependencyFileManager.cs b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/IDependencyFileManager.cs index c12de1b6f5..cdf97e404b 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/IDependencyFileManager.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/IDependencyFileManager.cs @@ -27,16 +27,16 @@ public interface IDependencyFileManager Task ParseVersionDetailsXmlAsync(string repoUri, string branch, bool includePinned = true); - Task ReadDotNetToolsConfigJsonAsync(string repoUri, string branch); + Task ReadDotNetToolsConfigJsonAsync(string repoUri, string branch, bool repoIsVmr); /// /// Get the tools.dotnet section of the global.json from a target repo URI /// /// repo to get the version from /// commit sha to query - Task ReadToolsDotnetVersionAsync(string repoUri, string commit); + Task ReadToolsDotnetVersionAsync(string repoUri, string commit, bool repoIsVmr); - Task ReadGlobalJsonAsync(string repoUri, string branch); + Task ReadGlobalJsonAsync(string repoUri, string branch, bool repoIsVmr); Task ReadNugetConfigAsync(string repoUri, string branch); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/LocalPath.cs b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/LocalPath.cs index 357965d1f1..b0cc507d34 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/LocalPath.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/LocalPath.cs @@ -63,9 +63,34 @@ protected string Combine(string left, string right) }; } - public override bool Equals(object? obj) => Path.Equals((obj as LocalPath)?.Path ?? obj as string); + public override bool Equals(object? obj) + { + if (obj is string str) + { + return Path.Equals(str); + } + + if (obj is LocalPath localPath) + { + return NormalizePath(Path).Equals(NormalizePath(localPath.Path)); + } + + return false; + } public override int GetHashCode() => Path.GetHashCode(); + + public static bool operator ==(LocalPath? left, LocalPath? right) + { + if (left is null) + { + return right is null; + } + + return left.Equals(right); + } + + public static bool operator !=(LocalPath? left, LocalPath? right) => !(left == right); } /// @@ -93,6 +118,18 @@ private NativePath(string path, bool normalize) : base(path, System.IO.Path.Dire protected override string NormalizePath(string s) => System.IO.Path.DirectorySeparatorChar == '/' ? s.Replace('\\', '/') : s.Replace('/', '\\'); + + public override bool Equals(object? obj) + { + if (obj is NativePath nativePath) + { + return Path.Equals(nativePath.Path); + } + + return base.Equals(obj); + } + + public override int GetHashCode() => Path.GetHashCode(); } /// @@ -116,6 +153,18 @@ private UnixPath(string path, bool normalize) : base(path, '/', normalize) protected override LocalPath CreateMergedPath(string path) => new UnixPath(path, false); protected override string NormalizePath(string s) => s.Replace('\\', '/'); + + public override bool Equals(object? obj) + { + if (obj is UnixPath nativePath) + { + return Path.Equals(nativePath.Path); + } + + return base.Equals(obj); + } + + public override int GetHashCode() => Path.GetHashCode(); } /// @@ -137,4 +186,16 @@ private WindowsPath(string path, bool normalize) : base(path, '\\', normalize) protected override LocalPath CreateMergedPath(string path) => new WindowsPath(path, false); protected override string NormalizePath(string s) => s.Replace('/', '\\'); + + public override bool Equals(object? obj) + { + if (obj is WindowsPath nativePath) + { + return Path.Equals(nativePath.Path); + } + + return base.Equals(obj); + } + + public override int GetHashCode() => Path.GetHashCode(); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/ILocalGitRepo.cs b/src/Microsoft.DotNet.Darc/DarcLib/ILocalGitRepo.cs index 2edee62dba..a2d0759917 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/ILocalGitRepo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/ILocalGitRepo.cs @@ -185,4 +185,9 @@ Task CommitAsync( /// Where to add the new argument into /// Where to add the new variables into public void AddGitAuthHeader(IList args, IDictionary envVars, string repoUri); + + /// + /// Runs an arbitrary git command in the repo. + /// + Task RunGitCommandAsync(string[] args, CancellationToken cancellationToken = default); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs index 7ae30fe800..e4253d314b 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs @@ -104,7 +104,7 @@ public interface IRemote /// URI of repo containing script files. /// Common to get script files at. /// Script files. - Task> GetCommonScriptFilesAsync(string repoUri, string commit); + Task> GetCommonScriptFilesAsync(string repoUri, string commit, bool repoIsVmr = false); /// /// Create a new branch in the specified repository. diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Local.cs b/src/Microsoft.DotNet.Darc/DarcLib/Local.cs index 69046f9cd8..3fea72ee0e 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Local.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Local.cs @@ -70,11 +70,21 @@ public async Task UpdateDependenciesAsync(List dependencies, I // If we are updating the arcade sdk we need to update the eng/common files as well DependencyDetail arcadeItem = dependencies.GetArcadeUpdate(); SemanticVersion targetDotNetVersion = null; + bool repoIsVmr = true; if (arcadeItem != null) { var fileManager = new DependencyFileManager(gitRepoFactory, _versionDetailsParser, _logger); - targetDotNetVersion = await fileManager.ReadToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit); + try + { + targetDotNetVersion = await fileManager.ReadToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit, repoIsVmr); + } + catch (DependencyFileNotFoundException) + { + // global.json not found in src/arcade meaning that repo is not the VMR + repoIsVmr = false; + targetDotNetVersion = await fileManager.ReadToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit, repoIsVmr); + } } var fileContainer = await _fileManager.UpdateDependencyFiles(dependencies, sourceDependency: null, _repoRootDir.Value, null, oldDependencies, targetDotNetVersion); @@ -85,7 +95,7 @@ public async Task UpdateDependenciesAsync(List dependencies, I try { IRemote arcadeRemote = await remoteFactory.CreateRemoteAsync(arcadeItem.RepoUri); - List engCommonFiles = await arcadeRemote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit); + List engCommonFiles = await arcadeRemote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit, repoIsVmr); filesToUpdate.AddRange(engCommonFiles); List localEngCommonFiles = GetFilesAtRelativeRepoPathAsync(Constants.CommonScriptFilesPath); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/LocalGitClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/LocalGitClient.cs index 661bf950d4..291ec5eec2 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/LocalGitClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/LocalGitClient.cs @@ -411,10 +411,11 @@ public async Task GetStagedFiles(string repoPath) public async Task GetFileFromGitAsync(string repoPath, string relativeFilePath, string revision = "HEAD", string? outputPath = null) { + // git show doesn't work with windows paths \\, so replace it with a / var args = new List { "show", - $"{revision}:{relativeFilePath.TrimStart('/')}" + $"{revision}:{relativeFilePath.Replace("\\", "/").TrimStart('/')}" }; if (outputPath != null) diff --git a/src/Microsoft.DotNet.Darc/DarcLib/LocalGitRepo.cs b/src/Microsoft.DotNet.Darc/DarcLib/LocalGitRepo.cs index 8ff1f58666..a4e0ccaf49 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/LocalGitRepo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/LocalGitRepo.cs @@ -89,6 +89,9 @@ public async Task SetConfigValue(string setting, string value) public async Task ResetWorkingTree(UnixPath? relativePath = null) => await _localGitClient.ResetWorkingTree(new NativePath(Path), relativePath); + public async Task RunGitCommandAsync(string[] args, CancellationToken cancellationToken = default) + => await _localGitClient.RunGitCommandAsync(Path, args, cancellationToken); + public async Task StageAsync(IEnumerable pathsToStage, CancellationToken cancellationToken = default) => await _localGitClient.StageAsync(Path, pathsToStage, cancellationToken); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/ManifestRecord.cs b/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/ManifestRecord.cs index b29120f490..bc8bc0a425 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/ManifestRecord.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/ManifestRecord.cs @@ -56,6 +56,7 @@ public string GetPublicUrl() public interface IVersionedSourceComponent : ISourceComponent { string? PackageVersion { get; } + public int? BarId { get; } } /// diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/SourceManifest.cs b/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/SourceManifest.cs index 4e140e26f4..55cd195174 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/SourceManifest.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/SourceManifest.cs @@ -19,8 +19,8 @@ public interface ISourceManifest string ToJson(); void RemoveRepository(string repository); - void RemoveSubmodule(SubmoduleRecord submodule); - void UpdateSubmodule(SubmoduleRecord submodule); + void RemoveSubmodule(ISourceComponent submodule); + void UpdateSubmodule(ISourceComponent submodule); void UpdateVersion(string repository, string uri, string sha, string? packageVersion, int? barId); VmrDependencyVersion? GetVersion(string repository); bool TryGetRepoVersion(string mappingName, [NotNullWhen(true)] out ISourceComponent? mapping); @@ -81,7 +81,7 @@ public void RemoveRepository(string repository) _submodules.RemoveWhere(s => s.Path.StartsWith(repository + "/")); } - public void RemoveSubmodule(SubmoduleRecord submodule) + public void RemoveSubmodule(ISourceComponent submodule) { var repo = _submodules.FirstOrDefault(r => r.Path == submodule.Path); if (repo != null) @@ -90,7 +90,7 @@ public void RemoveSubmodule(SubmoduleRecord submodule) } } - public void UpdateSubmodule(SubmoduleRecord submodule) + public void UpdateSubmodule(ISourceComponent submodule) { var repo = _submodules.FirstOrDefault(r => r.Path == submodule.Path); if (repo != null) @@ -100,7 +100,7 @@ public void UpdateSubmodule(SubmoduleRecord submodule) } else { - _submodules.Add(submodule); + _submodules.Add(new SubmoduleRecord(submodule.Path, submodule.RemoteUri, submodule.CommitSha)); } } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs index 4bae9ad589..8501a546d6 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs @@ -11,6 +11,7 @@ using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models; using Microsoft.DotNet.DarcLib.Models.Darc; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.Extensions.Logging; using NuGet.Versioning; @@ -188,11 +189,22 @@ public async Task> CommitUpdatesAsync( SemanticVersion targetDotNetVersion = null; var mayNeedArcadeUpdate = arcadeItem != null && repoUri != arcadeItem.RepoUri; + // If we find version files in src/arcade, we know we're working with a VMR + bool sourceRepoIsVmr = true; if (mayNeedArcadeUpdate) { IDependencyFileManager arcadeFileManager = await remoteFactory.CreateDependencyFileManagerAsync(arcadeItem.RepoUri); - targetDotNetVersion = await arcadeFileManager.ReadToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit); + try + { + targetDotNetVersion = await arcadeFileManager.ReadToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit, sourceRepoIsVmr); + } + catch (DependencyFileNotFoundException) + { + // global.json not found in src/arcade meaning that repo is not the VMR + sourceRepoIsVmr = false; + targetDotNetVersion = await arcadeFileManager.ReadToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit, sourceRepoIsVmr); + } } GitFileContentContainer fileContainer = await _fileManager.UpdateDependencyFiles( @@ -210,7 +222,19 @@ public async Task> CommitUpdatesAsync( // Files in the source arcade repo. We use the remote factory because the // arcade repo may be in github while this remote is targeted at AzDO. IRemote arcadeRemote = await remoteFactory.CreateRemoteAsync(arcadeItem.RepoUri); - List engCommonFiles = await arcadeRemote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit); + List engCommonFiles = await arcadeRemote.GetCommonScriptFilesAsync(arcadeItem.RepoUri, arcadeItem.Commit, repoIsVmr: sourceRepoIsVmr); + // If the engCommon files are coming from the VMR, we have to remove 'src/arcade/' from the file paths + if (sourceRepoIsVmr) + { + engCommonFiles = engCommonFiles + .Select(f => new GitFile( + f.FilePath.Replace(VmrInfo.ArcadeRepoDir, null).TrimStart('/'), + f.Content, + f.ContentEncoding, + f.Mode)) + .ToList(); + } + filesToCommit.AddRange(engCommonFiles); // Files in the target repo @@ -367,12 +391,15 @@ private void CheckForValidGitClient() } } - public async Task> GetCommonScriptFilesAsync(string repoUri, string commit) + public async Task> GetCommonScriptFilesAsync(string repoUri, string commit, bool repoIsVmr = false) { CheckForValidGitClient(); _logger.LogInformation("Generating commits for script files"); + string path = repoIsVmr ? + VmrInfo.ArcadeRepoDir / Constants.CommonScriptFilesPath : + Constants.CommonScriptFilesPath; - List files = await _remoteGitClient.GetFilesAtCommitAsync(repoUri, commit, Constants.CommonScriptFilesPath); + List files = await _remoteGitClient.GetFilesAtCommitAsync(repoUri, commit, path); _logger.LogInformation("Generating commits for script files succeeded!"); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/PcsVmrBackFlower.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/PcsVmrBackFlower.cs index 134f8879c3..165d730ddf 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/PcsVmrBackFlower.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/PcsVmrBackFlower.cs @@ -5,6 +5,7 @@ using System.Threading; using System.Threading.Tasks; using LibGit2Sharp; +using Microsoft.DotNet.DarcLib.Conflicts; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.DotNet.ProductConstructionService.Client.Models; @@ -60,9 +61,10 @@ public PcsVmrBackFlower( ILocalLibGit2Client libGit2Client, ICoherencyUpdateResolver coherencyUpdateResolver, IAssetLocationResolver assetLocationResolver, + IBackFlowConflictResolver conflictResolver, IFileSystem fileSystem, ILogger logger) - : base(vmrInfo, sourceManifest, dependencyTracker, dependencyFileManager, vmrCloneManager, repositoryCloneManager, localGitClient, localGitRepoFactory, versionDetailsParser, vmrPatchHandler, workBranchFactory, basicBarClient, libGit2Client, coherencyUpdateResolver, assetLocationResolver, fileSystem, logger) + : base(vmrInfo, sourceManifest, dependencyTracker, dependencyFileManager, vmrCloneManager, repositoryCloneManager, localGitClient, localGitRepoFactory, versionDetailsParser, vmrPatchHandler, workBranchFactory, basicBarClient, libGit2Client, coherencyUpdateResolver, assetLocationResolver, conflictResolver, fileSystem, logger) { _sourceManifest = sourceManifest; _dependencyTracker = dependencyTracker; diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/PcsVmrForwardFlower.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/PcsVmrForwardFlower.cs index 4784353bc1..738e71176a 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/PcsVmrForwardFlower.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/PcsVmrForwardFlower.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.DotNet.DarcLib.Conflicts; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.DotNet.ProductConstructionService.Client.Models; @@ -55,9 +56,10 @@ public PcsVmrForwardFlower( IProcessManager processManager, ICoherencyUpdateResolver coherencyUpdateResolver, IAssetLocationResolver assetLocationResolver, + IForwardFlowConflictResolver conflictResolver, IFileSystem fileSystem, ILogger logger) - : base(vmrInfo, sourceManifest, vmrUpdater, dependencyTracker, vmrCloneManager, dependencyFileManager, localGitClient, libGit2Client, basicBarClient, localGitRepoFactory, versionDetailsParser, processManager, coherencyUpdateResolver, assetLocationResolver, fileSystem, logger) + : base(vmrInfo, sourceManifest, vmrUpdater, dependencyTracker, vmrCloneManager, dependencyFileManager, localGitClient, libGit2Client, basicBarClient, localGitRepoFactory, versionDetailsParser, processManager, coherencyUpdateResolver, assetLocationResolver, conflictResolver, fileSystem, logger) { _sourceManifest = sourceManifest; _dependencyTracker = dependencyTracker; @@ -91,19 +93,15 @@ public async Task FlowForwardAsync( build.Commit, cancellationToken); - Codeflow lastFlow = await GetLastFlowAsync(mapping, sourceRepo, currentIsBackflow: false); - - return await FlowCodeAsync( - lastFlow, - new ForwardFlow(lastFlow.TargetSha, build.Commit), - sourceRepo, + return await FlowForwardAsync( mapping, + sourceRepo, build, subscription.ExcludedAssets, baseBranch, targetBranch, + targetBranchExisted, discardPatches: true, - rebaseConflicts: !targetBranchExisted, cancellationToken); } } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrBackflower.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrBackflower.cs index bce3babeed..3308593cc0 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrBackflower.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrBackflower.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using LibGit2Sharp; +using Microsoft.DotNet.DarcLib.Conflicts; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.DotNet.ProductConstructionService.Client.Models; @@ -88,11 +89,11 @@ internal class VmrBackFlower : VmrCodeFlower, IVmrBackFlower private readonly IVmrDependencyTracker _dependencyTracker; private readonly IVmrCloneManager _vmrCloneManager; private readonly IRepositoryCloneManager _repositoryCloneManager; - private readonly ILocalGitClient _localGitClient; private readonly ILocalGitRepoFactory _localGitRepoFactory; private readonly IVmrPatchHandler _vmrPatchHandler; private readonly IWorkBranchFactory _workBranchFactory; private readonly IBasicBarClient _barClient; + private readonly IBackFlowConflictResolver _conflictResolver; private readonly IFileSystem _fileSystem; private readonly ILogger _logger; @@ -112,6 +113,7 @@ public VmrBackFlower( ILocalLibGit2Client libGit2Client, ICoherencyUpdateResolver coherencyUpdateResolver, IAssetLocationResolver assetLocationResolver, + IBackFlowConflictResolver conflictResolver, IFileSystem fileSystem, ILogger logger) : base(vmrInfo, sourceManifest, dependencyTracker, localGitClient, libGit2Client, localGitRepoFactory, versionDetailsParser, dependencyFileManager, coherencyUpdateResolver, assetLocationResolver, fileSystem, logger) @@ -121,11 +123,11 @@ public VmrBackFlower( _dependencyTracker = dependencyTracker; _vmrCloneManager = vmrCloneManager; _repositoryCloneManager = repositoryCloneManager; - _localGitClient = localGitClient; _localGitRepoFactory = localGitRepoFactory; _vmrPatchHandler = vmrPatchHandler; _workBranchFactory = workBranchFactory; _barClient = basicBarClient; + _conflictResolver = conflictResolver; _fileSystem = fileSystem; _logger = logger; } @@ -260,6 +262,13 @@ protected async Task FlowBackAsync( hadPreviousChanges: hasChanges, cancellationToken); + if (hasChanges) + { + // We try to merge the target branch so that we can potentially + // resolve some expected conflicts in the version files + await _conflictResolver.TryMergingRepoBranch(targetRepo, targetBranch, baseBranch); + } + return hasChanges; } @@ -533,4 +542,7 @@ await _repositoryCloneManager.PrepareCloneAsync( return (targetBranchExisted, mapping); } + + protected override NativePath GetEngCommonPath(NativePath sourceRepo) => sourceRepo / VmrInfo.SourceDirName / "arcade" / Constants.CommonScriptFilesPath; + protected override bool TargetRepoIsVmr() => false; } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs index 2542f403d9..562f8cd98d 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs @@ -379,7 +379,7 @@ protected async Task UpdateDependenciesAndToolset( if (arcadeItem != null) { - targetDotNetVersion = await _dependencyFileManager.ReadToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit); + targetDotNetVersion = await _dependencyFileManager.ReadToolsDotnetVersionAsync(arcadeItem.RepoUri, arcadeItem.Commit, repoIsVmr: true); } GitFileContentContainer updatedFiles = await _dependencyFileManager.UpdateDependencyFiles( @@ -402,8 +402,16 @@ protected async Task UpdateDependenciesAndToolset( _fileSystem.DeleteDirectory(commonDir, true); } + // Check if the VMR contains src/arcade/eng/common + var arcadeEngCommonDir = GetEngCommonPath(sourceRepo); + if (!_fileSystem.DirectoryExists(arcadeEngCommonDir)) + { + _logger.LogWarning("VMR does not contain src/arcade/eng/common, skipping eng/common update"); + return hadUpdates; + } + _fileSystem.CopyDirectory( - sourceRepo / Constants.CommonScriptFilesPath, + arcadeEngCommonDir, targetRepo.Path / Constants.CommonScriptFilesPath, true); } @@ -423,7 +431,6 @@ protected async Task UpdateDependenciesAndToolset( { await targetRepo.CommitAsync("Updated dependencies", allowEmpty: true, cancellationToken: cancellationToken); } - return true; } @@ -465,4 +472,7 @@ protected record Backflow(string VmrSha, string RepoSha) : Codeflow(VmrSha, Repo { public override string Name { get; } = "back"; } + + protected abstract NativePath GetEngCommonPath(NativePath sourceRepo); + protected abstract bool TargetRepoIsVmr(); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs index 246ecbf372..e0125ad69e 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using LibGit2Sharp; +using Microsoft.DotNet.DarcLib.Conflicts; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.DotNet.ProductConstructionService.Client.Models; @@ -76,6 +77,7 @@ internal class VmrForwardFlower : VmrCodeFlower, IVmrForwardFlower private readonly IBasicBarClient _barClient; private readonly ILocalGitRepoFactory _localGitRepoFactory; private readonly IProcessManager _processManager; + private readonly IForwardFlowConflictResolver _conflictResolver; private readonly ILogger _logger; public VmrForwardFlower( @@ -93,6 +95,7 @@ public VmrForwardFlower( IProcessManager processManager, ICoherencyUpdateResolver coherencyUpdateResolver, IAssetLocationResolver assetLocationResolver, + IForwardFlowConflictResolver conflictResolver, IFileSystem fileSystem, ILogger logger) : base(vmrInfo, sourceManifest, dependencyTracker, localGitClient, libGit2Client, localGitRepoFactory, versionDetailsParser, dependencyFileManager, coherencyUpdateResolver, assetLocationResolver, fileSystem, logger) @@ -105,6 +108,7 @@ public VmrForwardFlower( _barClient = basicBarClient; _localGitRepoFactory = localGitRepoFactory; _processManager = processManager; + _conflictResolver = conflictResolver; _logger = logger; } @@ -142,11 +146,34 @@ public async Task FlowForwardAsync( ILocalGitRepo sourceRepo = _localGitRepoFactory.Create(repoPath); SourceMapping mapping = _dependencyTracker.GetMapping(mappingName); - ISourceComponent repoInfo = _sourceManifest.GetRepoVersion(mappingName); + + return await FlowForwardAsync( + mapping, + sourceRepo, + build, + excludedAssets, + baseBranch, + targetBranch, + targetBranchExisted, + discardPatches, + cancellationToken); + } + + protected async Task FlowForwardAsync( + SourceMapping mapping, + ILocalGitRepo sourceRepo, + Build build, + IReadOnlyCollection? excludedAssets, + string baseBranch, + string targetBranch, + bool targetBranchExisted, + bool discardPatches = false, + CancellationToken cancellationToken = default) + { + ISourceComponent repoInfo = _sourceManifest.GetRepoVersion(mapping.Name); // Refresh the repo await sourceRepo.FetchAllAsync([mapping.DefaultRemote, repoInfo.RemoteUri], cancellationToken); - await sourceRepo.CheckoutAsync(build.Commit); Codeflow lastFlow = await GetLastFlowAsync(mapping, sourceRepo, currentIsBackflow: false); @@ -164,14 +191,13 @@ public async Task FlowForwardAsync( rebaseConflicts: !targetBranchExisted, cancellationToken); - hasChanges |= await UpdateDependenciesAndToolset( - sourceRepo.Path, - _localGitRepoFactory.Create(_vmrInfo.VmrPath), - build, - excludedAssets, - sourceElementSha: null, - hasChanges, - cancellationToken); + if (hasChanges) + { + // We try to merge the target branch so that we can potentially + // resolve some expected conflicts in the version files + ILocalGitRepo vmr = _localGitRepoFactory.Create(_vmrInfo.VmrPath); + await _conflictResolver.TryMergingBranch(vmr, mapping.Name, targetBranch, baseBranch); + } return hasChanges; } @@ -389,4 +415,7 @@ .. submodules.Select(s => s.Path).Distinct().Select(VmrPatchHandler.GetExclusion amendReapplyCommit: true, cancellationToken: cancellationToken); } + + protected override NativePath GetEngCommonPath(NativePath sourceRepo) => sourceRepo / Constants.CommonScriptFilesPath; + protected override bool TargetRepoIsVmr() => true; } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs index 72a3120daf..83fe50f333 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrInfo.cs @@ -83,6 +83,8 @@ public class VmrInfo : IVmrInfo public const string CodeownersFileName = "CODEOWNERS"; public const string CredScanSuppressionsFileName = "CredScanSuppressions.json"; + public static UnixPath ArcadeRepoDir = SourcesDir / "arcade"; + public static UnixPath DefaultRelativeSourceMappingsPath { get; } = SourcesDir / SourceMappingsFileName; public static UnixPath DefaultRelativeSourceManifestPath { get; } = SourcesDir / SourceManifestFileName; diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs index ec6f4aeecd..5ac58bafae 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs @@ -7,6 +7,7 @@ using System.Security.Cryptography.X509Certificates; using Maestro.Common; using Maestro.Common.AzureDevOpsTokens; +using Microsoft.DotNet.DarcLib.Conflicts; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.Extensions.DependencyInjection; @@ -107,6 +108,8 @@ private static IServiceCollection AddVmrManagers( services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); + services.TryAddTransient(); + services.TryAddTransient(); services.TryAddScoped(); services.TryAddScoped(); diff --git a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/PullRequestUpdate.cs b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/PullRequestUpdate.cs index d1e6298ab2..01135213f1 100644 --- a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/PullRequestUpdate.cs +++ b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/PullRequestUpdate.cs @@ -1,17 +1,26 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using Newtonsoft.Json; namespace Microsoft.DotNet.ProductConstructionService.Client.Models { public partial class PullRequestUpdate { - public PullRequestUpdate() + public PullRequestUpdate(Guid subscriptionId, int buildId) { + SubscriptionId = subscriptionId; + BuildId = buildId; } [JsonProperty("sourceRepository")] public string SourceRepository { get; set; } + + [JsonProperty("subscriptionId")] + public Guid SubscriptionId { get; set; } + + [JsonProperty("buildId")] + public int BuildId { get; set; } } } diff --git a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/TrackedPullRequest.cs b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/TrackedPullRequest.cs index 6957ad9c48..d8a7db8ed0 100644 --- a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/TrackedPullRequest.cs +++ b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/TrackedPullRequest.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using Newtonsoft.Json; @@ -8,10 +9,16 @@ namespace Microsoft.DotNet.ProductConstructionService.Client.Models { public partial class TrackedPullRequest { - public TrackedPullRequest() + public TrackedPullRequest(bool sourceEnabled, DateTimeOffset lastUpdate, DateTimeOffset lastCheck) { + SourceEnabled = sourceEnabled; + LastUpdate = lastUpdate; + LastCheck = lastCheck; } + [JsonProperty("id")] + public string Id { get; set; } + [JsonProperty("url")] public string Url { get; set; } @@ -21,6 +28,18 @@ public TrackedPullRequest() [JsonProperty("targetBranch")] public string TargetBranch { get; set; } + [JsonProperty("sourceEnabled")] + public bool SourceEnabled { get; set; } + + [JsonProperty("lastUpdate")] + public DateTimeOffset LastUpdate { get; set; } + + [JsonProperty("lastCheck")] + public DateTimeOffset LastCheck { get; set; } + + [JsonProperty("nextCheck")] + public DateTimeOffset? NextCheck { get; set; } + [JsonProperty("updates")] public List Updates { get; set; } } diff --git a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/PullRequest.cs b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/PullRequest.cs index b2b57294e5..1237a93178 100644 --- a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/PullRequest.cs +++ b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/PullRequest.cs @@ -19,6 +19,11 @@ public partial interface IPullRequest CancellationToken cancellationToken = default ); + Task UntrackPullRequestAsync( + string id, + CancellationToken cancellationToken = default + ); + } internal partial class PullRequest : IServiceOperations, IPullRequest @@ -100,5 +105,66 @@ internal async Task OnGetTrackedPullRequestsFailed(Request req, Response res) Client.OnFailedRequest(ex); throw ex; } + + partial void HandleFailedUntrackPullRequestRequest(RestApiException ex); + + public async Task UntrackPullRequestAsync( + string id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/pull-requests/tracked/{id}".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Delete; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnUntrackPullRequestFailed(_req, _res).ConfigureAwait(false); + } + + + return; + } + } + } + + internal async Task OnUntrackPullRequestFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedUntrackPullRequestRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } } } diff --git a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/ProductConstructionServiceApi.cs b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/ProductConstructionServiceApi.cs index aa7dca36d5..c5cf66cf8b 100644 --- a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/ProductConstructionServiceApi.cs +++ b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/ProductConstructionServiceApi.cs @@ -2,15 +2,23 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using Newtonsoft.Json.Linq; -using System.Net; using System.IO; +using System.Net; using System.Net.Http; using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Newtonsoft.Json.Linq; #nullable enable namespace Microsoft.DotNet.ProductConstructionService.Client { + public partial interface IProductConstructionServiceApi + { + Task IsAdmin(CancellationToken cancellationToken = default); + } + public partial class ProductConstructionServiceApi { // Special error handler to consumes the generated MaestroApi code. If this method returns without throwing a specific exception @@ -40,6 +48,33 @@ partial void HandleFailedRequest(RestApiException ex) "Please make sure the PAT you're using is valid."); } } + + public async Task IsAdmin(CancellationToken cancellationToken = default) + { + var url = new RequestUriBuilder(); + url.Reset(Options.BaseUri); + url.AppendPath("/Account", false); + + using (var request = Pipeline.CreateRequest()) + { + request.Uri = url; + request.Method = RequestMethod.Get; + + using (var response = await SendAsync(request, cancellationToken).ConfigureAwait(false)) + { + if (response.Status < 200 || response.Status >= 300 || response.ContentStream == null) + { + throw new RestApiException(request, response, "Invalid response"); + } + + using (var _reader = new StreamReader(response.ContentStream)) + { + var content = await _reader.ReadToEndAsync().ConfigureAwait(false); + return content.Trim() == "Admin"; + } + } + } + } } internal partial class ProductConstructionServiceApiResponseClassifier diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/PullRequestController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/PullRequestController.cs index 6d1125dcad..6b7e07e93e 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/PullRequestController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/PullRequestController.cs @@ -6,9 +6,11 @@ using Maestro.Data; using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.EntityFrameworkCore; +using ProductConstructionService.Api.Configuration; using ProductConstructionService.Common; using ProductConstructionService.DependencyFlow; @@ -57,10 +59,11 @@ public PullRequestController( [ValidateModelState] public async Task GetTrackedPullRequests() { - var cache = _cacheFactory.Create(nameof(InProgressPullRequest) + "_"); + var keyPrefix = nameof(InProgressPullRequest) + "_"; + var cache = _cacheFactory.Create(keyPrefix); var prs = new List(); - await foreach (var key in cache.GetKeysAsync(nameof(InProgressPullRequest) + "_*")) + await foreach (var key in cache.GetKeysAsync(keyPrefix + "*")) { var pr = await _cacheFactory .Create(key, includeTypeInKey: false) @@ -99,19 +102,36 @@ public async Task GetTrackedPullRequests() var updates = subscriptions .Select(update => new PullRequestUpdate( TurnApiUrlToWebsite(update.SourceRepository, org, repoName), + pr.ContainedSubscriptions.First(s => s.SubscriptionId == update.Id).SubscriptionId, pr.ContainedSubscriptions.First(s => s.SubscriptionId == update.Id).BuildId)) .ToList(); prs.Add(new TrackedPullRequest( + key.Replace(keyPrefix, null, StringComparison.InvariantCultureIgnoreCase), TurnApiUrlToWebsite(pr.Url, org, repoName), sampleSub?.Channel?.Name, sampleSub?.TargetBranch, + sampleSub?.SourceEnabled ?? false, + pr.LastUpdate, + pr.LastCheck, + pr.NextCheck, updates)); } return Ok(prs.AsQueryable()); } + [HttpDelete("tracked/{id}")] + [Authorize(Policy = AuthenticationConfiguration.AdminAuthorizationPolicyName)] + [SwaggerApiResponse(HttpStatusCode.OK, Type = typeof(void), Description = "The pull request was successfully untracked")] + [SwaggerApiResponse(HttpStatusCode.NotFound, Type = typeof(void), Description = "The pull request was not found in the list of tracked pull requests")] + [ValidateModelState] + public async Task UntrackPullRequest(string id) + { + var cache = _cacheFactory.Create($"{nameof(InProgressPullRequest)}_{id}", includeTypeInKey: false); + return await cache.TryDeleteAsync() == null ? NotFound() : Ok(); + } + private static string TurnApiUrlToWebsite(string url, string? orgName, string? repoName) { var match = GitHubApiPrUrlRegex().Match(url); @@ -139,12 +159,18 @@ private static string TurnApiUrlToWebsite(string url, string? orgName, string? r } private record TrackedPullRequest( + string Id, string Url, string? Channel, string? TargetBranch, + bool SourceEnabled, + DateTime LastUpdate, + DateTime LastCheck, + DateTime? NextCheck, List Updates); private record PullRequestUpdate( string SourceRepository, + Guid SubscriptionId, int BuildId); } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Configuration/AuthenticationConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/AuthenticationConfiguration.cs index d231329214..63d9d5747f 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Configuration/AuthenticationConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/AuthenticationConfiguration.cs @@ -7,7 +7,7 @@ namespace ProductConstructionService.Api.Configuration; -public static class AuthenticationConfiguration +internal static class AuthenticationConfiguration { public const string EntraAuthorizationPolicyName = "Entra"; public const string MsftAuthorizationPolicyName = "msft"; @@ -87,20 +87,18 @@ public static void ConfigureAuthServices(this IServiceCollection services, IConf }); services - .AddAuthorization(options => - { - options.AddPolicy(MsftAuthorizationPolicyName, policy => + .AddAuthorizationBuilder() + .AddPolicy(MsftAuthorizationPolicyName, policy => { policy.AddAuthenticationSchemes(AuthenticationSchemes); policy.RequireAuthenticatedUser(); policy.RequireRole(userRole); - }); - options.AddPolicy(AdminAuthorizationPolicyName, policy => + }) + .AddPolicy(AdminAuthorizationPolicyName, policy => { policy.AddAuthenticationSchemes(AuthenticationSchemes); policy.RequireAuthenticatedUser(); - policy.RequireRole("Admin"); + policy.RequireRole(adminRole); }); - }); } } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/AccountController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/AccountController.cs index 3e42a78868..e829122d4f 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/AccountController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/AccountController.cs @@ -32,4 +32,15 @@ public IActionResult SignIn(string? returnUrl = null) new AuthenticationProperties() { RedirectUri = returnUrl }, OpenIdConnectDefaults.AuthenticationScheme); } + + [HttpGet("/Account")] + [Authorize] + public IActionResult Account() + { +#if DEBUG + return Ok("Admin"); +#else + return Ok(HttpContext.User.IsInRole("Admin") ? "Admin" : "User"); +#endif + } } diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/App.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/App.razor index 542d6e4642..c3d8ea657a 100644 --- a/src/ProductConstructionService/ProductConstructionService.BarViz/App.razor +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/App.razor @@ -16,6 +16,10 @@ + + + + @code { private IDisposable? _navigationHandlerRegistration = null; diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Code/Services/UserRoleManager.cs b/src/ProductConstructionService/ProductConstructionService.BarViz/Code/Services/UserRoleManager.cs new file mode 100644 index 0000000000..2373d16bc5 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Code/Services/UserRoleManager.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.ProductConstructionService.Client; + +namespace ProductConstructionService.BarViz.Code.Services; + +public class UserRoleManager(IProductConstructionServiceApi pcsApi) +{ + private readonly Lazy> _isAdmin = new( + () => pcsApi.IsAdmin(), + LazyThreadSafetyMode.ExecutionAndPublication); + + public Task IsAdmin => _isAdmin.Value; +} diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/PullRequestContextMenu.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/PullRequestContextMenu.razor new file mode 100644 index 0000000000..73d99d9cef --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/PullRequestContextMenu.razor @@ -0,0 +1,78 @@ +@using Microsoft.DotNet.ProductConstructionService.Client; +@using Microsoft.DotNet.ProductConstructionService.Client.Models; +@using System.ComponentModel.DataAnnotations +@inject IProductConstructionServiceApi PcsApi +@inject IToastService ToastService +@inject UserRoleManager UserRoleManager + + + + + + + + Re-trigger subscription + + + + + + + Untrack + + + + + + + +@code { + private bool _isContextMenuOpen = false; + private bool _isAdmin = false; + + [Parameter, EditorRequired] + public required TrackedPullRequest PullRequest { get; set; } + + [Parameter, EditorRequired] + public required Func Refresh { get; set; } + + protected override async Task OnInitializedAsync() + { + _isAdmin = await UserRoleManager.IsAdmin; + } + + async Task UntrackPullRequest() + { + try + { + await PcsApi.PullRequest.UntrackPullRequestAsync(PullRequest.Id); + await Refresh.Invoke(); + ToastService.ShowSuccess("PR untracked"); + } + catch (Exception e) + { + ToastService.ShowError("Failed to untrack the PR: " + e.ToString()); + } + } + + async Task TriggerSubscription() + { + try + { + foreach (var update in PullRequest.Updates) + { + await PcsApi.Subscriptions.TriggerSubscriptionAsync(update.SubscriptionId); + ToastService.ShowProgress("Subscriptions in the PR triggered"); + } + } + catch + { + ToastService.ShowError("Failed to trigger the subscription"); + } + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Pages/PullRequests.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/Pages/PullRequests.razor index c80998a1e8..c9be26cacb 100644 --- a/src/ProductConstructionService/ProductConstructionService.BarViz/Pages/PullRequests.razor +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Pages/PullRequests.razor @@ -1,46 +1,77 @@ @page "/pullrequests" @using Microsoft.DotNet.ProductConstructionService.Client -@using Microsoft.DotNet.ProductConstructionService.Client.Models; +@using Microsoft.DotNet.ProductConstructionService.Client.Models +@using Microsoft.FluentUI.AspNetCore.Components.Extensions @using System.Linq.Expressions -@inject IProductConstructionServiceApi api +@using ProductConstructionService.BarViz.Components +@inject IProductConstructionServiceApi PcsApi Tracked Pull Requests - +
+ +
+ + - - - @context.Url - - - @if (context.Channel != null) - { - @context.Channel - } - else - { - - + + + No pull requests found + + + + @context.Url + + + @(context.Channel ?? "N/A") + + + @(context.TargetBranch ?? "N/A") + + + + @(context.LastUpdate == default ? "N/A" : (DateTime.UtcNow - context.LastUpdate).ToTimeAgo()) - } - - - @if (context.TargetBranch != null) - { - @context.TargetBranch - } - else - { + @if (context.Updates.Count > 0) + { + + @foreach (var update in context.Updates) + { +
+
@update.SourceRepository
+ Build ID: @update.BuildId
+ Subscription ID: @update.SubscriptionId
+
+ } + + } + + - + @(context.LastCheck == default ? "N/A" : (DateTime.UtcNow - context.LastCheck).ToTimeAgo()) + @if (DateTime.UtcNow - context.LastCheck > TimeSpan.FromHours(1)) + { + + } + @if (!context.NextCheck.HasValue) + { + + } - } - + + + + + @@ -50,21 +81,38 @@ Timer? _timer; GridSort SortBy(Expression> sorter) => GridSort.ByAscending(sorter); + bool _autoRefresh = true; - protected override async Task OnInitializedAsync() + protected override void OnInitialized() { - await LoadDataAsync(); _timer = new Timer(async _ => await LoadDataAsync(), null, TimeSpan.Zero, TimeSpan.FromSeconds(30)); } - private async Task LoadDataAsync() + async Task LoadDataAsync() { - TrackedPullRequests = (await api.PullRequest.GetTrackedPullRequestsAsync()).AsQueryable(); + TrackedPullRequests = (await PcsApi.PullRequest.GetTrackedPullRequestsAsync()).AsQueryable(); await InvokeAsync(StateHasChanged); } + void SetAutoRefresh(bool value) + { + _autoRefresh = value; + + if (value) + { + OnInitialized(); + } + else + { + _timer?.Dispose(); + } + } + public void Dispose() { - _timer?.Dispose(); + if (_autoRefresh) + { + _timer?.Dispose(); + } } } diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Program.cs b/src/ProductConstructionService/ProductConstructionService.BarViz/Program.cs index d2e88dd743..fff69decbd 100644 --- a/src/ProductConstructionService/ProductConstructionService.BarViz/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Program.cs @@ -7,6 +7,7 @@ using Microsoft.AspNetCore.Components.WebAssembly.Hosting; using Microsoft.DotNet.ProductConstructionService.Client; using Microsoft.FluentUI.AspNetCore.Components; +using Microsoft.FluentUI.AspNetCore.Components.Components.Tooltip; using ProductConstructionService.BarViz; using ProductConstructionService.BarViz.Code.Services; using TextCopy; @@ -29,6 +30,8 @@ builder.Services.AddSingleton(PcsApiFactory.GetAnonymous(PcsApiBaseAddress)); builder.Services.InjectClipboard(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddBlazoredSessionStorage(); +builder.Services.AddScoped(); await builder.Build().RunAsync(); diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/InProgressPullRequest.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/InProgressPullRequest.cs index da6dbe7657..67950cdb91 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/InProgressPullRequest.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/InProgressPullRequest.cs @@ -41,4 +41,13 @@ public class InProgressPullRequest : DependencyFlowWorkItem, IPullRequest [DataMember] public bool? SourceRepoNotified { get; set; } + + [DataMember] + public DateTime LastUpdate { get; set; } = DateTime.UtcNow; + + [DataMember] + public DateTime LastCheck { get; set; } = DateTime.UtcNow; + + [DataMember] + public DateTime? NextCheck { get; set; } } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs index 33eacbb768..3f124e5c4c 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs @@ -501,7 +501,7 @@ private async Task AddDependencyFlowEventsAsync( .ToList(), CoherencyCheckSuccessful = repoDependencyUpdate.CoherencyCheckSuccessful, - CoherencyErrors = repoDependencyUpdate.CoherencyErrors + CoherencyErrors = repoDependencyUpdate.CoherencyErrors, }; if (!string.IsNullOrEmpty(prUrl)) @@ -619,6 +619,7 @@ await AddDependencyFlowEventsAsync( pullRequest.Title = await _pullRequestBuilder.GeneratePRTitleAsync(pr.ContainedSubscriptions, targetBranch); await darcRemote.UpdatePullRequestAsync(pr.Url, pullRequest); + pr.LastUpdate = DateTime.UtcNow; await SetPullRequestCheckReminder(pr, isCodeFlow: update.SubscriptionType == SubscriptionType.DependenciesAndSources); _logger.LogInformation("Pull request '{prUrl}' updated", pr.Url); @@ -787,6 +788,10 @@ private async Task SetPullRequestCheckReminder(InProgressPullRequest prState, bo Url = prState.Url, IsCodeFlow = isCodeFlow }; + + prState.LastCheck = DateTime.UtcNow; + prState.NextCheck = prState.LastCheck + DefaultReminderDelay; + await _pullRequestCheckReminders.SetReminderAsync(reminder, DefaultReminderDelay, isCodeFlow); await _pullRequestState.SetAsync(prState); } @@ -1006,6 +1011,7 @@ private async Task UpdateAssetsAndSources(SubscriptionUpdateWorkItem updat Description = description }); + pullRequest.LastUpdate = DateTime.UtcNow; await SetPullRequestCheckReminder(pullRequest, true); await _pullRequestUpdateReminders.UnsetReminderAsync(true); @@ -1146,6 +1152,7 @@ await AddDependencyFlowEventsAsync( MergePolicyCheckResult.PendingPolicies, prUrl); + inProgressPr.LastUpdate = DateTime.UtcNow; await SetPullRequestCheckReminder(inProgressPr, isCodeFlow); await _pullRequestUpdateReminders.UnsetReminderAsync(isCodeFlow); diff --git a/src/ProductConstructionService/ProductConstructionService.ReproTool/CompactConsoleLoggerFormatter.cs b/src/ProductConstructionService/ProductConstructionService.ReproTool/CompactConsoleLoggerFormatter.cs new file mode 100644 index 0000000000..e3938fa519 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.ReproTool/CompactConsoleLoggerFormatter.cs @@ -0,0 +1,193 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging.Console; +using Microsoft.Extensions.Options; + +// TODO (https://github.com/dotnet/arcade/issues/8836): Use the formatter from Arcade.Common once we're able to consume latest Arcade + +namespace ProductConstructionService.ReproTool; + +/// +/// Copied over from SimpleConsoleFormatter. Leaves out the logger name and new line, turning +/// info: test[0] +/// Log message +/// Second line of the message +/// +/// into +/// +/// info: Log message +/// Second line of the message +/// +/// Only using SimpleConsoleFormatterOptions.SingleLine didn't help because multi-line messages +/// were put together on a single line so things like stack traces of exceptions were unreadable. +/// +/// See https://github.com/dotnet/runtime/blob/0817e748b7698bef1e812fd74c8a3558b7f86421/src/libraries/Microsoft.Extensions.Logging.Console/src/SimpleConsoleFormatter.cs +/// +public class CompactConsoleLoggerFormatter : ConsoleFormatter +{ + private const string LoglevelPadding = ": "; + private const string DefaultForegroundColor = "\x1B[39m\x1B[22m"; // reset to default foreground color + private const string DefaultBackgroundColor = "\x1B[49m"; // reset to the background color + + public const string FormatterName = "compact"; + + private readonly SimpleConsoleFormatterOptions _options; + private readonly string _messagePadding; + private readonly string _newLineWithMessagePadding; + + public CompactConsoleLoggerFormatter(IOptionsMonitor options) + : base(FormatterName) + { + _options = options.CurrentValue; + _messagePadding = new string(' ', GetLogLevelString(LogLevel.Information).Length + LoglevelPadding.Length + (_options.TimestampFormat?.Length ?? 0)); + _newLineWithMessagePadding = Environment.NewLine + _messagePadding; + } + + public override void Write(in LogEntry logEntry, IExternalScopeProvider? scopeProvider, TextWriter textWriter) + { + if (logEntry.Formatter == null) + { + return; + } + + var message = logEntry.Formatter(logEntry.State, logEntry.Exception); + if (logEntry.Exception == null && message == null) + { + return; + } + + LogLevel logLevel = logEntry.LogLevel; + var logLevelColors = GetLogLevelConsoleColors(logLevel); + var logLevelString = GetLogLevelString(logLevel); + + if (_options.TimestampFormat != null) + { + var timestamp = DateTimeOffset.Now.ToString(_options.TimestampFormat); + textWriter.Write(timestamp); + } + + WriteColoredMessage(textWriter, logLevelString, logLevelColors.Background, logLevelColors.Foreground); + + textWriter.Write(LoglevelPadding); + + WriteMessage(textWriter, message, false); + + // Example: + // System.InvalidOperationException + // at Namespace.Class.Function() in File:line X + if (logEntry.Exception != null) + { + // exception message + WriteMessage(textWriter, logEntry.Exception.ToString()); + } + } + + private void WriteMessage(TextWriter textWriter, string message, bool includePadding = true) + { + if (message == null) + { + return; + } + + if (includePadding) + { + textWriter.Write(_messagePadding); + } + + textWriter.WriteLine(message.Replace(Environment.NewLine, _newLineWithMessagePadding)); + } + + private static string GetLogLevelString(LogLevel logLevel) => logLevel switch + { + LogLevel.Trace => "trce", + LogLevel.Debug => "dbug", + LogLevel.Information => "info", + LogLevel.Warning => "warn", + LogLevel.Error => "fail", + LogLevel.Critical => "crit", + _ => throw new ArgumentOutOfRangeException(nameof(logLevel)) + }; + + private (ConsoleColor? Foreground, ConsoleColor? Background) GetLogLevelConsoleColors(LogLevel logLevel) + { + if (_options.ColorBehavior == LoggerColorBehavior.Disabled) + { + return (null, null); + } + + // We must explicitly set the background color if we are setting the foreground color, + // since just setting one can look bad on the users console. + return logLevel switch + { + LogLevel.Trace => (ConsoleColor.Gray, ConsoleColor.Black), + LogLevel.Debug => (ConsoleColor.Gray, ConsoleColor.Black), + LogLevel.Information => (ConsoleColor.DarkGreen, ConsoleColor.Black), + LogLevel.Warning => (ConsoleColor.Yellow, ConsoleColor.Black), + LogLevel.Error => (ConsoleColor.Black, ConsoleColor.DarkRed), + LogLevel.Critical => (ConsoleColor.White, ConsoleColor.DarkRed), + _ => (null, null) + }; + } + + private static void WriteColoredMessage(TextWriter textWriter, string message, ConsoleColor? background, ConsoleColor? foreground) + { + // Order: backgroundcolor, foregroundcolor, Message, reset foregroundcolor, reset backgroundcolor + if (background.HasValue) + { + textWriter.Write(GetBackgroundColorEscapeCode(background.Value)); + } + + if (foreground.HasValue) + { + textWriter.Write(GetForegroundColorEscapeCode(foreground.Value)); + } + + textWriter.Write(message); + + if (foreground.HasValue) + { + textWriter.Write(DefaultForegroundColor); // reset to default foreground color + } + + if (background.HasValue) + { + textWriter.Write(DefaultBackgroundColor); // reset to the background color + } + } + + private static string GetForegroundColorEscapeCode(ConsoleColor color) => color switch + { + ConsoleColor.Black => "\x1B[30m", + ConsoleColor.DarkRed => "\x1B[31m", + ConsoleColor.DarkGreen => "\x1B[32m", + ConsoleColor.DarkYellow => "\x1B[33m", + ConsoleColor.DarkBlue => "\x1B[34m", + ConsoleColor.DarkMagenta => "\x1B[35m", + ConsoleColor.DarkCyan => "\x1B[36m", + ConsoleColor.Gray => "\x1B[37m", + ConsoleColor.Red => "\x1B[1m\x1B[31m", + ConsoleColor.Green => "\x1B[1m\x1B[32m", + ConsoleColor.Yellow => "\x1B[1m\x1B[33m", + ConsoleColor.Blue => "\x1B[1m\x1B[34m", + ConsoleColor.Magenta => "\x1B[1m\x1B[35m", + ConsoleColor.Cyan => "\x1B[1m\x1B[36m", + ConsoleColor.White => "\x1B[1m\x1B[37m", + _ => DefaultForegroundColor // default foreground color + }; + + private static string GetBackgroundColorEscapeCode(ConsoleColor color) => color switch + { + ConsoleColor.Black => "\x1B[40m", + ConsoleColor.DarkRed => "\x1B[41m", + ConsoleColor.DarkGreen => "\x1B[42m", + ConsoleColor.DarkYellow => "\x1B[43m", + ConsoleColor.DarkBlue => "\x1B[44m", + ConsoleColor.DarkMagenta => "\x1B[45m", + ConsoleColor.DarkCyan => "\x1B[46m", + ConsoleColor.Gray => "\x1B[47m", + _ => DefaultBackgroundColor // Use default background color + }; +} diff --git a/src/ProductConstructionService/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj b/src/ProductConstructionService/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj index 102995a1e3..b0749c91aa 100644 --- a/src/ProductConstructionService/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj +++ b/src/ProductConstructionService/ProductConstructionService.ReproTool/ProductConstructionService.ReproTool.csproj @@ -6,10 +6,12 @@ enable enable False + d1deb1c4-c45b-4d37-8e76-cc23515470a4 + diff --git a/src/ProductConstructionService/ProductConstructionService.ReproTool/Program.cs b/src/ProductConstructionService/ProductConstructionService.ReproTool/Program.cs index e0ab8ebe5b..c5c907161f 100644 --- a/src/ProductConstructionService/ProductConstructionService.ReproTool/Program.cs +++ b/src/ProductConstructionService/ProductConstructionService.ReproTool/Program.cs @@ -2,13 +2,21 @@ // The .NET Foundation licenses this file to you under the MIT license. using CommandLine; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using ProductConstructionService.ReproTool; Parser.Default.ParseArguments(args) .WithParsed(o => { - ServiceCollection services = new ServiceCollection(); + IConfiguration userSecrets = new ConfigurationBuilder() + .AddUserSecrets() + .Build(); + o.GitHubToken ??= userSecrets["GITHUB_TOKEN"]; + o.GitHubToken ??= Environment.GetEnvironmentVariable("GITHUB_TOKEN"); + ArgumentNullException.ThrowIfNull(o.GitHubToken, "GitHub must be provided via env variable, user secret or an option"); + + var services = new ServiceCollection(); services.RegisterServices(o); diff --git a/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproTool.cs b/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproTool.cs index 1ce9a520d5..f55b99663d 100644 --- a/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproTool.cs +++ b/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproTool.cs @@ -145,30 +145,41 @@ internal async Task ReproduceCodeFlow() await TriggerSubscriptionAsync(testSubscription.Value); - logger.LogInformation("Code flow successfully recreated. Press enter to finish and cleanup"); - Console.ReadLine(); - if (options.SkipCleanup) { logger.LogInformation("Skipping cleanup. If you want to re-trigger the reproduced subscription run \"darc trigger-subscriptions --ids {subscriptionId}\" --bar-uri {barUri}", testSubscription.Value, ProductConstructionServiceApiOptions.PcsLocalUri); + return; + } + + logger.LogInformation("Code flow successfully recreated. Press enter to finish and cleanup"); + Console.ReadLine(); + + // Cleanup + if (isForwardFlow) + { + await DeleteDarcPRBranchAsync(VmrForkRepoName, vmrTmpBranch.Value); } else { - // Cleanup - await DeleteDarcPRBranchAsync( - isForwardFlow ? VmrForkRepoName : productRepoUri.Split('/').Last(), - isForwardFlow ? vmrTmpBranch.Value : productRepoTmpBranch.Value); + await DeleteDarcPRBranchAsync(productRepoUri.Split('/').Last(), productRepoTmpBranch.Value); } } private async Task DeleteDarcPRBranchAsync(string repo, string targetBranch) { var branch = (await ghClient.Repository.Branch.GetAll(MaestroAuthTestOrgName, repo)) - .FirstOrDefault(branch => branch.Name.StartsWith($"{DarcPRBranchPrefix}-{targetBranch}")) - ?? throw new Exception($"Couldn't find darc PR branch targeting branch {targetBranch}"); - await DeleteGitHubBranchAsync(repo, branch.Name); + .FirstOrDefault(branch => branch.Name.StartsWith($"{DarcPRBranchPrefix}-{targetBranch}")); + + if (branch == null) + { + logger.LogWarning("Couldn't find darc PR branch targeting branch {targetBranch}", targetBranch); + } + else + { + await DeleteGitHubBranchAsync(repo, branch.Name); + } } private async Task AddRepositoryToBarIfMissingAsync(string repositoryName) @@ -193,7 +204,7 @@ private async Task CreateBuildAsync(string repositoryUrl, string branch, commit: commit, azureDevOpsAccount: "test", azureDevOpsProject: "test", - azureDevOpsBuildNumber: $"{DateTime.UtcNow.ToString("yyyyMMdd")}.{new Random().Next(1, 75)}", + azureDevOpsBuildNumber: $"{DateTime.UtcNow:yyyyMMdd}.{new Random().Next(1, 75)}", azureDevOpsRepository: repositoryUrl, azureDevOpsBranch: branch, released: false, @@ -207,14 +218,16 @@ private async Task CreateBuildAsync(string repositoryUrl, string branch, return build; } - private List CreateAssetDataFromBuild(Build build) + private static List CreateAssetDataFromBuild(Build build) { - return build.Assets.Select(asset => new AssetData(false) - { - Name = asset.Name, - Version = asset.Version, - Locations = asset.Locations.Select(location => new AssetLocationData(location.Type) { Location = location.Location}).ToList() - }).ToList(); + return build.Assets + .Select(asset => new AssetData(false) + { + Name = asset.Name, + Version = asset.Version, + Locations = asset.Locations?.Select(location => new AssetLocationData(location.Type) { Location = location.Location}).ToList() + }) + .ToList(); } private async Task TriggerSubscriptionAsync(string subscriptionId) @@ -252,12 +265,13 @@ private async Task UpdateRemoteVmrForkFileAsync(string branch, string productRep logger.LogInformation("Updating file {file} on branch {branch} in the VMR fork", filePath, branch); // Fetch remote file and replace the product repo URI with the repo we're testing on var sourceMappingsFile = (await ghClient.Repository.Content.GetAllContentsByRef( - MaestroAuthTestOrgName, - VmrForkRepoName, - filePath, - branch)).FirstOrDefault() ?? - throw new Exception($"Failed to find file {SourceMappingsPath} in {MaestroAuthTestOrgName}" + - $"/{VmrForkRepoName} on branch {SourceMappingsPath}"); + MaestroAuthTestOrgName, + VmrForkRepoName, + filePath, + branch)) + .FirstOrDefault() + ?? throw new Exception($"Failed to find file {SourceMappingsPath} in {MaestroAuthTestOrgName}" + + $"/{VmrForkRepoName} on branch {SourceMappingsPath}"); // Replace the product repo uri with the forked one var updatedSourceMappings = sourceMappingsFile.Content.Replace(productRepoUri, productRepoForkUri); @@ -312,7 +326,7 @@ private async Task SyncForkAsync(string originOrg, string repoName, string branc private async Task> CreateTmpBranchAsync(string repoName, string originalBranch, bool skipCleanup) { - var newBranchName = $"repro/{Guid.NewGuid().ToString()}"; + var newBranchName = $"repro/{Guid.NewGuid()}"; logger.LogInformation("Creating temporary branch {branch} in {repo}", newBranchName, $"{MaestroAuthTestOrgName}/{repoName}"); var baseBranch = await ghClient.Git.Reference.Get(MaestroAuthTestOrgName, repoName, $"heads/{originalBranch}"); diff --git a/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproToolConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproToolConfiguration.cs index e90611054f..f3c467e7e6 100644 --- a/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproToolConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproToolConfiguration.cs @@ -15,6 +15,7 @@ using Microsoft.DotNet.ProductConstructionService.Client; using GitHubClient = Octokit.GitHubClient; using Octokit; +using Microsoft.Extensions.Logging.Console; namespace ProductConstructionService.ReproTool; internal static class ReproToolConfiguration @@ -23,10 +24,15 @@ internal static class ReproToolConfiguration private const string MaestroProdUri = "https://maestro.dot.net"; internal const string PcsLocalUri = "https://localhost:53180"; - internal static ServiceCollection RegisterServices(this ServiceCollection services, ReproToolOptions options) + internal static ServiceCollection RegisterServices( + this ServiceCollection services, + ReproToolOptions options) { services.AddSingleton(options); - services.AddLogging(builder => builder.AddConsole()); + services.AddLogging(b => b + .AddConsole(o => o.FormatterName = CompactConsoleLoggerFormatter.FormatterName) + .AddConsoleFormatter() + .SetMinimumLevel(LogLevel.Information)); services.AddSingleton(sp => sp.GetRequiredService>()); services.AddSingleton(sp => new BarApiClient( diff --git a/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproToolOptions.cs b/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproToolOptions.cs index 6dbcc5e403..f520e96a71 100644 --- a/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproToolOptions.cs +++ b/src/ProductConstructionService/ProductConstructionService.ReproTool/ReproToolOptions.cs @@ -10,8 +10,8 @@ internal class ReproToolOptions [Option('s', "subscription", HelpText = "Subscription that's getting reproduced", Required = true)] public required string Subscription { get; init; } - [Option("github-token", HelpText = "GitHub token", Required = true)] - public required string GitHubToken { get; init; } + [Option("github-token", HelpText = "GitHub token", Required = false)] + public string? GitHubToken { get; set; } [Option("commit", HelpText = "Commit to flow. Use when not flowing a build. If neither commit or build is specified, the latest commit in the subscription's source repository is flown", Required = false)] public string? Commit { get; init; } diff --git a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScope.cs b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScope.cs index 71dabec87d..b7bb5d126f 100644 --- a/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScope.cs +++ b/src/ProductConstructionService/ProductConstructionService.WorkItems/WorkItemScope.cs @@ -60,8 +60,8 @@ public async Task RunWorkItemAsync(JsonNode node, CancellationToken cancellation async Task ProcessWorkItemAsync() { - using (ITelemetryScope telemetryScope = _telemetryRecorder.RecordWorkItemCompletion(type)) using (var operation = telemetryClient.StartOperation(type)) + using (ITelemetryScope telemetryScope = _telemetryRecorder.RecordWorkItemCompletion(type)) using (logger.BeginScope(processor.GetLoggingContextData(workItem))) { try @@ -71,6 +71,11 @@ async Task ProcessWorkItemAsync() if (success) { telemetryScope.SetSuccess(); + logger.LogInformation("Work item {type} processed successfully", type); + } + else + { + logger.LogInformation("Work item {type} processed unsuccessfully", type); } } catch (Exception e) diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrBackflowTest.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrBackflowTest.cs index f1e545b5be..3403141c8f 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrBackflowTest.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrBackflowTest.cs @@ -27,10 +27,6 @@ public async Task OnlyBackflowsTest() var hadUpdates = await ChangeVmrFileAndFlowIt("New content from the VMR", branchName); hadUpdates.ShouldHaveUpdates(); - // Verify that the update dependencies commit got amended - var commitMessage = await GitOperations.GetRepoLastCommitMessage(ProductRepoPath); - commitMessage.Should().StartWith("[VMR] Codeflow"); - await GitOperations.MergePrBranch(ProductRepoPath, branchName); CheckFileContents(_productRepoFilePath, "New content from the VMR"); // Backflow again - should be a no-op @@ -150,21 +146,23 @@ await File.WriteAllTextAsync(ProductRepoPath / VersionFiles.VersionProps, """); + Directory.CreateDirectory(ArcadeInVmrPath); + await File.WriteAllTextAsync(ArcadeInVmrPath / VersionFiles.GlobalJson, Constants.GlobalJsonTemplate); + // Level the repo and the VMR await GitOperations.CommitAll(ProductRepoPath, "Changing version files"); - var repoSha = await GitOperations.GetRepoLastCommit(ProductRepoPath); var hadUpdates = await CallDarcForwardflow(Constants.ProductRepoName, ProductRepoPath, branchName); hadUpdates.ShouldHaveUpdates(); await GitOperations.MergePrBranch(VmrPath, branchName); // Update global.json in the VMR - var updatedGlobalJson = await File.ReadAllTextAsync(VmrPath / VersionFiles.GlobalJson); - await File.WriteAllTextAsync(VmrPath / VersionFiles.GlobalJson, updatedGlobalJson.Replace("9.0.100", "9.0.200")); + var updatedGlobalJson = await File.ReadAllTextAsync(ArcadeInVmrPath / VersionFiles.GlobalJson); + await File.WriteAllTextAsync(ArcadeInVmrPath / VersionFiles.GlobalJson, updatedGlobalJson.Replace("9.0.100", "9.0.200")); // Update an eng/common file in the VMR - Directory.CreateDirectory(VmrPath / DarcLib.Constants.CommonScriptFilesPath); - await File.WriteAllTextAsync(VmrPath / DarcLib.Constants.CommonScriptFilesPath / "darc-init.ps1", "Some other script file"); + Directory.CreateDirectory(ArcadeInVmrPath / DarcLib.Constants.CommonScriptFilesPath); + await File.WriteAllTextAsync(ArcadeInVmrPath / DarcLib.Constants.CommonScriptFilesPath / "darc-init.ps1", "Some other script file"); await GitOperations.CommitAll(VmrPath, "Changing a VMR's global.json and eng/common file"); @@ -290,11 +288,11 @@ .. GetExpectedVersionFiles(ProductRepoPath), // Verify that global.json got updated DependencyFileManager dependencyFileManager = GetDependencyFileManager(); - JObject globalJson = await dependencyFileManager.ReadGlobalJsonAsync(ProductRepoPath, branchName + "-pr"); + JObject globalJson = await dependencyFileManager.ReadGlobalJsonAsync(ProductRepoPath, branchName + "-pr", repoIsVmr: false); JToken? arcadeVersion = globalJson.SelectToken($"msbuild-sdks.['{DependencyFileManager.ArcadeSdkPackageName}']", true); arcadeVersion?.ToString().Should().Be("1.0.2"); - var dotnetVersion = await dependencyFileManager.ReadToolsDotnetVersionAsync(ProductRepoPath, branchName + "-pr"); + var dotnetVersion = await dependencyFileManager.ReadToolsDotnetVersionAsync(ProductRepoPath, branchName + "-pr", repoIsVmr: false); dotnetVersion.ToString().Should().Be("9.0.200"); await GitOperations.MergePrBranch(ProductRepoPath, branchName + "-pr"); @@ -411,8 +409,9 @@ public async Task BackflowingSubsequentCommitsTest() await EnsureTestRepoIsInitialized(); // Update an eng/common file in the VMR - Directory.CreateDirectory(VmrPath / DarcLib.Constants.CommonScriptFilesPath); - await File.WriteAllTextAsync(VmrPath / DarcLib.Constants.CommonScriptFilesPath / "darc-init.ps1", "Some other script file"); + Directory.CreateDirectory(ArcadeInVmrPath / DarcLib.Constants.CommonScriptFilesPath); + await File.WriteAllTextAsync(ArcadeInVmrPath / VersionFiles.GlobalJson, Constants.GlobalJsonTemplate); + await File.WriteAllTextAsync(ArcadeInVmrPath / DarcLib.Constants.CommonScriptFilesPath / "darc-init.ps1", "Some other script file"); await GitOperations.CommitAll(VmrPath, "Creating VMR's eng/common"); await File.WriteAllTextAsync(ProductRepoPath / VersionFiles.VersionDetailsXml, @@ -489,5 +488,83 @@ await File.WriteAllTextAsync(ProductRepoPath / VersionFiles.VersionProps, dependencies = await productRepo.GetDependenciesAsync(); dependencies.Should().BeEquivalentTo(GetDependencies(build2)); } + + [Test] + public async Task BackflowingCorrectEngCommonTest() + { + const string branchName = nameof(BackflowingDependenciesTest); + + await EnsureTestRepoIsInitialized(); + + // Setup product repo with an arcade dependency + await File.WriteAllTextAsync(ProductRepoPath / VersionFiles.VersionDetailsXml, + $""" + + + + + + https://github.com/dotnet/arcade + a01 + + + + + + """); + + await File.WriteAllTextAsync(ProductRepoPath / VersionFiles.VersionProps, + $""" + + + + $(MSBuildAllProjects);$(MSBuildThisFileFullPath) + + + 9.0.100 + + + + <{VersionFiles.GetVersionPropsPackageVersionElementName(DependencyFileManager.ArcadeSdkPackageName)}>1.0.0 + + + + """); + + await GitOperations.CommitAll(ProductRepoPath, "Changing version files"); + + var hadUpdates = await CallDarcForwardflow(Constants.ProductRepoName, ProductRepoPath, branchName); + hadUpdates.ShouldHaveUpdates(); + await GitOperations.MergePrBranch(VmrPath, branchName); + + string baseRepoFileName = "a.txt"; + string arcadeRepoFileName = "b.txt"; + // create eng/common in the VMR in / and in /src/arcade + Directory.CreateDirectory(VmrPath / DarcLib.Constants.CommonScriptFilesPath); + await File.WriteAllTextAsync(VmrPath / DarcLib.Constants.CommonScriptFilesPath / baseRepoFileName, "Not important"); + + Directory.CreateDirectory(ArcadeInVmrPath / DarcLib.Constants.CommonScriptFilesPath); + await File.WriteAllTextAsync(ArcadeInVmrPath / VersionFiles.GlobalJson, Constants.GlobalJsonTemplate); + await File.WriteAllTextAsync(ArcadeInVmrPath / DarcLib.Constants.CommonScriptFilesPath / arcadeRepoFileName, "Not important"); + + await GitOperations.CommitAll(VmrPath, "Creating test eng/commons"); + + var build1 = await CreateNewVmrBuild( + [ + (DependencyFileManager.ArcadeSdkPackageName, "1.0.1") + ]); + + // Flow changes back from the VMR + hadUpdates = await CallDarcBackflow( + Constants.ProductRepoName, + ProductRepoPath, + branchName + "-backflow", + buildToFlow: build1); + hadUpdates.ShouldHaveUpdates(); + await GitOperations.MergePrBranch(ProductRepoPath, branchName + "-backflow"); + + // Verify that the product repo has the eng/common from src/arcade + CheckFileContents(ProductRepoPath / DarcLib.Constants.CommonScriptFilesPath / arcadeRepoFileName, "Not important"); + } } diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrCodeFlowTests.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrCodeFlowTests.cs index fb29dc57a4..0851caafaa 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrCodeFlowTests.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrCodeFlowTests.cs @@ -80,27 +80,13 @@ await GetLocal(ProductRepoPath).AddDependencyAsync(new DependencyDetail await GitOperations.CommitAll(ProductRepoPath, "Adding Arcade dependency"); - // We also add Arcade SDK to VMR so that we can verify eng/common updates - await GetLocal(VmrPath).AddDependencyAsync(new DependencyDetail - { - Name = DependencyFileManager.ArcadeSdkPackageName, - Version = "1.0.0", - RepoUri = VmrPath, - Commit = vmrSha, - Type = DependencyType.Toolset, - Pinned = false, - }); - - await GitOperations.CommitAll(VmrPath, "Adding Arcade to the VMR"); - await InitializeRepoAtLastCommit(Constants.ProductRepoName, ProductRepoPath); await GitOperations.Checkout(ProductRepoPath, "main"); var expectedFiles = GetExpectedFilesInVmr( VmrPath, [Constants.ProductRepoName], - [_productRepoVmrFilePath, _productRepoVmrPath / DarcLib.Constants.CommonScriptFilesPath / "build.ps1"], - hasVersionFiles: true); + [_productRepoVmrFilePath, _productRepoVmrPath / DarcLib.Constants.CommonScriptFilesPath / "build.ps1"]); CheckDirectoryContents(VmrPath, expectedFiles); CompareFileContents(_productRepoVmrFilePath, _productRepoFileName); @@ -166,7 +152,8 @@ protected override async Task CopyReposForCurrentTest() protected override async Task CopyVmrForCurrentTest() { - await CopyRepoAndCreateVersionFiles("vmr"); + var repoPath = CurrentTestDirectory / "vmr"; + CopyDirectory(VmrTestsOneTimeSetUp.TestsDirectory / "vmr", repoPath); var sourceMappings = new SourceMappingFile() { diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrForwardFlowTest.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrForwardFlowTest.cs index 0b0f1b2f3c..3e3d0fa792 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrForwardFlowTest.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrForwardFlowTest.cs @@ -2,11 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.IO; -using System.Linq; using System.Threading.Tasks; -using FluentAssertions; -using Microsoft.DotNet.DarcLib.Helpers; -using Microsoft.DotNet.DarcLib.Models.Darc; using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using NUnit.Framework; @@ -56,81 +52,5 @@ await GitOperations.VerifyMergeConflict(VmrPath, branchName, hadUpdates = await CallDarcBackflow(Constants.ProductRepoName, ProductRepoPath, branchName); CheckFileContents(_productRepoVmrFilePath, "A completely different change"); } - - [Test] - public async Task ForwardFlowingDependenciesTest() - { - const string branchName = nameof(ForwardFlowingDependenciesTest); - - await EnsureTestRepoIsInitialized(); - - var vmrSha = await GitOperations.GetRepoLastCommit(VmrPath); - - await GetLocal(VmrPath).AddDependencyAsync(new DependencyDetail - { - Name = "Package.A1", - Version = "1.0.0", - RepoUri = ProductRepoPath, - Commit = "123abc", - Type = DependencyType.Product, - Pinned = false, - }); - await GitOperations.CommitAll(VmrPath, "Added Package.A1 dependency"); - - // Flow a build into the VMR - await GitOperations.Checkout(ProductRepoPath, "main"); - await File.WriteAllTextAsync(_productRepoFilePath, "New content in the repository"); - await GitOperations.CommitAll(ProductRepoPath, "Changing a repo file"); - - var build1 = await CreateNewRepoBuild( - [ - ("Package.A1", "1.0.1"), - ]); - - var hadUpdates = await CallDarcForwardflow(Constants.ProductRepoName, ProductRepoPath, branchName, buildToFlow: build1); - hadUpdates.ShouldHaveUpdates(); - await GitOperations.MergePrBranch(VmrPath, branchName); - - // Verify that VMR's version files have the new versions - var vmr = GetLocal(VmrPath); - var dependencies = await vmr.GetDependenciesAsync(); - dependencies.Where(d => d.Name != DependencyFileManager.ArcadeSdkPackageName) - .Should().BeEquivalentTo(GetDependencies(build1)); - - // Now we will change something in the repo and flow it to the VMR - // Then we will change something in the repo again but before we flow it, we will make a conflicting change in the PR branch - await File.WriteAllTextAsync(_productRepoFilePath, "New content again in the repo #1"); - await GitOperations.CommitAll(ProductRepoPath, "Changing a repo file again #1"); - - var build2 = await CreateNewRepoBuild( - [ - ("Package.A1", "1.0.5"), - ]); - - await File.WriteAllTextAsync(_productRepoFilePath, "New content again in the repo #2"); - await GitOperations.CommitAll(ProductRepoPath, "Changing a repo file again #2"); - - var build3 = await CreateNewRepoBuild( - [ - ("Package.A1", "1.0.6"), - ]); - - // Flow the first build - hadUpdates = await CallDarcForwardflow(Constants.ProductRepoName, ProductRepoPath, branchName, buildToFlow: build2); - hadUpdates.ShouldHaveUpdates(); - - // We make a conflicting change in the PR branch - await GitOperations.Checkout(VmrPath, branchName); - await File.WriteAllTextAsync(_productRepoVmrFilePath, "New content again but this time in the PR directly"); - await GitOperations.CommitAll(VmrPath, "Changing a file in the PR"); - - // Flow the second build - this should throw as there's a conflict in the PR branch - await this.Awaiting(_ => CallDarcForwardflow(Constants.ProductRepoName, ProductRepoPath, branchName, buildToFlow: build3)) - .Should().ThrowAsync(); - - // The state of the branch should be the same as before - vmr.Checkout(branchName); - CheckFileContents(_productRepoVmrFilePath, "New content again but this time in the PR directly"); - } } diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs index a2d92333ea..6e131ee1f2 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTestsBase.cs @@ -32,6 +32,7 @@ internal abstract class VmrTestsBase protected NativePath DependencyRepoPath { get; private set; } = null!; protected NativePath SyncDisabledRepoPath { get; private set; } = null!; protected NativePath InstallerRepoPath { get; private set; } = null!; + protected NativePath ArcadeInVmrPath { get; private set;} = null!; protected GitOperationsHelper GitOperations { get; } = new(); protected IServiceProvider ServiceProvider { get; private set; } = null!; @@ -54,6 +55,7 @@ public async Task Setup() DependencyRepoPath = CurrentTestDirectory / Constants.DependencyRepoName; InstallerRepoPath = CurrentTestDirectory / Constants.InstallerRepoName; SyncDisabledRepoPath = CurrentTestDirectory / Constants.SyncDisabledRepoName; + ArcadeInVmrPath = VmrPath / VmrInfo.SourcesDir / "arcade"; Directory.CreateDirectory(TmpPath); @@ -94,8 +96,7 @@ public void DeleteCurrentTestDirectory() protected static List GetExpectedFilesInVmr( NativePath vmrPath, string[] reposWithVersionFiles, - List reposFiles, - bool hasVersionFiles = false) + List reposFiles) { List expectedFiles = [ @@ -111,11 +112,6 @@ protected static List GetExpectedFilesInVmr( expectedFiles.AddRange(reposFiles); - if (hasVersionFiles) - { - expectedFiles.AddRange(GetExpectedVersionFiles(vmrPath)); - } - return expectedFiles; } @@ -341,9 +337,7 @@ protected async Task CopyRepoAndCreateVersionFiles( var versionProps = string.Format(Constants.VersionPropsTemplate, propsString); File.WriteAllText(repoPath / VersionFiles.VersionProps, versionProps); - - File.WriteAllText(repoPath / VersionFiles.GlobalJson, Constants.GlobalJsonTemplate); - + File.WriteAllText(repoPath/ VersionFiles.GlobalJson, Constants.GlobalJsonTemplate); File.WriteAllText(repoPath / VersionFiles.NugetConfig, Constants.NuGetConfigTemplate); await GitOperations.CommitAll(repoPath, "Update version files"); diff --git a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTwoWayCodeflowTest.cs b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTwoWayCodeflowTest.cs index 7a64a264cc..a232746498 100644 --- a/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTwoWayCodeflowTest.cs +++ b/test/Microsoft.DotNet.Darc.VirtualMonoRepo.E2E.Tests/VmrTwoWayCodeflowTest.cs @@ -112,6 +112,7 @@ public async Task SubmoduleCodeFlowTest() } // This one simulates what would happen if PR both ways are open and the one that was open later merges first. + // In this case, a conflict in the version files will have to be auto-resolved. // The diagram it follows is here (O are commits): /* repo VMR @@ -121,20 +122,20 @@ repo VMR │ │ 4. │ 3.O───┼────────────►O │ │ │ │ │ - │ ┌─┘ │ │ - │ │ │ │ - 5.O◄┘ └──►O 6. - │ │ - |────────────────────►O 7. - │ │ - */ + │ │ │ │ + 5.O◄──┘ └──►O 6. + │ 7. │ + |────────────────►O │ + │ └──►O 8. + │ │ + */ [Test] - public async Task OutOfOrderMergesTest() + public async Task ForwardFlowConflictResolutionTest() { await EnsureTestRepoIsInitialized(); - const string backBranchName = nameof(OutOfOrderMergesTest); - const string forwardBranchName = nameof(OutOfOrderMergesTest) + "-ff"; + const string backBranchName = nameof(ForwardFlowConflictResolutionTest); + const string forwardBranchName = nameof(ForwardFlowConflictResolutionTest) + "-ff"; // 1. Change file in VMR await File.WriteAllTextAsync(_productRepoVmrPath / "1a.txt", "one"); @@ -177,7 +178,102 @@ public async Task OutOfOrderMergesTest() await GitOperations.Checkout(VmrPath, "main"); hadUpdates = await CallDarcForwardflow(Constants.ProductRepoName, ProductRepoPath, branch: forwardBranchName); hadUpdates.ShouldHaveUpdates(); - await GitOperations.VerifyMergeConflict(VmrPath, forwardBranchName, mergeTheirs: true); + + // 8. Merge the forward flow PR - any conflicts in version files are dealt with automatically + // The conflict is described in the ForwardFlowConflictResolver class + await GitOperations.MergePrBranch(VmrPath, forwardBranchName); + + // Both VMR and repo need to have the version from the VMR as it flowed to the repo and back + (string, string)[] expectedFiles = + [ + ("1a.txt", "one"), + ("1b.txt", "one again"), + ("3a.txt", "three"), + ("3b.txt", "three again"), + ]; + + foreach (var (file, content) in expectedFiles) + { + CheckFileContents(_productRepoVmrPath / file, content); + CheckFileContents(ProductRepoPath / file, content); + } + + await GitOperations.CheckAllIsCommitted(VmrPath); + await GitOperations.CheckAllIsCommitted(ProductRepoPath); + } + + // This one simulates what would happen if PR both ways are open and the one that was open later merges first. + // In this case, a conflict in the version files will have to be auto-resolved. + // The diagram it follows is here (O are commits): + /* + repo VMR + O────────────────────►O + │ 2. │ + 1.O────────────────O │ + │ 4. │ │ + │ O───────────┼────O 3. + │ │ │ │ + │ │ │ │ + 6.O◄───┘ └───►O 5. + │ 7. │ + │ O───────────────| + 8.O◄────┘ │ + │ │ + */ + [Test] + public async Task BackwardFlowConflictResolutionTest() + { + await EnsureTestRepoIsInitialized(); + + const string backBranchName = nameof(BackwardFlowConflictResolutionTest); + const string forwardBranchName = nameof(BackwardFlowConflictResolutionTest) + "-ff"; + + // 1. Change file in the repo + await File.WriteAllTextAsync(ProductRepoPath / "1a.txt", "one"); + await GitOperations.CommitAll(ProductRepoPath, "1a.txt"); + + // 2. Open a forward flow PR + var hadUpdates = await CallDarcForwardflow(Constants.ProductRepoName, ProductRepoPath, forwardBranchName); + hadUpdates.ShouldHaveUpdates(); + // We make another commit in the repo and add it to the PR branch (this is not in the diagram above) + await GitOperations.Checkout(ProductRepoPath, "main"); + await File.WriteAllTextAsync(ProductRepoPath / "1b.txt", "one again"); + await GitOperations.CommitAll(ProductRepoPath, "1b.txt"); + hadUpdates = await CallDarcForwardflow(Constants.ProductRepoName, ProductRepoPath, forwardBranchName); + hadUpdates.ShouldHaveUpdates(); + + // 3. Change file in the VMR + await GitOperations.Checkout(VmrPath, "main"); + await File.WriteAllTextAsync(_productRepoVmrPath / "3a.txt", "three"); + await GitOperations.CommitAll(VmrPath, "3a.txt"); + + // 4. Open a backflow PR + hadUpdates = await CallDarcBackflow(Constants.ProductRepoName, ProductRepoPath, backBranchName); + hadUpdates.ShouldHaveUpdates(); + // We make another commit in the repo and add it to the PR branch (this is not in the diagram above) + await GitOperations.Checkout(VmrPath, "main"); + await File.WriteAllTextAsync(_productRepoVmrPath / "3b.txt", "three again"); + await GitOperations.CommitAll(VmrPath, "3b.txt"); + hadUpdates = await CallDarcBackflow(Constants.ProductRepoName, ProductRepoPath, backBranchName); + hadUpdates.ShouldHaveUpdates(); + + // 5. Merge the forward flow PR + await GitOperations.MergePrBranch(VmrPath, forwardBranchName); + + // 6. Merge the backflow PR + await GitOperations.MergePrBranch(ProductRepoPath, backBranchName); + + // 7. Flow back again so the VMR version of the file will flow back to the repo + await GitOperations.Checkout(ProductRepoPath, "main"); + await GitOperations.Checkout(VmrPath, "main"); + hadUpdates = await CallDarcBackflow(Constants.ProductRepoName, ProductRepoPath, branch: backBranchName); + hadUpdates.ShouldHaveUpdates(); + + // 8. Merge the forward flow PR - any conflicts in version files are dealt with automatically + // The conflict is described in the BackwardFlowConflictResolver class + // TODO https://github.com/dotnet/arcade-services/issues/4196: The conflict should get resolved automatically + // await GitOperations.MergePrBranch(ProductRepoPath, backBranchName); + await GitOperations.VerifyMergeConflict(ProductRepoPath, backBranchName, "eng/Version.Details.xml", mergeTheirs: true); // Both VMR and repo need to have the version from the VMR as it flowed to the repo and back (string, string)[] expectedFiles = @@ -222,8 +318,8 @@ public async Task OutOfOrderMergesWithConflictsTest() const string aFileContent = "Added a new file in the repo"; const string bFileContent = "Added a new file in the VMR"; - const string backBranchName = nameof(OutOfOrderMergesTest); - const string forwardBranchName = nameof(OutOfOrderMergesTest) + "-ff"; + const string backBranchName = nameof(OutOfOrderMergesWithConflictsTest); + const string forwardBranchName = nameof(OutOfOrderMergesWithConflictsTest) + "-ff"; // 1. Change file in VMR // 2. Open a backflow PR diff --git a/test/ProductConstructionService.DependencyFlow.Tests/UpdaterTests.cs b/test/ProductConstructionService.DependencyFlow.Tests/UpdaterTests.cs index 4cef21dd49..86020f1a18 100644 --- a/test/ProductConstructionService.DependencyFlow.Tests/UpdaterTests.cs +++ b/test/ProductConstructionService.DependencyFlow.Tests/UpdaterTests.cs @@ -81,6 +81,15 @@ public void UpdaterTests_SetUp() [TearDown] public void UpdaterTests_TearDown() { + foreach (var pair in Cache.Data) + { + if (pair.Value is InProgressPullRequest pr) + { + pr.LastCheck = (ExpectedCacheState[pair.Key] as InProgressPullRequest)!.LastCheck; + pr.LastUpdate = (ExpectedCacheState[pair.Key] as InProgressPullRequest)!.LastUpdate; + pr.NextCheck = (ExpectedCacheState[pair.Key] as InProgressPullRequest)!.NextCheck; + } + } Cache.Data.Should().BeEquivalentTo(ExpectedCacheState); Reminders.Reminders.Should().BeEquivalentTo(ExpectedReminders); } diff --git a/test/ProductConstructionService.ScenarioTests/CodeFlowScenarioTestBase.cs b/test/ProductConstructionService.ScenarioTests/CodeFlowScenarioTestBase.cs index 10ebc82bd9..a8ef31add7 100644 --- a/test/ProductConstructionService.ScenarioTests/CodeFlowScenarioTestBase.cs +++ b/test/ProductConstructionService.ScenarioTests/CodeFlowScenarioTestBase.cs @@ -15,8 +15,6 @@ protected async Task CheckForwardFlowGitHubPullRequest( string[] testFiles, Dictionary testFilePatches) { - var expectedPRTitle = GetCodeFlowPRName(targetBranch, sourceRepoName); - PullRequest pullRequest = await WaitForPullRequestAsync(targetRepoName, targetBranch); await using (CleanUpPullRequestAfter(TestParameters.GitHubTestOrg, targetRepoName, pullRequest)) @@ -51,8 +49,6 @@ protected async Task CheckBackwardFlowGitHubPullRequest( string commitSha, int buildId) { - var expectedPRTitle = GetCodeFlowPRName(targetBranch, sourceRepoName); - PullRequest pullRequest = await WaitForPullRequestAsync(targetRepoName, targetBranch); await using (CleanUpPullRequestAfter(TestParameters.GitHubTestOrg, targetRepoName, pullRequest)) diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs index 09e22ced1b..34cf5d75a3 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTestBase.cs @@ -262,7 +262,6 @@ protected async Task CheckNonBatchedGitHubPullRequest( await CheckGitHubPullRequest(expectedPRTitle, targetRepoName, targetBranch, expectedDependencies, repoDirectory, isCompleted, isUpdated, cleanUp); } - protected static string GetCodeFlowPRName(string targetBranch, string sourceRepoName) => $"[{targetBranch}] Source code changes from {TestParameters.GitHubTestOrg}/{sourceRepoName}"; protected static string GetExpectedCodeFlowDependencyVersionEntry(string repo, string sha, int buildId) => $"Source Uri=\"{GetGitHubRepoUrl(repo)}\" Sha=\"{sha}\" BarId=\"{buildId}\" />"; @@ -1052,4 +1051,20 @@ protected static IAsyncDisposable CleanUpPullRequestAfter(string owner, string r // Closed already } }); + + protected async Task CreateTargetBranchAndExecuteTest(string targetBranchName, TemporaryDirectory targetDirectory, Func test) + { + // first create a new target branch + using (ChangeDirectory(targetDirectory.Directory)) + { + await using (await CheckoutBranchAsync(targetBranchName)) + { + // and push it to GH + await using (await PushGitBranchAsync("origin", targetBranchName)) + { + await test(); + } + } + } + } } diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests/ScenarioTests_CodeFlow.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests/ScenarioTests_CodeFlow.cs index 04fae2bc79..b9b5dc2a11 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTests/ScenarioTests_CodeFlow.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests/ScenarioTests_CodeFlow.cs @@ -157,20 +157,4 @@ await CreateTargetBranchAndExecuteTest(targetBranchName, testRepoFolder, async ( } }); } - - private async Task CreateTargetBranchAndExecuteTest(string targetBranchName, TemporaryDirectory targetDirectory, Func test) - { - // first create a new target branch in the VMR - using (ChangeDirectory(targetDirectory.Directory)) - { - await using (await CheckoutBranchAsync(targetBranchName)) - { - // and push it to GH - await using (await PushGitBranchAsync("origin", targetBranchName)) - { - await test(); - } - } - } - } } diff --git a/test/ProductConstructionService.ScenarioTests/ScenarioTests/ScenarioTests_SdkUpdate.cs b/test/ProductConstructionService.ScenarioTests/ScenarioTests/ScenarioTests_SdkUpdate.cs index d7a5969827..aa7d729047 100644 --- a/test/ProductConstructionService.ScenarioTests/ScenarioTests/ScenarioTests_SdkUpdate.cs +++ b/test/ProductConstructionService.ScenarioTests/ScenarioTests/ScenarioTests_SdkUpdate.cs @@ -6,6 +6,7 @@ using Microsoft.DotNet.DarcLib.Models.Darc; using NUnit.Framework; using Microsoft.DotNet.ProductConstructionService.Client.Models; +using Octokit; namespace ProductConstructionService.ScenarioTests; @@ -138,4 +139,91 @@ await RunDarcAsync("add-dependency", } } } + + // This test verifies that we're able to flow eng/common and global.json during Arcade SDK updates from the VMR + [Test] + public async Task ArcadeSdkVmrUpdate_E2E() + { + var testChannelName = GetTestChannelName(); + var targetBranch = GetTestBranchName(); + var vmrBranch = GetTestBranchName(); + + const string sourceRepo = "maestro-test-vmr"; + const string sourceRepoUri = $"https://github.com/{TestRepository.TestOrg}/{sourceRepo}"; + const string sourceBranch = "dependencyflow-tests"; + const string newArcadeSdkVersion = "2.1.0"; + const string arcadeEngCommonPath = "src/arcade/eng/common"; + const string engCommonFile = "file.txt"; + const string arcadeGlobalJsonPath = "src/arcade/global.json"; + + const string globalJsonFile = """ + { + "tools": { + "dotnet": "2.2.203" + }, + "msbuild-sdks": { + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19251.6", + "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19251.6" + } + } + """; + var sourceBuildNumber = _random.Next(int.MaxValue).ToString(); + + List sourceAssets = + [ + new AssetData(true) + { + Name = DependencyFileManager.ArcadeSdkPackageName, + Version = newArcadeSdkVersion + } + ]; + + await using AsyncDisposableValue channel = await CreateTestChannelAsync(testChannelName); + await using AsyncDisposableValue sub = + await CreateSubscriptionAsync(testChannelName, sourceRepo, TestRepository.TestRepo1Name, targetBranch, "none", TestRepository.TestOrg); + + TemporaryDirectory testRepoFolder = await CloneRepositoryAsync(TestRepository.TestRepo1Name); + TemporaryDirectory vmrFolder = await CloneRepositoryAsync(TestRepository.VmrTestRepoName); + + await CreateTargetBranchAndExecuteTest(targetBranch, testRepoFolder, async () => + { + using (ChangeDirectory(vmrFolder.Directory)) + { + await using (await CheckoutBranchAsync(vmrBranch)) + { + // Create an arcade repo in the VMR + Directory.CreateDirectory(arcadeEngCommonPath); + await File.WriteAllTextAsync(Path.Combine(arcadeEngCommonPath, engCommonFile), "test"); + await File.WriteAllTextAsync(arcadeGlobalJsonPath, globalJsonFile); + + await GitAddAllAsync(); + await GitCommitAsync("Add arcade files"); + + var repoSha = (await GitGetCurrentSha()).TrimEnd(); + Build build = await CreateBuildAsync(GetRepoUrl(TestRepository.TestOrg, sourceRepo), sourceBranch, repoSha, sourceBuildNumber, sourceAssets); + + await using IAsyncDisposable _ = await AddBuildToChannelAsync(build.Id, testChannelName); + + // and push it to GH + await using (await PushGitBranchAsync("origin", vmrBranch)) + { + await TriggerSubscriptionAsync(sub.Value); + + var expectedTitle = $"[{targetBranch}] Update dependencies from {TestRepository.TestOrg}/{sourceRepo}"; + + PullRequest pullRequest = await WaitForPullRequestAsync(TestRepository.TestRepo1Name, targetBranch); + + await using (CleanUpPullRequestAfter(TestParameters.GitHubTestOrg, TestRepository.TestRepo1Name, pullRequest)) + { + IReadOnlyList files = await GitHubApi.PullRequest.Files(TestParameters.GitHubTestOrg, TestRepository.TestRepo1Name, pullRequest.Number); + + files.Should().Contain(files => files.FileName == "global.json"); + files.Should().Contain(files => files.FileName == "eng/common/file.txt"); + } + } + } + } + }); + + } }