diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 5aa9bf38b7..a3f4123928 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "microsoft.dnceng.secretmanager": { - "version": "1.1.0-beta.24321.4", + "version": "1.1.0-beta.24325.1", "commands": [ "secret-manager" ] @@ -15,7 +15,7 @@ ] }, "microsoft.dnceng.configuration.bootstrap": { - "version": "1.1.0-beta.24321.4", + "version": "1.1.0-beta.24325.1", "commands": [ "bootstrap-dnceng-configuration" ] diff --git a/Directory.Packages.props b/Directory.Packages.props index 85ed3abd0d..db97a9b60a 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -20,10 +20,11 @@ - + + @@ -128,7 +129,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f17b8329a5..d45cb5bb23 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,127 +1,127 @@ - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/dnceng-shared - 77c5d1699eba47a9fd73a111cabc56a0f4bb7c3f + 5f8639a65b7e5333bb4a4f5c629e5384c6eb7dc2 - + https://github.com/dotnet/arcade - 748cd976bf8b0f69b809e569943635ab8be36dc8 + 761c516b64fee3941d8909d24205ced835eed83e - + https://github.com/dotnet/arcade - 748cd976bf8b0f69b809e569943635ab8be36dc8 + 761c516b64fee3941d8909d24205ced835eed83e - + https://github.com/dotnet/arcade - 748cd976bf8b0f69b809e569943635ab8be36dc8 + 761c516b64fee3941d8909d24205ced835eed83e - + https://github.com/dotnet/arcade - 748cd976bf8b0f69b809e569943635ab8be36dc8 + 761c516b64fee3941d8909d24205ced835eed83e - + https://github.com/dotnet/arcade - 748cd976bf8b0f69b809e569943635ab8be36dc8 + 761c516b64fee3941d8909d24205ced835eed83e - + https://github.com/dotnet/arcade - 748cd976bf8b0f69b809e569943635ab8be36dc8 + 761c516b64fee3941d8909d24205ced835eed83e - + https://github.com/dotnet/dnceng - 44c25f86ba374d93ba9c22f12552deeed2221588 + b6850078ffe074f4ad7d4952486d4997b1e029b6 - + https://github.com/dotnet/dnceng - 44c25f86ba374d93ba9c22f12552deeed2221588 + b6850078ffe074f4ad7d4952486d4997b1e029b6 diff --git a/eng/Versions.props b/eng/Versions.props index 1e39a902b6..a60051b5ea 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -9,36 +9,36 @@ true 1.0.0-preview.1 - 8.0.0-beta.24324.1 - 8.0.0-beta.24324.1 - 8.0.0-beta.24324.1 - 8.0.0-beta.24324.1 - 8.0.0-beta.24324.1 + 8.0.0-beta.24328.2 + 8.0.0-beta.24328.2 + 8.0.0-beta.24328.2 + 8.0.0-beta.24328.2 + 8.0.0-beta.24328.2 17.4.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24319.1 - 1.1.0-beta.24321.4 - 1.1.0-beta.24321.4 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24353.1 + 1.1.0-beta.24325.1 + 1.1.0-beta.24325.1 diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 5a3a32ea8d..238945cb5a 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -2,7 +2,6 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $PublishingInfraVersion, [Parameter(Mandatory=$true)][string] $AzdoToken, - [Parameter(Mandatory=$true)][string] $MaestroToken, [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, @@ -31,13 +30,13 @@ try { } & $darc add-build-to-channel ` - --id $buildId ` - --publishing-infra-version $PublishingInfraVersion ` - --default-channels ` - --source-branch main ` - --azdev-pat $AzdoToken ` - --bar-uri $MaestroApiEndPoint ` - --password $MaestroToken ` + --id $buildId ` + --publishing-infra-version $PublishingInfraVersion ` + --default-channels ` + --source-branch main ` + --azdev-pat "$AzdoToken" ` + --bar-uri "$MaestroApiEndPoint" ` + --ci ` @optionalParams if ($LastExitCode -ne 0) { diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 589ac80a18..d01739c128 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -76,13 +76,16 @@ jobs: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Build Assets inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: > + -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) @@ -144,7 +147,6 @@ jobs: arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index da1f40958b..0dfa387e7b 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -272,14 +272,16 @@ stages: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 8ec0151def..9fd69fa7c9 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -74,13 +74,16 @@ jobs: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Build Assets inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: > + -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) @@ -140,7 +143,6 @@ jobs: arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index aba44a25a3..2db4933468 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -268,14 +268,16 @@ stages: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 0c87f149a4..64b9abc685 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -11,13 +11,14 @@ steps: artifactName: ReleaseConfigs checkDownloadedFiles: true - - task: PowerShell@2 + - task: AzureCLI@2 name: setReleaseVars displayName: Set Release Configs Vars inputs: - targetType: inline - pwsh: true - script: | + azureSubscription: "Darc: Maestro Production" + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | try { if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt @@ -31,15 +32,16 @@ steps: $AzureDevOpsBuildId = $Env:Build_BuildId } else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + . $(Build.SourcesDirectory)\eng\common\tools.ps1 + $darc = Get-Darc + $buildInfo = & $darc get-build ` + --id ${{ parameters.BARBuildId }} ` + --extended ` + --output-format json ` + --ci ` + | convertFrom-Json - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId + $BarId = ${{ parameters.BARBuildId }} $Channels = $Env:PromoteToMaestroChannels -split "," $Channels = $Channels -join "][" $Channels = "[$Channels]" @@ -65,6 +67,4 @@ steps: exit 1 } env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/service-templates/ProductConstructionService/provision.bicep b/eng/service-templates/ProductConstructionService/provision.bicep index 58ca8f9b8f..ec98260aa5 100644 --- a/eng/service-templates/ProductConstructionService/provision.bicep +++ b/eng/service-templates/ProductConstructionService/provision.bicep @@ -300,6 +300,9 @@ resource virtualNetwork 'Microsoft.Network/virtualNetworks@2023-04-01' = { ] } } + tags: { + 'ms.inv.v0.networkUsage': 'mixedTraffic' + } } // subnet for the product construction service diff --git a/eng/templates/stages/deploy.yaml b/eng/templates/stages/deploy.yaml index cc253f8c3b..e34d8ee128 100644 --- a/eng/templates/stages/deploy.yaml +++ b/eng/templates/stages/deploy.yaml @@ -197,6 +197,7 @@ stages: .\darc\darc.exe get-default-channels --source-repo arcade-services --ci --bar-uri "$(GetAuthInfo.BarUri)" --debug displayName: Test Azure CLI authentication + continueOnError: true - powershell: .\darc\darc.exe get-default-channels --source-repo arcade-services --ci -t "$(GetAuthInfo.FederatedToken)" --bar-uri "$(GetAuthInfo.BarUri)" --debug diff --git a/global.json b/global.json index 8136e023fa..45aa063ffb 100644 --- a/global.json +++ b/global.json @@ -15,6 +15,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24324.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24328.2" } } diff --git a/src/Maestro/DependencyUpdater/.config/settings.Development.json b/src/Maestro/DependencyUpdater/.config/settings.Development.json index 887d9725c1..c84178fb69 100644 --- a/src/Maestro/DependencyUpdater/.config/settings.Development.json +++ b/src/Maestro/DependencyUpdater/.config/settings.Development.json @@ -13,10 +13,14 @@ "UseAzCliAuthentication": true }, "AzureDevOps": { - "Tokens": { - "dnceng": "[vault(dn-bot-dnceng-build-rw-code-rw-release-rw)]", - "devdiv": "[vault(dn-bot-devdiv-build-rw-code-rw-release-rw)]", - "domoreexp": "[vault(dn-bot-domoreexp-build-rw-code-rw-release-rw)]" + "dnceng": { + "Token": "[vault(dn-bot-dnceng-build-rw-code-rw-release-rw)]" + }, + "devdiv": { + "Token": "[vault(dn-bot-devdiv-build-rw-code-rw-release-rw)]" + }, + "domoreexp": { + "Token": "[vault(dn-bot-domoreexp-build-rw-code-rw-release-rw)]" } } } \ No newline at end of file diff --git a/src/Maestro/DependencyUpdater/.config/settings.Production.json b/src/Maestro/DependencyUpdater/.config/settings.Production.json index 7b67bbc51c..4f1633ebf4 100644 --- a/src/Maestro/DependencyUpdater/.config/settings.Production.json +++ b/src/Maestro/DependencyUpdater/.config/settings.Production.json @@ -10,5 +10,10 @@ "Kusto": { "Database": "engineeringdata", "KustoClusterUri": "https://engsrvprod.westus.kusto.windows.net" + }, + "AzureDevOps": { + "default": { + "ManagedIdentityId": "system" + } } } \ No newline at end of file diff --git a/src/Maestro/DependencyUpdater/.config/settings.Staging.json b/src/Maestro/DependencyUpdater/.config/settings.Staging.json index 270efe35a7..fac1ad31f4 100644 --- a/src/Maestro/DependencyUpdater/.config/settings.Staging.json +++ b/src/Maestro/DependencyUpdater/.config/settings.Staging.json @@ -10,5 +10,10 @@ "Kusto": { "Database": "engineeringdata", "KustoClusterUri": "https://engdata.westus2.kusto.windows.net" + }, + "AzureDevOps": { + "default": { + "ManagedIdentityId": "system" + } } } diff --git a/src/Maestro/DependencyUpdater/.config/settings.json b/src/Maestro/DependencyUpdater/.config/settings.json index 4b7dd5d419..e4b7dddd35 100644 --- a/src/Maestro/DependencyUpdater/.config/settings.json +++ b/src/Maestro/DependencyUpdater/.config/settings.json @@ -2,10 +2,5 @@ "GitHub": { "GitHubAppId": "[vault(github-app-id)]", "PrivateKey": "[vault(github-app-private-key)]" - }, - "AzureDevOps": { - "ManagedIdentities": { - "default": "system" - } } } diff --git a/src/Maestro/DependencyUpdater/Program.cs b/src/Maestro/DependencyUpdater/Program.cs index e26e7956d4..21a7a62d53 100644 --- a/src/Maestro/DependencyUpdater/Program.cs +++ b/src/Maestro/DependencyUpdater/Program.cs @@ -7,6 +7,7 @@ using Maestro.DataProviders; using Microsoft.DncEng.Configuration.Extensions; using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.GitHub.Authentication; using Microsoft.DotNet.Kusto; using Microsoft.DotNet.ServiceFabric.ServiceHost; @@ -57,6 +58,7 @@ public static void Configure(IServiceCollection services) // in such a way that will work with sizing. services.AddSingleton(); + services.AddTransient(sp => ActivatorUtilities.CreateInstance(sp, "git")); services.AddTransient(); services.AddScoped(); services.AddTransient(); diff --git a/src/Maestro/FeedCleanerService/.config/settings.Development.json b/src/Maestro/FeedCleanerService/.config/settings.Development.json index 4aa6c7834f..b2a3eca14b 100644 --- a/src/Maestro/FeedCleanerService/.config/settings.Development.json +++ b/src/Maestro/FeedCleanerService/.config/settings.Development.json @@ -8,8 +8,8 @@ "ConnectionString": "Data Source=localhost\\SQLEXPRESS;Initial Catalog=BuildAssetRegistry;Integrated Security=true" }, "AzureDevOps": { - "Tokens": { - "dnceng": "[vault(dn-bot-dnceng-packaging-rwm)]" + "dnceng": { + "Token": "[vault(dn-bot-dnceng-packaging-rwm)]" } } } \ No newline at end of file diff --git a/src/Maestro/FeedCleanerService/.config/settings.Production.json b/src/Maestro/FeedCleanerService/.config/settings.Production.json index 68ae60b3d1..8b1b69796e 100644 --- a/src/Maestro/FeedCleanerService/.config/settings.Production.json +++ b/src/Maestro/FeedCleanerService/.config/settings.Production.json @@ -9,5 +9,10 @@ }, "BuildAssetRegistry": { "ConnectionString": "Data Source=tcp:maestro-prod.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" + }, + "AzureDevOps": { + "default": { + "ManagedIdentityId": "system" + } } } \ No newline at end of file diff --git a/src/Maestro/FeedCleanerService/.config/settings.Staging.json b/src/Maestro/FeedCleanerService/.config/settings.Staging.json index f322dd3dff..7e364edb10 100644 --- a/src/Maestro/FeedCleanerService/.config/settings.Staging.json +++ b/src/Maestro/FeedCleanerService/.config/settings.Staging.json @@ -6,5 +6,10 @@ "KeyVaultUri": "https://maestroint.vault.azure.net/", "BuildAssetRegistry": { "ConnectionString": "Data Source=tcp:maestro-int-server.database.windows.net,1433; Initial Catalog=BuildAssetRegistry; Authentication=Active Directory Managed Identity; Persist Security Info=False; MultipleActiveResultSets=True; Connect Timeout=30; Encrypt=True; TrustServerCertificate=False;" + }, + "AzureDevOps": { + "default": { + "ManagedIdentityId": "system" + } } } diff --git a/src/Maestro/FeedCleanerService/.config/settings.json b/src/Maestro/FeedCleanerService/.config/settings.json index b87d8cc284..65c05be4e0 100644 --- a/src/Maestro/FeedCleanerService/.config/settings.json +++ b/src/Maestro/FeedCleanerService/.config/settings.json @@ -23,10 +23,5 @@ "Name": "dotnet-tools" } ] - }, - "AzureDevOps": { - "ManagedIdentities": { - "default": "system" - } } } \ No newline at end of file diff --git a/src/Maestro/FeedCleanerService/FeedCleanerService.cs b/src/Maestro/FeedCleanerService/FeedCleanerService.cs index 5496a0c768..3f540f3002 100644 --- a/src/Maestro/FeedCleanerService/FeedCleanerService.cs +++ b/src/Maestro/FeedCleanerService/FeedCleanerService.cs @@ -9,7 +9,6 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using Maestro.Common.AzureDevOpsTokens; using Maestro.Contracts; using Maestro.Data; using Maestro.Data.Models; @@ -17,7 +16,6 @@ using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.ServiceFabric.ServiceHost; using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -29,19 +27,16 @@ namespace FeedCleanerService; public sealed class FeedCleanerService : IFeedCleanerService, IServiceImplementation { public FeedCleanerService( + IAzureDevOpsClient azureDevOpsClient, ILogger logger, BuildAssetRegistryContext context, IOptions options) { + _azureDevOpsClient = azureDevOpsClient; Logger = logger; Context = context; _httpClient = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true }); _options = options; - AzureDevOpsClients = []; - foreach (string account in _options.Value.AzdoAccounts) - { - AzureDevOpsClients.Add(account, GetAzureDevOpsClientForAccountAsync(account)); - } } public ILogger Logger { get; } @@ -49,9 +44,8 @@ public FeedCleanerService( public FeedCleanerOptions Options => _options.Value; - public Dictionary AzureDevOpsClients { get; set; } - private readonly HttpClient _httpClient; + private readonly IAzureDevOpsClient _azureDevOpsClient; private readonly IOptions _options; public Task RunAsync(CancellationToken cancellationToken) @@ -74,8 +68,7 @@ public async Task CleanManagedFeedsAsync() foreach (var azdoAccount in Options.AzdoAccounts) { - var azdoClient = AzureDevOpsClients[azdoAccount]; - List allFeeds = await azdoClient.GetFeedsAsync(azdoAccount); + List allFeeds = await _azureDevOpsClient.GetFeedsAsync(azdoAccount); IEnumerable managedFeeds = allFeeds.Where(f => Regex.IsMatch(f.Name, FeedConstants.MaestroManagedFeedNamePattern)); foreach (var feed in managedFeeds) @@ -107,20 +100,6 @@ public async Task CleanManagedFeedsAsync() } } - /// - /// Returns an Azure DevOps client for a given account - /// - /// Azure DevOps account that the client will get its token from - /// Azure DevOps client for the given account - private AzureDevOpsClient GetAzureDevOpsClientForAccountAsync(string account) - { - IAzureDevOpsTokenProvider azdoTokenProvider = Context.GetService(); - string accessToken = azdoTokenProvider.GetTokenForAccount(account).GetAwaiter().GetResult(); - - // FeedCleaner does not need Git, or a temporaryRepositoryPath - return new AzureDevOpsClient(null, accessToken, Logger, null); - } - /// /// Get a mapping of feed -> (package, versions) for the release feeds so it /// can be easily queried whether a version of a package is in a feed. @@ -162,9 +141,8 @@ private static string ComputeAzureArtifactsNuGetFeedUrl(string feedName, string /// Dictionary where the key is the package name, and the value is a HashSet of the versions of the package in the feed private async Task>> GetPackageVersionsForFeedAsync(string account, string project, string feedName) { - var azdoClient = AzureDevOpsClients[account]; var packagesWithVersions = new Dictionary>(); - List packagesInFeed = await azdoClient.GetPackagesForFeedAsync(account, project, feedName); + List packagesInFeed = await _azureDevOpsClient.GetPackagesForFeedAsync(account, project, feedName); foreach (AzureDevOpsPackage package in packagesInFeed) { packagesWithVersions.Add(package.Name, new HashSet(StringComparer.OrdinalIgnoreCase)); @@ -266,14 +244,13 @@ private async Task> UpdateReleasedVersionsForPackageAsync( /// private async Task DeletePackageVersionsFromFeedAsync(AzureDevOpsFeed feed, string packageName, HashSet versionsToDelete) { - var azdoClient = AzureDevOpsClients[feed.Account]; foreach (string version in versionsToDelete) { try { Logger.LogInformation($"Deleting package {packageName}.{version} from feed {feed.Name}"); - await azdoClient.DeleteNuGetPackageVersionFromFeedAsync(feed.Account, + await _azureDevOpsClient.DeleteNuGetPackageVersionFromFeedAsync(feed.Account, feed.Project?.Name, feed.Name, packageName, @@ -343,6 +320,6 @@ private async Task IsPackageAvailableInNugetOrgAsync(string name, string v /// private async Task PopulatePackagesForFeedAsync(AzureDevOpsFeed feed) { - feed.Packages = await AzureDevOpsClients[feed.Account].GetPackagesForFeedAsync(feed.Account, feed.Project?.Name, feed.Name); + feed.Packages = await _azureDevOpsClient.GetPackagesForFeedAsync(feed.Account, feed.Project?.Name, feed.Name); } } diff --git a/src/Maestro/FeedCleanerService/Program.cs b/src/Maestro/FeedCleanerService/Program.cs index 22032ade7a..7a93562160 100644 --- a/src/Maestro/FeedCleanerService/Program.cs +++ b/src/Maestro/FeedCleanerService/Program.cs @@ -1,14 +1,15 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Generic; -using System.Linq; using Maestro.Common.AzureDevOpsTokens; using Maestro.Data; using Microsoft.DncEng.Configuration.Extensions; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.ServiceFabric.ServiceHost; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace FeedCleanerService; @@ -39,12 +40,9 @@ public static void Configure(IServiceCollection services) options.ReleasePackageFeeds.Add((token1.GetValue("Account"), token1.GetValue("Project"), token1.GetValue("Name"))); } - AzureDevOpsTokenProviderOptions azdoConfig = new(); + AzureDevOpsTokenProviderOptions azdoConfig = []; config.GetSection("AzureDevOps").Bind(azdoConfig); - IEnumerable allOrgs = azdoConfig.Tokens.Keys - .Concat(azdoConfig.ManagedIdentities.Keys) - .Distinct(); - options.AzdoAccounts.AddRange(allOrgs); + options.AzdoAccounts.AddRange(azdoConfig.Keys); }); services.AddDefaultJsonConfiguration(); services.AddBuildAssetRegistry((provider, options) => @@ -54,5 +52,10 @@ public static void Configure(IServiceCollection services) }); services.AddAzureDevOpsTokenProvider(); services.Configure("AzureDevOps", (o, s) => s.Bind(o)); + services.AddTransient(); + services.AddTransient(sp => + new ProcessManager( + sp.GetRequiredService>(), + "git")); } } diff --git a/src/Maestro/Maestro.Common/AppCredentials/AppCredential.cs b/src/Maestro/Maestro.Common/AppCredentials/AppCredential.cs index ad9732d8b5..571e4e759e 100644 --- a/src/Maestro/Maestro.Common/AppCredentials/AppCredential.cs +++ b/src/Maestro/Maestro.Common/AppCredentials/AppCredential.cs @@ -20,7 +20,7 @@ public class AppCredential : TokenCredential private readonly TokenRequestContext _requestContext; private readonly TokenCredential _tokenCredential; - private AppCredential(TokenCredential credential, TokenRequestContext requestContext) + public AppCredential(TokenCredential credential, TokenRequestContext requestContext) { _requestContext = requestContext; _tokenCredential = credential; @@ -42,9 +42,13 @@ public override ValueTask GetTokenAsync(TokenRequestContext _, Canc /// Use this for user-based flows. /// public static AppCredential CreateUserCredential(string appId, string userScope = ".default") - { - var requestContext = new TokenRequestContext(new string[] { $"api://{appId}/{userScope}" }); + => CreateUserCredential(appId, new TokenRequestContext([$"api://{appId}/{userScope}"])); + /// + /// Use this for user-based flows. + /// + public static AppCredential CreateUserCredential(string appId, TokenRequestContext requestContext) + { var authRecordPath = Path.Combine(AUTH_CACHE, $"{AUTH_RECORD_PREFIX}-{appId}"); var credential = GetInteractiveCredential(appId, requestContext, authRecordPath); @@ -122,7 +126,7 @@ public static AppCredential CreateFederatedCredential(string appId, string feder appId, token => Task.FromResult(federatedToken)); - var requestContext = new TokenRequestContext(new string[] { $"api://{appId}/.default" }); + var requestContext = new TokenRequestContext([$"api://{appId}/.default"]); return new AppCredential(credential, requestContext); } @@ -139,9 +143,9 @@ public static AppCredential CreateManagedIdentityCredential(string appId, string var appCredential = new ClientAssertionCredential( TENANT_ID, appId, - async (ct) => (await miCredential.GetTokenAsync(new TokenRequestContext(new string[] { "api://AzureADTokenExchange" }), ct)).Token); + async (ct) => (await miCredential.GetTokenAsync(new TokenRequestContext(["api://AzureADTokenExchange"]), ct)).Token); - var requestContext = new TokenRequestContext(new string[] { $"api://{appId}/.default" }); + var requestContext = new TokenRequestContext([$"api://{appId}/.default"]); return new AppCredential(appCredential, requestContext); } @@ -150,7 +154,7 @@ public static AppCredential CreateManagedIdentityCredential(string appId, string /// public static AppCredential CreateNonUserCredential(string appId) { - var requestContext = new TokenRequestContext(new string[] { $"{appId}/.default" }); + var requestContext = new TokenRequestContext([$"{appId}/.default"]); var credential = new AzureCliCredential(); return new AppCredential(credential, requestContext); } diff --git a/src/Maestro/Maestro.Common/AppCredentials/AppCredentialResolverOptions.cs b/src/Maestro/Maestro.Common/AppCredentials/AppCredentialResolverOptions.cs index dac9841887..56eab349ba 100644 --- a/src/Maestro/Maestro.Common/AppCredentials/AppCredentialResolverOptions.cs +++ b/src/Maestro/Maestro.Common/AppCredentials/AppCredentialResolverOptions.cs @@ -8,7 +8,7 @@ public class AppCredentialResolverOptions : CredentialResolverOptions /// /// Client ID of the Azure application to request the token for /// - public string AppId { get; set; } + public string AppId { get; } /// /// User scope to request the token for (in case of user flows). diff --git a/src/Maestro/Maestro.Common/AppCredentials/ResolvedCredential.cs b/src/Maestro/Maestro.Common/AppCredentials/ResolvedCredential.cs index 17bd08e7df..92ef1b01cd 100644 --- a/src/Maestro/Maestro.Common/AppCredentials/ResolvedCredential.cs +++ b/src/Maestro/Maestro.Common/AppCredentials/ResolvedCredential.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Threading; using Azure.Core; namespace Maestro.Common.AppCredentials; @@ -8,22 +9,31 @@ namespace Maestro.Common.AppCredentials; /// /// Credential with a set token. /// -public class ResolvedCredential : TokenCredential +public class ResolvedCredential(string token) : TokenCredential { - public ResolvedCredential(string token) + public override AccessToken GetToken(TokenRequestContext requestContext, CancellationToken cancellationToken) { - Token = token; + return new AccessToken(token, DateTimeOffset.MaxValue); } - public string Token { get; } + public override ValueTask GetTokenAsync(TokenRequestContext requestContext, CancellationToken cancellationToken) + { + return new ValueTask(new AccessToken(token, DateTimeOffset.MaxValue)); + } +} - public override AccessToken GetToken(TokenRequestContext requestContext, CancellationToken cancellationToken) +/// +/// Credential that resolves the token on each request. +/// +public class ResolvingCredential(Func tokenResolver) : TokenCredential +{ + public override AccessToken GetToken(TokenRequestContext context, CancellationToken cancellationToken) { - return new AccessToken(Token, DateTimeOffset.MaxValue); + return new AccessToken(tokenResolver(context, cancellationToken), DateTimeOffset.UtcNow); } - public override ValueTask GetTokenAsync(TokenRequestContext requestContext, CancellationToken cancellationToken) + public override ValueTask GetTokenAsync(TokenRequestContext context, CancellationToken cancellationToken) { - return new ValueTask(new AccessToken(Token, DateTimeOffset.MaxValue)); + return new ValueTask(new AccessToken(tokenResolver(context, cancellationToken), DateTimeOffset.UtcNow)); } } diff --git a/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsCredentialResolverOptions.cs b/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsCredentialResolverOptions.cs new file mode 100644 index 0000000000..79646ebb76 --- /dev/null +++ b/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsCredentialResolverOptions.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Maestro.Common.AppCredentials; + +namespace Maestro.Common.AzureDevOpsTokens; + +public class AzureDevOpsCredentialResolverOptions : AppCredentialResolverOptions +{ + public AzureDevOpsCredentialResolverOptions() + : base("499b84ac-1321-427f-aa17-267ca6975798") + { + } +} diff --git a/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProvider.cs b/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProvider.cs index 37c2de674a..0ada4cacfe 100644 --- a/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProvider.cs +++ b/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProvider.cs @@ -1,8 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.RegularExpressions; using Azure.Core; using Azure.Identity; +using Maestro.Common.AppCredentials; using Microsoft.Extensions.Options; namespace Maestro.Common.AzureDevOpsTokens; @@ -26,42 +28,123 @@ namespace Maestro.Common.AzureDevOpsTokens; public class AzureDevOpsTokenProvider : IAzureDevOpsTokenProvider { private const string AzureDevOpsScope = "499b84ac-1321-427f-aa17-267ca6975798/.default"; + private static readonly Regex AccountNameRegex = new(@"^https://dev\.azure\.com/(?[a-zA-Z0-9]+)/"); - private readonly Dictionary _tokenCredentials = []; - private readonly IOptionsMonitor _options; + private readonly Dictionary _accountCredentials; public AzureDevOpsTokenProvider(IOptionsMonitor options) + // We don't mind locking down the current value of the option because non-token settings are not expected to change + // Tokens are always read fresh through the second argument + : this(GetCredentials(options.CurrentValue, (account, _, _) => options.CurrentValue[account].Token!)) { - _options = options; + } + + public static AzureDevOpsTokenProvider FromStaticOptions(AzureDevOpsTokenProviderOptions options) + => new(GetCredentials(options, (account, _, _) => options[account].Token!)); + + private AzureDevOpsTokenProvider(Dictionary accountCredentials) + { + _accountCredentials = accountCredentials; + } + + public string GetTokenForAccount(string account) + { + var credential = GetCredential(account); + return credential.GetToken(new TokenRequestContext([AzureDevOpsScope]), cancellationToken: default).Token; + } + + public async Task GetTokenForAccountAsync(string account) + { + var credential = GetCredential(account); + return (await credential.GetTokenAsync(new TokenRequestContext([AzureDevOpsScope]), cancellationToken: default)).Token; + } - foreach (var credential in options.CurrentValue.ManagedIdentities) + public string GetTokenForRepository(string repositoryUrl) + { + Match m = AccountNameRegex.Match(repositoryUrl); + if (!m.Success) { - _tokenCredentials[credential.Key] = credential.Value == "system" - ? new ManagedIdentityCredential() - : new ManagedIdentityCredential(credential.Value); + throw new ArgumentException($"{repositoryUrl} is not a valid Azure DevOps repository URL"); } + + var account = m.Groups["account"].Value; + return GetTokenForAccount(account); } - public async Task GetTokenForAccount(string account) + public async Task GetTokenForRepositoryAsync(string repositoryUrl) { - if (_options.CurrentValue.Tokens.TryGetValue(account, out var pat) && !string.IsNullOrEmpty(pat)) + Match m = AccountNameRegex.Match(repositoryUrl); + if (!m.Success) { - return pat; + throw new ArgumentException($"{repositoryUrl} is not a valid Azure DevOps repository URL"); } - if (_tokenCredentials.TryGetValue(account, out var credential)) + var account = m.Groups["account"].Value; + return await GetTokenForAccountAsync(account); + } + + private TokenCredential GetCredential(string account) + { + if (_accountCredentials.TryGetValue(account, out var credential)) { - return (await credential.GetTokenAsync(new TokenRequestContext([AzureDevOpsScope]))).Token; + return credential; } - // We can also define just one MI for all accounts - if (_tokenCredentials.TryGetValue("default", out var defaultCredential)) + if (_accountCredentials.TryGetValue("default", out var defaultCredential)) { - return (await defaultCredential.GetTokenAsync(new TokenRequestContext([AzureDevOpsScope]))).Token; + return defaultCredential; } throw new ArgumentOutOfRangeException( $"Azure DevOps account {account} does not have a configured PAT or credential. " + $"Please add the account to the 'AzureDevOps.Tokens' or 'AzureDevOps.ManagedIdentities' configuration section"); } + + private static Dictionary GetCredentials( + AzureDevOpsTokenProviderOptions options, + Func patResolver) + { + Dictionary credentials = []; + + foreach (var pair in options) + { + var account = pair.Key; + var option = pair.Value; + + // 1. AzDO PAT from configuration + if (!string.IsNullOrEmpty(option.Token)) + { + credentials[account] = new ResolvingCredential((context, cancellationToken) => patResolver(account, context, cancellationToken)); + continue; + } + + // 2. Federated token that can be used to fetch an app token (for CI scenarios) + if (!string.IsNullOrEmpty(option.FederatedToken)) + { + credentials[account] = AppCredential.CreateFederatedCredential(option.AppId, option.FederatedToken!); + continue; + } + + // 3. Managed identity (for server-to-AzDO scenarios) + if (!string.IsNullOrEmpty(option.ManagedIdentityId)) + { + credentials[account] = option.ManagedIdentityId == "system" + ? new ManagedIdentityCredential() + : new ManagedIdentityCredential(option.ManagedIdentityId); + continue; + } + + // 4. Azure CLI authentication setup by the caller (for CI scenarios) + if (option.DisableInteractiveAuth) + { + credentials[account] = AppCredential.CreateNonUserCredential(option.AppId); + continue; + } + + // 5. Interactive login (user-based scenario) + credentials[account] = new DefaultAzureCredential(includeInteractiveCredentials: true); + } + + return credentials; + } } diff --git a/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProviderOptions.cs b/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProviderOptions.cs index cd8905ab69..7b2b5c836c 100644 --- a/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProviderOptions.cs +++ b/src/Maestro/Maestro.Common/AzureDevOpsTokens/AzureDevOpsTokenProviderOptions.cs @@ -3,9 +3,6 @@ namespace Maestro.Common.AzureDevOpsTokens; -public class AzureDevOpsTokenProviderOptions +public class AzureDevOpsTokenProviderOptions : Dictionary { - public Dictionary Tokens { get; } = []; - - public Dictionary ManagedIdentities { get; } = []; } diff --git a/src/Maestro/Maestro.Common/AzureDevOpsTokens/IAzureDevOpsTokenProvider.cs b/src/Maestro/Maestro.Common/AzureDevOpsTokens/IAzureDevOpsTokenProvider.cs index a43042e143..28eeca8929 100644 --- a/src/Maestro/Maestro.Common/AzureDevOpsTokens/IAzureDevOpsTokenProvider.cs +++ b/src/Maestro/Maestro.Common/AzureDevOpsTokens/IAzureDevOpsTokenProvider.cs @@ -1,27 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Text.RegularExpressions; - namespace Maestro.Common.AzureDevOpsTokens; -public interface IAzureDevOpsTokenProvider -{ - Task GetTokenForAccount(string account); -} - -public static class AzureDevOpsTokenProviderExtensions +public interface IAzureDevOpsTokenProvider : IRemoteTokenProvider { - private static readonly Regex AccountNameRegex = new(@"^https://dev\.azure\.com/(?[a-zA-Z0-9]+)/"); + string GetTokenForAccount(string account); - public static Task GetTokenForRepository(this IAzureDevOpsTokenProvider that, string repositoryUrl) - { - Match m = AccountNameRegex.Match(repositoryUrl); - if (!m.Success) - { - throw new ArgumentException($"{repositoryUrl} is not a valid Azure DevOps repository URL"); - } - var account = m.Groups["account"].Value; - return that.GetTokenForAccount(account); - } + Task GetTokenForAccountAsync(string account); } diff --git a/src/Maestro/Maestro.Common/AzureDevOpsTokens/MaestroAzureDevOpsServiceCollectionExtensions.cs b/src/Maestro/Maestro.Common/AzureDevOpsTokens/MaestroAzureDevOpsServiceCollectionExtensions.cs index 0eee5b95f7..601659182c 100644 --- a/src/Maestro/Maestro.Common/AzureDevOpsTokens/MaestroAzureDevOpsServiceCollectionExtensions.cs +++ b/src/Maestro/Maestro.Common/AzureDevOpsTokens/MaestroAzureDevOpsServiceCollectionExtensions.cs @@ -7,8 +7,19 @@ namespace Maestro.Common.AzureDevOpsTokens; public static class MaestroAzureDevOpsServiceCollectionExtensions { - public static IServiceCollection AddAzureDevOpsTokenProvider(this IServiceCollection services) + /// + /// Registers the Azure DevOps token provider. + /// + /// If provided, will initialize these options. Otherwise will try to monitor configuration. + public static IServiceCollection AddAzureDevOpsTokenProvider( + this IServiceCollection services, + AzureDevOpsTokenProviderOptions? staticOptions = null) { + if (staticOptions != null) + { + services.AddSingleton(staticOptions); + } + return services.AddSingleton(); } } diff --git a/src/Maestro/Maestro.Common/IRemoteTokenProvider.cs b/src/Maestro/Maestro.Common/IRemoteTokenProvider.cs new file mode 100644 index 0000000000..9069fcd889 --- /dev/null +++ b/src/Maestro/Maestro.Common/IRemoteTokenProvider.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Maestro.Common; + +public interface IRemoteTokenProvider +{ + string? GetTokenForRepository(string repoUri); + + Task GetTokenForRepositoryAsync(string repoUri); +} + +public class ResolvedTokenProvider(string? token) : IRemoteTokenProvider +{ + public string? GetTokenForRepository(string repoUri) => token; + + public Task GetTokenForRepositoryAsync(string repoUri) => Task.FromResult(token); +} diff --git a/src/Maestro/Maestro.DataProviders/DarcRemoteFactory.cs b/src/Maestro/Maestro.DataProviders/DarcRemoteFactory.cs index fd5967a587..7cffd9f56d 100644 --- a/src/Maestro/Maestro.DataProviders/DarcRemoteFactory.cs +++ b/src/Maestro/Maestro.DataProviders/DarcRemoteFactory.cs @@ -17,25 +17,27 @@ public class DarcRemoteFactory : IRemoteFactory { private readonly IVersionDetailsParser _versionDetailsParser; private readonly OperationManager _operations; + private readonly IProcessManager _processManager; private readonly BuildAssetRegistryContext _context; private readonly DarcRemoteMemoryCache _cache; private readonly IGitHubTokenProvider _gitHubTokenProvider; - private readonly IAzureDevOpsTokenProvider _azureDevOpsTokenProvider; + private readonly IAzureDevOpsTokenProvider _azdoTokenProvider; public DarcRemoteFactory( BuildAssetRegistryContext context, IGitHubTokenProvider gitHubTokenProvider, - IAzureDevOpsTokenProvider azureDevOpsTokenProvider, + IAzureDevOpsTokenProvider azdoTokenProvider, IVersionDetailsParser versionDetailsParser, DarcRemoteMemoryCache memoryCache, - OperationManager operations) + OperationManager operations, + IProcessManager processManager) { _operations = operations; + _processManager = processManager; _versionDetailsParser = versionDetailsParser; - _context = context; _gitHubTokenProvider = gitHubTokenProvider; - _azureDevOpsTokenProvider = azureDevOpsTokenProvider; + _azdoTokenProvider = azdoTokenProvider; _cache = memoryCache; } @@ -72,32 +74,17 @@ private async Task GetRemoteGitClient(string repoUrl, ILogger lo throw new GithubApplicationInstallationException($"No installation is available for repository '{normalizedUrl}'"); } - var remoteConfiguration = repoType switch - { - GitRepoType.GitHub => new RemoteTokenProvider( - gitHubToken: await _gitHubTokenProvider.GetTokenForInstallationAsync(installationId)), - GitRepoType.AzureDevOps => new RemoteTokenProvider( - azureDevOpsToken: await _azureDevOpsTokenProvider.GetTokenForRepository(normalizedUrl)), - - _ => throw new NotImplementedException($"Unknown repo url type {normalizedUrl}"), - }; - return repoType switch { GitRepoType.GitHub => installationId == default ? throw new GithubApplicationInstallationException($"No installation is available for repository '{normalizedUrl}'") : new GitHubClient( - gitExecutable: null, - remoteConfiguration.GitHubToken, + new Microsoft.DotNet.DarcLib.GitHubTokenProvider(_gitHubTokenProvider), + _processManager, logger, - temporaryRepositoryPath: null, _cache.Cache), - GitRepoType.AzureDevOps => new AzureDevOpsClient( - gitExecutable: null, - remoteConfiguration.AzureDevOpsToken, - logger, - temporaryRepositoryPath: null), + GitRepoType.AzureDevOps => new AzureDevOpsClient(_azdoTokenProvider, _processManager, logger), _ => throw new NotImplementedException($"Unknown repo url type {normalizedUrl}"), }; diff --git a/src/Maestro/Maestro.Web/.config/settings.Development.json b/src/Maestro/Maestro.Web/.config/settings.Development.json index 70b6f27c98..6f961d63d9 100644 --- a/src/Maestro/Maestro.Web/.config/settings.Development.json +++ b/src/Maestro/Maestro.Web/.config/settings.Development.json @@ -19,10 +19,14 @@ "UseAzCliAuthentication": true }, "AzureDevOps": { - "Tokens": { - "dnceng": "[vault(dn-bot-dnceng-build-rw-code-rw-release-rw)]", - "devdiv": "[vault(dn-bot-devdiv-build-rw-code-rw-release-rw)]", - "domoreexp": "[vault(dn-bot-domoreexp-build-rw-code-rw-release-rw)]" + "dnceng": { + "Token": "[vault(dn-bot-dnceng-build-rw-code-rw-release-rw)]" + }, + "devdiv": { + "Token": "[vault(dn-bot-devdiv-build-rw-code-rw-release-rw)]" + }, + "domoreexp": { + "Token": "[vault(dn-bot-domoreexp-build-rw-code-rw-release-rw)]" } } } \ No newline at end of file diff --git a/src/Maestro/Maestro.Web/.config/settings.Production.json b/src/Maestro/Maestro.Web/.config/settings.Production.json index 89ede15ffe..d3e979fcb6 100644 --- a/src/Maestro/Maestro.Web/.config/settings.Production.json +++ b/src/Maestro/Maestro.Web/.config/settings.Production.json @@ -27,5 +27,10 @@ "ClientId": "54c17f3d-7325-4eca-9db7-f090bfc765a8", "UserRole": "User", "RedirectUri": "https://maestro.dot.net/signin-oidc" + }, + "AzureDevOps": { + "default": { + "ManagedIdentityId": "system" + } } } diff --git a/src/Maestro/Maestro.Web/.config/settings.Staging.json b/src/Maestro/Maestro.Web/.config/settings.Staging.json index 2fd1c14a95..796b839997 100644 --- a/src/Maestro/Maestro.Web/.config/settings.Staging.json +++ b/src/Maestro/Maestro.Web/.config/settings.Staging.json @@ -23,5 +23,10 @@ "ClientId": "baf98f1b-374e-487d-af42-aa33807f11e4", "UserRole": "User", "RedirectUri": "https://maestro.int-dot.net/signin-oidc" + }, + "AzureDevOps": { + "default": { + "ManagedIdentityId": "system" + } } } diff --git a/src/Maestro/Maestro.Web/.config/settings.json b/src/Maestro/Maestro.Web/.config/settings.json index 81098fab20..85a503c094 100644 --- a/src/Maestro/Maestro.Web/.config/settings.json +++ b/src/Maestro/Maestro.Web/.config/settings.json @@ -11,11 +11,6 @@ "GitHubAppId": "[vault(github-app-id)]", "PrivateKey": "[vault(github-app-private-key)]" }, - "AzureDevOps": { - "ManagedIdentities": { - "default": "system" - } - }, "WebHooks": { "github": { "SecretKey": { diff --git a/src/Maestro/Maestro.Web/Controllers/AzDevController.cs b/src/Maestro/Maestro.Web/Controllers/AzDevController.cs index 3b1b7c9497..5a4556ecfc 100644 --- a/src/Maestro/Maestro.Web/Controllers/AzDevController.cs +++ b/src/Maestro/Maestro.Web/Controllers/AzDevController.cs @@ -38,7 +38,7 @@ public AzDevController(IAzureDevOpsTokenProvider tokenProvider) [HttpGet("build/status/{account}/{project}/{definitionId}/{*branch}")] public async Task GetBuildStatus(string account, string project, int definitionId, string branch, int count, string status) { - string token = await TokenProvider.GetTokenForAccount(account); + string token = await TokenProvider.GetTokenForAccountAsync(account); return await HttpContext.ProxyRequestAsync( s_lazyClient.Value, diff --git a/src/Maestro/Maestro.Web/Startup.cs b/src/Maestro/Maestro.Web/Startup.cs index 924ad3255d..b869b446d8 100644 --- a/src/Maestro/Maestro.Web/Startup.cs +++ b/src/Maestro/Maestro.Web/Startup.cs @@ -48,6 +48,7 @@ using Newtonsoft.Json; using Swashbuckle.AspNetCore.Swagger; using Maestro.Common.AzureDevOpsTokens; +using Microsoft.DotNet.DarcLib.Helpers; namespace Maestro.Web; @@ -233,9 +234,10 @@ public override void ConfigureServices(IServiceCollection services) ?.InformationalVersion); }); services.Configure(Configuration.GetSection("GitHub")); + + services.Configure(Configuration.GetSection("AzureDevOps")); services.AddAzureDevOpsTokenProvider(); - services.Configure("AzureDevOps", Configuration); services.AddKustoClientProvider("Kusto"); // We do not use AddMemoryCache here. We use our own cache because we wish to @@ -243,6 +245,10 @@ public override void ConfigureServices(IServiceCollection services) // in such a way that will work with sizing. services.AddSingleton(); + services.AddTransient(sp => + new ProcessManager( + sp.GetRequiredService>(), + "git")); services.AddTransient(); services.AddScoped(); services.AddTransient(); diff --git a/src/Maestro/SubscriptionActorService/.config/settings.Development.json b/src/Maestro/SubscriptionActorService/.config/settings.Development.json index 914577e8a6..d43389d25a 100644 --- a/src/Maestro/SubscriptionActorService/.config/settings.Development.json +++ b/src/Maestro/SubscriptionActorService/.config/settings.Development.json @@ -17,10 +17,14 @@ "UseAzCliAuthentication": true }, "AzureDevOps": { - "Tokens": { - "dnceng": "[vault(dn-bot-dnceng-build-rw-code-rw-release-rw)]", - "devdiv": "[vault(dn-bot-devdiv-build-rw-code-rw-release-rw)]", - "domoreexp": "[vault(dn-bot-domoreexp-build-rw-code-rw-release-rw)]" + "dnceng": { + "Token": "[vault(dn-bot-dnceng-build-rw-code-rw-release-rw)]" + }, + "devdiv": { + "Token": "[vault(dn-bot-devdiv-build-rw-code-rw-release-rw)]" + }, + "domoreexp": { + "Token": "[vault(dn-bot-domoreexp-build-rw-code-rw-release-rw)]" } } } \ No newline at end of file diff --git a/src/Maestro/SubscriptionActorService/.config/settings.Production.json b/src/Maestro/SubscriptionActorService/.config/settings.Production.json index 58d3937003..61284f40b4 100644 --- a/src/Maestro/SubscriptionActorService/.config/settings.Production.json +++ b/src/Maestro/SubscriptionActorService/.config/settings.Production.json @@ -15,5 +15,10 @@ "Kusto": { "Database": "engineeringdata", "KustoClusterUri": "https://engsrvprod.westus.kusto.windows.net" + }, + "AzureDevOps": { + "default": { + "ManagedIdentityId": "system" + } } } \ No newline at end of file diff --git a/src/Maestro/SubscriptionActorService/.config/settings.Staging.json b/src/Maestro/SubscriptionActorService/.config/settings.Staging.json index 0767e7b93f..10f3d23849 100644 --- a/src/Maestro/SubscriptionActorService/.config/settings.Staging.json +++ b/src/Maestro/SubscriptionActorService/.config/settings.Staging.json @@ -14,5 +14,10 @@ "Kusto": { "Database": "engineeringdata", "KustoClusterUri": "https://engdata.westus2.kusto.windows.net" + }, + "AzureDevOps": { + "default": { + "ManagedIdentityId": "system" + } } } \ No newline at end of file diff --git a/src/Maestro/SubscriptionActorService/.config/settings.json b/src/Maestro/SubscriptionActorService/.config/settings.json index 4b7dd5d419..e4b7dddd35 100644 --- a/src/Maestro/SubscriptionActorService/.config/settings.json +++ b/src/Maestro/SubscriptionActorService/.config/settings.json @@ -2,10 +2,5 @@ "GitHub": { "GitHubAppId": "[vault(github-app-id)]", "PrivateKey": "[vault(github-app-private-key)]" - }, - "AzureDevOps": { - "ManagedIdentities": { - "default": "system" - } } } diff --git a/src/Maestro/SubscriptionActorService/DarcRemoteFactory.cs b/src/Maestro/SubscriptionActorService/DarcRemoteFactory.cs index fab9ed035a..375273931c 100644 --- a/src/Maestro/SubscriptionActorService/DarcRemoteFactory.cs +++ b/src/Maestro/SubscriptionActorService/DarcRemoteFactory.cs @@ -3,6 +3,7 @@ using System; using System.Threading.Tasks; +using Maestro.Common; using Maestro.Common.AzureDevOpsTokens; using Maestro.Data; using Microsoft.DotNet.DarcLib; @@ -22,10 +23,10 @@ public class DarcRemoteFactory : IRemoteFactory private readonly IAzureDevOpsTokenProvider _azureDevOpsTokenProvider; private readonly BuildAssetRegistryContext _context; private readonly DarcRemoteMemoryCache _cache; - private readonly TemporaryFiles _tempFiles; private readonly ILocalGit _localGit; private readonly IVersionDetailsParser _versionDetailsParser; + private readonly IProcessManager _processManager; private readonly OperationManager _operations; public DarcRemoteFactory( @@ -37,11 +38,13 @@ public DarcRemoteFactory( TemporaryFiles tempFiles, ILocalGit localGit, IVersionDetailsParser versionDetailsParser, + IProcessManager processManager, OperationManager operations) { _tempFiles = tempFiles; _localGit = localGit; _versionDetailsParser = versionDetailsParser; + _processManager = processManager; _operations = operations; _configuration = configuration; _gitHubTokenProvider = gitHubTokenProvider; @@ -91,32 +94,20 @@ private async Task GetRemoteGitClient(string repoUrl, ILogger lo throw new GithubApplicationInstallationException($"No installation is available for repository '{normalizedUrl}'"); } - var remoteConfiguration = repoType switch - { - GitRepoType.GitHub => new RemoteTokenProvider( - gitHubToken: await _gitHubTokenProvider.GetTokenForInstallationAsync(installationId)), - GitRepoType.AzureDevOps => new RemoteTokenProvider( - azureDevOpsToken: await _azureDevOpsTokenProvider.GetTokenForRepository(normalizedUrl)), - - _ => throw new NotImplementedException($"Unknown repo url type {normalizedUrl}"), - }; - - var gitExe = _localGit.GetPathToLocalGit(); - return GitRepoUrlParser.ParseTypeFromUri(normalizedUrl) switch { GitRepoType.GitHub => installationId == default ? throw new GithubApplicationInstallationException($"No installation is available for repository '{normalizedUrl}'") : new GitHubClient( - gitExe, - remoteConfiguration.GitHubToken, + new ResolvedTokenProvider(await _gitHubTokenProvider.GetTokenForInstallationAsync(installationId)), + _processManager, logger, temporaryRepositoryRoot, _cache.Cache), GitRepoType.AzureDevOps => new AzureDevOpsClient( - gitExe, - remoteConfiguration.AzureDevOpsToken, + _azureDevOpsTokenProvider, + _processManager, logger, temporaryRepositoryRoot), diff --git a/src/Maestro/SubscriptionActorService/Program.cs b/src/Maestro/SubscriptionActorService/Program.cs index 71363bfb89..6b94205759 100644 --- a/src/Maestro/SubscriptionActorService/Program.cs +++ b/src/Maestro/SubscriptionActorService/Program.cs @@ -8,6 +8,7 @@ using Maestro.MergePolicies; using Microsoft.DncEng.Configuration.Extensions; using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.GitHub.Authentication; using Microsoft.DotNet.Kusto; using Microsoft.DotNet.ServiceFabric.ServiceHost; @@ -39,6 +40,8 @@ private static void Main() public static void Configure(IServiceCollection services) { services.TryAddTransient(sp => sp.GetRequiredService>()); + services.AddTransient(sp => + ActivatorUtilities.CreateInstance(sp, sp.GetRequiredService().GetPathToLocalGit())); services.AddSingleton(); services.AddSingleton(); services.AddTransient(); diff --git a/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs b/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs index 96c081234b..a94452e498 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs @@ -52,8 +52,8 @@ private static IRemoteGitRepo GetRemoteGitClient(ICommandLineOptions options, st { GitRepoType.GitHub => new GitHubClient( - options.GitLocation, - options.GitHubPat, + options.GetGitHubTokenProvider(), + new ProcessManager(logger, options.GitLocation), logger, temporaryRepositoryRoot, // Caching not in use for Darc local client. @@ -61,8 +61,8 @@ private static IRemoteGitRepo GetRemoteGitClient(ICommandLineOptions options, st GitRepoType.AzureDevOps => new AzureDevOpsClient( - options.GitLocation, - options.AzureDevOpsPat, + options.GetAzdoTokenProvider(), + new ProcessManager(logger, options.GitLocation), logger, temporaryRepositoryRoot), diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/AddBuildToChannelOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/AddBuildToChannelOperation.cs index 03d5a4c69b..5f63fff42b 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/AddBuildToChannelOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/AddBuildToChannelOperation.cs @@ -213,13 +213,6 @@ private async Task PromoteBuildAsync(Build build, List targetChann return Constants.SuccessCode; } - if (string.IsNullOrEmpty(_options.AzureDevOpsPat)) - { - Console.WriteLine($"Promoting build {build.Id} with the given parameters would require starting the Build Promotion pipeline, however an AzDO PAT was not found."); - Console.WriteLine("Either specify an AzDO PAT as a parameter or add the --skip-assets-publishing parameter when calling Darc add-build-to-channel."); - return Constants.ErrorCode; - } - var (arcadeSDKSourceBranch, arcadeSDKSourceSHA) = await GetSourceBranchInfoAsync(build).ConfigureAwait(false); // This condition can happen when for some reason we failed to determine the source branch/sha @@ -230,7 +223,7 @@ private async Task PromoteBuildAsync(Build build, List targetChann return Constants.ErrorCode; } - var azdoClient = new AzureDevOpsClient(gitExecutable: null, _options.AzureDevOpsPat, Logger, temporaryRepositoryPath: null); + var azdoClient = Provider.GetRequiredService(); var targetAzdoBuildStatus = await ValidateAzDOBuildAsync(azdoClient, build.AzureDevOpsAccount, build.AzureDevOpsProject, build.AzureDevOpsBuildId.Value) .ConfigureAwait(false); diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/AddDependencyOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/AddDependencyOperation.cs index 3b8f88f746..4802fddeaf 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/AddDependencyOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/AddDependencyOperation.cs @@ -24,7 +24,7 @@ public override async Task ExecuteAsync() { DependencyType type = _options.Type.ToLower() == "toolset" ? DependencyType.Toolset : DependencyType.Product; - var local = new Local(_options.GetRemoteConfiguration(), Logger); + var local = new Local(_options.GetRemoteTokenProvider(), Logger); DependencyDetail dependency = new DependencyDetail { diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/CloneOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/CloneOperation.cs index 78eb321223..763df81d8e 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/CloneOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/CloneOperation.cs @@ -79,7 +79,7 @@ public override async Task ExecuteAsync() if (string.IsNullOrWhiteSpace(_options.RepoUri)) { - var local = new Local(_options.GetRemoteConfiguration(), Logger); + var local = new Local(_options.GetRemoteTokenProvider(), Logger); IEnumerable rootDependencies = await local.GetDependenciesAsync(); IEnumerable stripped = rootDependencies.Select(StrippedDependency.GetDependency); foreach (StrippedDependency d in stripped) @@ -231,7 +231,7 @@ private Local HandleRepoAtSpecificHash(string repoPath, string commit, string ma if (Directory.Exists(repoPath)) { Logger.LogDebug($"Repo path {repoPath} already exists, assuming we cloned already and skipping"); - local = new Local(_options.GetRemoteConfiguration(), Logger, repoPath); + local = new Local(_options.GetRemoteTokenProvider(), Logger, repoPath); } else { @@ -239,7 +239,7 @@ private Local HandleRepoAtSpecificHash(string repoPath, string commit, string ma Directory.CreateDirectory(repoPath); File.WriteAllText(Path.Combine(repoPath, ".git"), GetGitDirRedirectString(masterRepoGitDirPath)); Logger.LogInformation($"Checking out {commit} in {repoPath}"); - local = new Local(_options.GetRemoteConfiguration(), Logger, repoPath); + local = new Local(_options.GetRemoteTokenProvider(), Logger, repoPath); local.Checkout(commit, true); } @@ -256,7 +256,7 @@ private async Task HandleMasterCopy(IRemoteFactory remoteFactory, string repoUrl { await HandleMasterCopyWithDefaultGitDir(remoteFactory, repoUrl, masterGitRepoPath, masterRepoGitDirPath); } - var local = new Local(_options.GetRemoteConfiguration(), Logger, masterGitRepoPath); + var local = new Local(_options.GetRemoteTokenProvider(), Logger, masterGitRepoPath); await local.AddRemoteIfMissingAsync(masterGitRepoPath, repoUrl); } @@ -361,7 +361,7 @@ private void HandleMasterCopyWithExistingGitDir(string masterGitRepoPath, string Logger.LogDebug($"Master .gitdir exists and master folder {masterGitRepoPath} does not. Creating master folder."); Directory.CreateDirectory(masterGitRepoPath); File.WriteAllText(Path.Combine(masterGitRepoPath, ".git"), gitDirRedirect); - var masterLocal = new Local(_options.GetRemoteConfiguration(), Logger, masterGitRepoPath); + var masterLocal = new Local(_options.GetRemoteTokenProvider(), Logger, masterGitRepoPath); Logger.LogDebug($"Checking out default commit in {masterGitRepoPath}"); masterLocal.Checkout(null, true); } diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/GetDependenciesOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/GetDependenciesOperation.cs index 6e80501a5b..3045a40a45 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/GetDependenciesOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/GetDependenciesOperation.cs @@ -23,7 +23,7 @@ public GetDependenciesOperation(GetDependenciesCommandLineOptions options) public override async Task ExecuteAsync() { - var local = new Local(_options.GetRemoteConfiguration(), Logger); + var local = new Local(_options.GetRemoteTokenProvider(), Logger); try { diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/GetDependencyGraphOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/GetDependencyGraphOperation.cs index 3e4c06719d..ac7a35f7d4 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/GetDependencyGraphOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/GetDependencyGraphOperation.cs @@ -27,7 +27,7 @@ public GetDependencyGraphOperation(GetDependencyGraphCommandLineOptions options) { _options = options; _gitClient = new LocalLibGit2Client( - options.GetRemoteConfiguration(), + options.GetRemoteTokenProvider(), new NoTelemetryRecorder(), new ProcessManager(Logger, _options.GitLocation), new FileSystem(), @@ -95,7 +95,7 @@ public override async Task ExecuteAsync() Console.WriteLine($"Getting root dependencies from local repository..."); // Grab root dependency set from local repo - var local = new Local(_options.GetRemoteConfiguration(), Logger); + var local = new Local(_options.GetRemoteTokenProvider(), Logger); rootDependencies = await local.GetDependenciesAsync( _options.AssetName); } @@ -131,7 +131,7 @@ public override async Task ExecuteAsync() { Console.WriteLine($"Getting root dependencies from local repository..."); - var local = new Local(_options.GetRemoteConfiguration(), Logger); + var local = new Local(_options.GetRemoteTokenProvider(), Logger); rootDependencies = await local.GetDependenciesAsync( _options.AssetName); diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs index 7027f7be39..78008d0a9f 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/Operation.cs @@ -3,6 +3,7 @@ using System; using System.Threading.Tasks; +using Maestro.Common.AzureDevOpsTokens; using Microsoft.Arcade.Common; using Microsoft.DotNet.Darc.Helpers; using Microsoft.DotNet.Darc.Options; @@ -47,16 +48,27 @@ protected Operation(ICommandLineOptions options, IServiceCollection? services = .AddConsole(o => o.FormatterName = CompactConsoleLoggerFormatter.FormatterName) .AddConsoleFormatter() .SetMinimumLevel(level)); - + services.AddSingleton(options); services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddTransient(sp => ActivatorUtilities.CreateInstance(sp, options.GitLocation)); services.TryAddSingleton(sp => RemoteFactory.GetBarClient(options, sp.GetRequiredService>())); services.TryAddSingleton(sp => sp.GetRequiredService()); - services.TryAddSingleton(options.GetRemoteConfiguration()); services.TryAddTransient(sp => sp.GetRequiredService>()); services.TryAddTransient(); + services.Configure(o => + { + o["default"] = new AzureDevOpsCredentialResolverOptions + { + Token = options.AzureDevOpsPat, + FederatedToken = options.FederatedToken, + DisableInteractiveAuth = options.IsCi, + }; + }); + services.TryAddSingleton(); + services.TryAddSingleton(); + services.TryAddSingleton(); Provider = services.BuildServiceProvider(); Logger = Provider.GetRequiredService>(); diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/UpdateDependenciesOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/UpdateDependenciesOperation.cs index 3688e92862..99140df331 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/UpdateDependenciesOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/UpdateDependenciesOperation.cs @@ -41,7 +41,7 @@ public override async Task ExecuteAsync() IBarApiClient barClient = Provider.GetRequiredService(); var coherencyUpdateResolver = new CoherencyUpdateResolver(barClient, Logger); - var local = new Local(_options.GetRemoteConfiguration(), Logger); + var local = new Local(_options.GetRemoteTokenProvider(), Logger); List dependenciesToUpdate = []; bool someUpToDate = false; string finalMessage = $"Local dependencies updated from channel '{_options.Channel}'."; diff --git a/src/Microsoft.DotNet.Darc/Darc/Operations/VerifyOperation.cs b/src/Microsoft.DotNet.Darc/Darc/Operations/VerifyOperation.cs index 0a56fe24d1..5f11943d51 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Operations/VerifyOperation.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Operations/VerifyOperation.cs @@ -26,7 +26,7 @@ public VerifyOperation(VerifyCommandLineOptions options) /// Process exit code. public override async Task ExecuteAsync() { - var local = new Local(_options.GetRemoteConfiguration(), Logger); + var local = new Local(_options.GetRemoteTokenProvider(), Logger); try { diff --git a/src/Microsoft.DotNet.Darc/Darc/Options/CommandLineOptions.cs b/src/Microsoft.DotNet.Darc/Darc/Options/CommandLineOptions.cs index e1b26c132c..cfb1f56b3a 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Options/CommandLineOptions.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Options/CommandLineOptions.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using CommandLine; +using Maestro.Common; +using Maestro.Common.AzureDevOpsTokens; using Microsoft.DotNet.Darc.Helpers; using Microsoft.DotNet.Darc.Operations; using Microsoft.DotNet.DarcLib; @@ -55,11 +57,25 @@ public abstract class CommandLineOptions : ICommandLineOptions public abstract Operation GetOperation(); - public RemoteTokenProvider GetRemoteConfiguration() + public IRemoteTokenProvider GetRemoteTokenProvider() + => new RemoteTokenProvider(GetAzdoTokenProvider(), GetGitHubTokenProvider()); + + public IAzureDevOpsTokenProvider GetAzdoTokenProvider() { - return new RemoteTokenProvider(GitHubPat, AzureDevOpsPat); + var azdoOptions = new AzureDevOpsTokenProviderOptions + { + ["default"] = new AzureDevOpsCredentialResolverOptions + { + Token = AzureDevOpsPat, + FederatedToken = FederatedToken, + DisableInteractiveAuth = IsCi, + } + }; + return AzureDevOpsTokenProvider.FromStaticOptions(azdoOptions); } + public IRemoteTokenProvider GetGitHubTokenProvider() => new ResolvedTokenProvider(GitHubPat); + public void InitializeFromSettings(ILogger logger) { var localSettings = LocalSettings.GetSettings(this, logger); diff --git a/src/Microsoft.DotNet.Darc/Darc/Options/ICommandLineOptions.cs b/src/Microsoft.DotNet.Darc/Darc/Options/ICommandLineOptions.cs index 75a3482e3e..697ccd6cc1 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Options/ICommandLineOptions.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Options/ICommandLineOptions.cs @@ -1,8 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Maestro.Common; +using Maestro.Common.AzureDevOpsTokens; using Microsoft.DotNet.Darc.Operations; -using Microsoft.DotNet.DarcLib; using Microsoft.Extensions.Logging; namespace Microsoft.DotNet.Darc.Options; @@ -20,7 +21,9 @@ public interface ICommandLineOptions bool IsCi { get; set; } Operation GetOperation(); - RemoteTokenProvider GetRemoteConfiguration(); + IRemoteTokenProvider GetRemoteTokenProvider(); + IAzureDevOpsTokenProvider GetAzdoTokenProvider(); + IRemoteTokenProvider GetGitHubTokenProvider(); /// /// Reads missing options from the local settings. diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Actions/Local.cs b/src/Microsoft.DotNet.Darc/DarcLib/Actions/Local.cs index 14a9151c4f..5f04abefa5 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Actions/Local.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Actions/Local.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Maestro.Common; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models; using Microsoft.Extensions.Logging; @@ -30,11 +31,11 @@ public class Local /// private const string GitExecutable = "git"; - public Local(RemoteTokenProvider remoteConfiguration, ILogger logger, string overrideRootPath = null) + public Local(IRemoteTokenProvider tokenProvider, ILogger logger, string overrideRootPath = null) { _logger = logger; _versionDetailsParser = new VersionDetailsParser(); - _gitClient = new LocalLibGit2Client(remoteConfiguration, new NoTelemetryRecorder(), new ProcessManager(logger, GitExecutable), new FileSystem(), logger); + _gitClient = new LocalLibGit2Client(tokenProvider, new NoTelemetryRecorder(), new ProcessManager(logger, GitExecutable), new FileSystem(), logger); _fileManager = new DependencyFileManager(_gitClient, _versionDetailsParser, logger); _repoRootDir = new(() => overrideRootPath ?? _gitClient.GetRootDirAsync().GetAwaiter().GetResult(), LazyThreadSafetyMode.PublicationOnly); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs index b150ea7196..701e667f04 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs @@ -10,7 +10,9 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Maestro.Common.AzureDevOpsTokens; using Maestro.MergePolicyEvaluation; +using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models.AzureDevOps; using Microsoft.DotNet.Services.Utility; using Microsoft.Extensions.Logging; @@ -47,24 +49,25 @@ public class AzureDevOpsClient : RemoteRepoBase, IRemoteGitRepo, IAzureDevOpsCli // Azure DevOps uses this id when creating a new branch as well as when deleting a branch private static readonly string BaseObjectId = "0000000000000000000000000000000000000000"; + + private readonly IAzureDevOpsTokenProvider _tokenProvider; private readonly ILogger _logger; - private readonly string _personalAccessToken; private readonly JsonSerializerSettings _serializerSettings; - /// - /// Create a new azure devops client. - /// - /// - /// PAT for Azure DevOps. This PAT should cover all - /// organizations that may be accessed in a single operation. - /// - /// - /// The AzureDevopsClient currently does not utilize the memory cache - /// - public AzureDevOpsClient(string gitExecutable, string accessToken, ILogger logger, string temporaryRepositoryPath) - : base(gitExecutable, temporaryRepositoryPath, null, logger, new RemoteTokenProvider(azureDevOpsToken: accessToken)) + public AzureDevOpsClient(IAzureDevOpsTokenProvider tokenProvider, IProcessManager processManager, ILogger logger) + : this(tokenProvider, processManager, logger, null) { - _personalAccessToken = accessToken; + } + + public AzureDevOpsClient(IAzureDevOpsTokenProvider tokenProvider, IProcessManager processManager, ILogger logger) + : this(tokenProvider, processManager, (ILogger)logger) + { + } + + public AzureDevOpsClient(IAzureDevOpsTokenProvider tokenProvider, IProcessManager processManager, ILogger logger, string temporaryRepositoryPath) + : base(tokenProvider, processManager, temporaryRepositoryPath, null, logger) + { + _tokenProvider = tokenProvider; _logger = logger; _serializerSettings = new JsonSerializerSettings { @@ -909,7 +912,7 @@ private HttpClient CreateHttpClient(string accountName, string projectName = nul $"application/json;api-version={versionOverride ?? DefaultApiVersion}"); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( "Basic", - Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", _personalAccessToken)))); + Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", _tokenProvider.GetTokenForAccount(accountName))))); return client; } @@ -922,7 +925,7 @@ private HttpClient CreateHttpClient(string accountName, string projectName = nul private VssConnection CreateVssConnection(string accountName) { var accountUri = new Uri($"https://dev.azure.com/{accountName}"); - var creds = new VssCredentials(new VssBasicCredential("", _personalAccessToken)); + var creds = new VssCredentials(new VssBasicCredential("", _tokenProvider.GetTokenForAccount(accountName))); return new VssConnection(accountUri, creds); } @@ -1005,18 +1008,16 @@ public static (string accountName, string projectName, string repoName, int id) /// Branch to push to /// Commit message /// - public Task CommitFilesAsync(List filesToCommit, string repoUri, string branch, string commitMessage) - { - return CommitFilesAsync( + public async Task CommitFilesAsync(List filesToCommit, string repoUri, string branch, string commitMessage) + => await CommitFilesAsync( filesToCommit, repoUri, branch, commitMessage, _logger, - _personalAccessToken, + await _tokenProvider.GetTokenForRepositoryAsync(repoUri), "DotNet-Bot", "dn-bot@microsoft.com"); - } /// /// If the release pipeline doesn't have an artifact source a new one is added. diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs index 5b03ad1578..36c3118e2f 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs @@ -20,6 +20,8 @@ using Microsoft.DotNet.Services.Utility; using System.Collections.Immutable; using Maestro.MergePolicyEvaluation; +using Maestro.Common; +using Microsoft.DotNet.DarcLib.Helpers; namespace Microsoft.DotNet.DarcLib; @@ -38,8 +40,8 @@ public class GitHubClient : RemoteRepoBase, IRemoteGitRepo new(@"^/repos/(?[^/]+)/(?[^/]+)/pulls/(?\d+)$"); private readonly Lazy _lazyClient; + private readonly IRemoteTokenProvider _tokenProvider; private readonly ILogger _logger; - private readonly string _personalAccessToken; private readonly JsonSerializerSettings _serializerSettings; private readonly string _userAgent = $"DarcLib-{DarcLibVersion}"; @@ -51,10 +53,24 @@ static GitHubClient() _product = new ProductHeaderValue("DarcLib", version); } - public GitHubClient(string gitExecutable, string accessToken, ILogger logger, string temporaryRepositoryPath, IMemoryCache cache) - : base(gitExecutable, temporaryRepositoryPath, cache, logger, new RemoteTokenProvider(gitHubToken: accessToken, null)) + public GitHubClient( + IRemoteTokenProvider remoteTokenProvider, + IProcessManager processManager, + ILogger logger, + IMemoryCache cache) + : this(remoteTokenProvider, processManager, logger, null, cache) + { + } + + public GitHubClient( + IRemoteTokenProvider remoteTokenProvider, + IProcessManager processManager, + ILogger logger, + string temporaryRepositoryPath, + IMemoryCache cache) + : base(remoteTokenProvider, processManager, temporaryRepositoryPath, cache, logger) { - _personalAccessToken = accessToken; + _tokenProvider = remoteTokenProvider; _logger = logger; _serializerSettings = new JsonSerializerSettings { @@ -99,6 +115,7 @@ private async Task GetFileContentsAsync(string owner, string repo, strin { using (HttpResponseMessage response = await ExecuteRemoteGitCommandAsync( HttpMethod.Get, + $"https://github.com/{owner}/{repo}", $"repos/{owner}/{repo}/contents/{filePath}?ref={branch}", _logger, logFailure: false)) @@ -144,6 +161,7 @@ public async Task CreateBranchAsync(string repoUri, string newBranch, string bas using (await ExecuteRemoteGitCommandAsync( HttpMethod.Get, + $"https://github.com/{owner}/{repo}", $"repos/{owner}/{repo}/branches/{newBranch}", _logger, retryCount: 0)) { } @@ -152,6 +170,7 @@ public async Task CreateBranchAsync(string repoUri, string newBranch, string bas body = JsonConvert.SerializeObject(githubRef, _serializerSettings); using (await ExecuteRemoteGitCommandAsync( new HttpMethod("PATCH"), + $"https://github.com/{owner}/{repo}", $"repos/{owner}/{repo}/git/{gitRef}", _logger, body)) { } @@ -165,6 +184,7 @@ public async Task CreateBranchAsync(string repoUri, string newBranch, string bas body = JsonConvert.SerializeObject(githubRef, _serializerSettings); using (await ExecuteRemoteGitCommandAsync( HttpMethod.Post, + $"https://github.com/{owner}/{repo}", $"repos/{owner}/{repo}/git/refs", _logger, body)) { } @@ -261,6 +281,7 @@ public async Task> SearchPullRequestsAsync( JObject responseContent; using (HttpResponseMessage response = await ExecuteRemoteGitCommandAsync( HttpMethod.Get, + $"https://github.com/{owner}/{repo}", $"search/issues?q={query}", _logger)) { @@ -284,8 +305,11 @@ public async Task GetPullRequestStatusAsync(string pullRequestUrl) (string owner, string repo, int id) = ParsePullRequestUri(pullRequestUrl); JObject responseContent; - using (HttpResponseMessage response = await ExecuteRemoteGitCommandAsync(HttpMethod.Get, - $"repos/{owner}/{repo}/pulls/{id}", _logger)) + using (HttpResponseMessage response = await ExecuteRemoteGitCommandAsync( + HttpMethod.Get, + $"https://github.com/{owner}/{repo}", + $"repos/{owner}/{repo}/pulls/{id}", + _logger)) { responseContent = JObject.Parse(await response.Content.ReadAsStringAsync()); } @@ -732,6 +756,7 @@ private async Task GetGitItemImpl(string path, TreeItem treeItem, strin /// private async Task ExecuteRemoteGitCommandAsync( HttpMethod method, + string repoUri, string requestUri, ILogger logger, string body = null, @@ -743,7 +768,7 @@ private async Task ExecuteRemoteGitCommandAsync( { retryCount = 0; } - using (HttpClient client = CreateHttpClient()) + using (HttpClient client = CreateHttpClient(repoUri)) { var requestManager = new HttpRequestManager(client, method, requestUri, logger, body, versionOverride, logFailure); try @@ -767,16 +792,17 @@ private async Task ExecuteRemoteGitCommandAsync( /// Create a new http client for talking to github. /// /// New http client CheckIfFileExistsAsync(string repoUri, string filePath JObject content; using (response = await ExecuteRemoteGitCommandAsync( HttpMethod.Get, + $"https://github.com/{owner}/{repo}", $"repos/{owner}/{repo}/contents/{filePath}?ref={branch}", _logger)) { @@ -874,6 +901,7 @@ private async Task GetLastCommitShaAsync(string owner, string repo, stri JObject content; using (HttpResponseMessage response = await ExecuteRemoteGitCommandAsync( HttpMethod.Get, + $"https://github.com/{owner}/{repo}", $"repos/{owner}/{repo}/commits/{branch}", _logger)) { @@ -1001,7 +1029,8 @@ private async Task> GetChecksFromChecksApiAsync(string owner, strin private Octokit.GitHubClient CreateGitHubClient() { - if (string.IsNullOrEmpty(_personalAccessToken)) + var token = _tokenProvider.GetTokenForRepository(GitHubApiUri); + if (string.IsNullOrEmpty(token)) { throw new DarcException( "GitHub personal access token is required for this operation. " + @@ -1010,7 +1039,7 @@ private Octokit.GitHubClient CreateGitHubClient() return new Octokit.GitHubClient(_product) { - Credentials = new Credentials(_personalAccessToken) + Credentials = new Credentials(token) }; } @@ -1084,6 +1113,7 @@ private async Task GetCommitMapForPathAsync( using (HttpResponseMessage response = await ExecuteRemoteGitCommandAsync( HttpMethod.Get, + $"https://github.com/{owner}/{repo}", $"repos/{owner}/{repo}/contents/{path}?ref={assetsProducedInCommit}", _logger)) { @@ -1158,17 +1188,16 @@ public static (string owner, string repo, int id) ParsePullRequestUri(string uri /// Remote repository URI /// Branch to push to /// Commit message - /// - public Task CommitFilesAsync(List filesToCommit, string repoUri, string branch, string commitMessage) + public async Task CommitFilesAsync(List filesToCommit, string repoUri, string branch, string commitMessage) { - return CommitFilesAsync( - filesToCommit, - repoUri, - branch, - commitMessage, - _logger, - _personalAccessToken, - Constants.DarcBotName, + await CommitFilesAsync( + filesToCommit, + repoUri, + branch, + commitMessage, + _logger, + await _tokenProvider.GetTokenForRepositoryAsync(GitHubApiUri), + Constants.DarcBotName, Constants.DarcBotEmail); } @@ -1190,6 +1219,7 @@ public async Task GitDiffAsync(string repoUri, string baseVersion, stri JObject content; using (HttpResponseMessage response = await ExecuteRemoteGitCommandAsync( HttpMethod.Get, + $"https://github.com/{owner}/{repo}", $"repos/{owner}/{repo}/compare/{baseVersion}...{targetVersion}", _logger)) { @@ -1224,6 +1254,7 @@ public async Task RepoExistsAsync(string repoUri) { using (await ExecuteRemoteGitCommandAsync( HttpMethod.Get, + $"https://github.com/{owner}/{repo}", $"repos/{owner}/{repo}", _logger, logFailure: false)) { } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubTokenProvider.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubTokenProvider.cs new file mode 100644 index 0000000000..a4609ce932 --- /dev/null +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubTokenProvider.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading.Tasks; +using Maestro.Common; + +#nullable enable +namespace Microsoft.DotNet.DarcLib; + +public class GitHubTokenProvider(GitHub.Authentication.IGitHubTokenProvider tokenProvider) : IRemoteTokenProvider +{ + public string? GetTokenForRepository(string repoUri) + { + return tokenProvider.GetTokenForRepository(repoUri).GetAwaiter().GetResult(); + } + + public async Task GetTokenForRepositoryAsync(string repoUri) + { + return await tokenProvider.GetTokenForRepository(repoUri); + } +} diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitNativeRepoCloner.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitNativeRepoCloner.cs index c7d2306650..d4bd00b95d 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitNativeRepoCloner.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitNativeRepoCloner.cs @@ -52,7 +52,7 @@ private async Task CloneAsync( var args = new List(); var envVars = new Dictionary(); - _localGitClient.AddGitAuthHeader(args, envVars, repoUri); + await _localGitClient.AddGitAuthHeader(args, envVars, repoUri); if (gitDirectory != null) { diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitRepoCloner.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitRepoCloner.cs index 76527c5a85..9cd9f468ba 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitRepoCloner.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitRepoCloner.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Threading.Tasks; using LibGit2Sharp; +using Maestro.Common; using Microsoft.Extensions.Logging; #nullable enable @@ -13,13 +14,13 @@ namespace Microsoft.DotNet.DarcLib; public class GitRepoCloner : IGitRepoCloner { - private readonly RemoteTokenProvider _remoteConfiguration; + private readonly IRemoteTokenProvider _remoteTokenProvider; private readonly ILocalLibGit2Client _localGitClient; private readonly ILogger _logger; - public GitRepoCloner(RemoteTokenProvider remoteConfiguration, ILocalLibGit2Client localGitClient, ILogger logger) + public GitRepoCloner(IRemoteTokenProvider remoteTokenProvider, ILocalLibGit2Client localGitClient, ILogger logger) { - _remoteConfiguration = remoteConfiguration; + _remoteTokenProvider = remoteTokenProvider; _localGitClient = localGitClient; _logger = logger; } @@ -60,7 +61,7 @@ private Task CloneAsync(string repoUri, string? commit, string targetDirectory, // The PAT is actually the only thing that matters here, the username // will be ignored. Username = RemoteTokenProvider.GitRemoteUser, - Password = _remoteConfiguration.GetTokenForUri(repoUri), + Password = _remoteTokenProvider.GetTokenForRepository(repoUri), }, }; diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs index 58bf97c302..df9c0e4e8f 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using Maestro.Common; +using Maestro.Common.AzureDevOpsTokens; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.Extensions.Logging; @@ -12,9 +14,11 @@ public interface IGitRepoFactory { IGitRepo CreateClient(string repoUri); } + public class GitRepoFactory : IGitRepoFactory { - private readonly RemoteTokenProvider _remoteConfiguration; + private readonly IRemoteTokenProvider _remoteTokenProvider; + private readonly IAzureDevOpsTokenProvider _azdoTokenProvider; private readonly ITelemetryRecorder _telemetryRecorder; private readonly IProcessManager _processManager; private readonly IFileSystem _fileSystem; @@ -22,14 +26,16 @@ public class GitRepoFactory : IGitRepoFactory private readonly string? _temporaryPath = null; public GitRepoFactory( - RemoteTokenProvider remoteConfiguration, + IRemoteTokenProvider remoteTokenProvider, + IAzureDevOpsTokenProvider azdoTokenProvider, ITelemetryRecorder telemetryRecorder, IProcessManager processManager, IFileSystem fileSystem, ILoggerFactory loggerFactory, string temporaryPath) { - _remoteConfiguration = remoteConfiguration; + _remoteTokenProvider = remoteTokenProvider; + _azdoTokenProvider = azdoTokenProvider; _telemetryRecorder = telemetryRecorder; _processManager = processManager; _fileSystem = fileSystem; @@ -40,21 +46,21 @@ public GitRepoFactory( public IGitRepo CreateClient(string repoUri) => GitRepoUrlParser.ParseTypeFromUri(repoUri) switch { GitRepoType.AzureDevOps => new AzureDevOpsClient( - _processManager.GitExecutable, - _remoteConfiguration.AzureDevOpsToken, + _azdoTokenProvider, + _processManager, _loggerFactory.CreateLogger(), _temporaryPath), GitRepoType.GitHub => new GitHubClient( - _processManager.GitExecutable, - _remoteConfiguration.GitHubToken, + _remoteTokenProvider, + _processManager, _loggerFactory.CreateLogger(), _temporaryPath, // Caching not in use for Darc local client. null), GitRepoType.Local => new LocalLibGit2Client( - _remoteConfiguration, + _remoteTokenProvider, _telemetryRecorder, _processManager, _fileSystem, diff --git a/src/Microsoft.DotNet.Darc/DarcLib/ILocalGitClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/ILocalGitClient.cs index d81e7bd54d..ccda984313 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/ILocalGitClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/ILocalGitClient.cs @@ -159,7 +159,7 @@ Task StageAsync( /// /// Where to add the new argument into /// Where to add the new variables into - void AddGitAuthHeader(IList args, IDictionary envVars, string repoUri); + Task AddGitAuthHeader(IList args, IDictionary envVars, string repoUri); /// /// Gets a value of a given git configuration setting. diff --git a/src/Microsoft.DotNet.Darc/DarcLib/LocalGitClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/LocalGitClient.cs index b8473702b7..9a64839ec1 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/LocalGitClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/LocalGitClient.cs @@ -9,6 +9,7 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using Maestro.Common; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.Extensions.Logging; @@ -21,14 +22,14 @@ namespace Microsoft.DotNet.DarcLib; /// public class LocalGitClient : ILocalGitClient { - private readonly RemoteTokenProvider _remoteConfiguration; + private readonly IRemoteTokenProvider _remoteConfiguration; private readonly ITelemetryRecorder _telemetryRecorder; private readonly IProcessManager _processManager; private readonly IFileSystem _fileSystem; private readonly ILogger _logger; public LocalGitClient( - RemoteTokenProvider remoteConfiguration, + IRemoteTokenProvider remoteConfiguration, ITelemetryRecorder telemetryRecorder, IProcessManager processManager, IFileSystem fileSystem, @@ -280,14 +281,14 @@ public async Task UpdateRemoteAsync(string repoPath, string remoteName, Cancella List args = [ "remote", "update", remoteName ]; var envVars = new Dictionary(); - AddGitAuthHeader(args, envVars, remoteUri); + await AddGitAuthHeader(args, envVars, remoteUri); result = await _processManager.ExecuteGit(repoPath, args, envVars, cancellationToken: cancellationToken); result.ThrowIfFailed($"Failed to update {repoPath} from remote {remoteName}"); args = [ "fetch", "--tags", "--force", remoteName ]; envVars = []; - AddGitAuthHeader(args, envVars, remoteUri); + await AddGitAuthHeader(args, envVars, remoteUri); result = await _processManager.ExecuteGit(repoPath, args, envVars, cancellationToken: cancellationToken); result.ThrowIfFailed($"Failed to update {repoPath} from remote {remoteName}"); @@ -428,10 +429,9 @@ public async Task BlameLineAsync(string repoPath, string relativeFilePat return result.StandardOutput.Trim().Split(' ').First(); } - public void AddGitAuthHeader(IList args, IDictionary envVars, string repoUri) + public async Task AddGitAuthHeader(IList args, IDictionary envVars, string repoUri) { - var token = _remoteConfiguration.GetTokenForUri(repoUri); - + var token = await _remoteConfiguration.GetTokenForRepositoryAsync(repoUri); if (token == null) { return; diff --git a/src/Microsoft.DotNet.Darc/DarcLib/LocalLibGit2Client.cs b/src/Microsoft.DotNet.Darc/DarcLib/LocalLibGit2Client.cs index 22d7cb7081..14d14b6546 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/LocalLibGit2Client.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/LocalLibGit2Client.cs @@ -9,6 +9,7 @@ using System.Text; using System.Threading.Tasks; using LibGit2Sharp; +using Maestro.Common; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.Extensions.Logging; @@ -20,14 +21,19 @@ namespace Microsoft.DotNet.DarcLib; /// public class LocalLibGit2Client : LocalGitClient, ILocalLibGit2Client { - private readonly RemoteTokenProvider _remoteConfiguration; + private readonly IRemoteTokenProvider _remoteTokenProvider; private readonly IProcessManager _processManager; private readonly ILogger _logger; - public LocalLibGit2Client(RemoteTokenProvider remoteConfiguration, ITelemetryRecorder telemetryRecorder, IProcessManager processManager, IFileSystem fileSystem, ILogger logger) - : base(remoteConfiguration, telemetryRecorder, processManager, fileSystem, logger) + public LocalLibGit2Client( + IRemoteTokenProvider remoteTokenProvider, + ITelemetryRecorder telemetryRecorder, + IProcessManager processManager, + IFileSystem fileSystem, + ILogger logger) + : base(remoteTokenProvider, telemetryRecorder, processManager, fileSystem, logger) { - _remoteConfiguration = remoteConfiguration; + _remoteTokenProvider = remoteTokenProvider; _processManager = processManager; _logger = logger; } @@ -350,7 +356,7 @@ public async Task Push( CredentialsProvider = (url, user, cred) => new UsernamePasswordCredentials { - Username = _remoteConfiguration.GetTokenForUri(remoteUrl), + Username = _remoteTokenProvider.GetTokenForRepository(remoteUrl), Password = string.Empty } }; diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Microsoft.DotNet.DarcLib.csproj b/src/Microsoft.DotNet.Darc/DarcLib/Microsoft.DotNet.DarcLib.csproj index ffdd86c3b2..d47abaac5b 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Microsoft.DotNet.DarcLib.csproj +++ b/src/Microsoft.DotNet.Darc/DarcLib/Microsoft.DotNet.DarcLib.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Models/Darc/DependencyGraph.cs b/src/Microsoft.DotNet.Darc/DarcLib/Models/Darc/DependencyGraph.cs index 9b0debd8e6..60ab96ea33 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Models/Darc/DependencyGraph.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Models/Darc/DependencyGraph.cs @@ -782,7 +782,7 @@ private static async Task GetRepoPathAsync( // If a repo folder or a mapping was not set we use the current parent's // parent folder. var gitClient = new LocalLibGit2Client( - new RemoteTokenProvider(null, null), + new RemoteTokenProvider((string)null, null), new NoTelemetryRecorder(), new ProcessManager(logger, gitExecutable), new FileSystem(), diff --git a/src/Microsoft.DotNet.Darc/DarcLib/RemoteRepoBase.cs b/src/Microsoft.DotNet.Darc/DarcLib/RemoteRepoBase.cs index 72d917f4d7..71f6cf0b82 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/RemoteRepoBase.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/RemoteRepoBase.cs @@ -1,9 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.DarcLib.Helpers; -using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.Logging; using System; using System.Collections.Generic; using System.Diagnostics; @@ -11,56 +8,42 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Maestro.Common; +using Microsoft.DotNet.DarcLib.Helpers; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; namespace Microsoft.DotNet.DarcLib; public class RemoteRepoBase : GitRepoCloner { private readonly ILogger _logger; - private readonly ProcessManager _processManager; + private readonly IProcessManager _processManager; protected RemoteRepoBase( - string gitExecutable, + IRemoteTokenProvider remoteConfiguration, + IProcessManager processManager, string temporaryRepositoryPath, IMemoryCache cache, - ILogger logger, - RemoteTokenProvider remoteConfiguration) - : this(gitExecutable, temporaryRepositoryPath, cache, logger, new ProcessManager(logger, gitExecutable), remoteConfiguration) - { - } - - private RemoteRepoBase( - string gitExecutable, - string temporaryRepositoryPath, - IMemoryCache cache, - ILogger logger, - ProcessManager processManager, - RemoteTokenProvider remoteConfiguration) + ILogger logger) : base(remoteConfiguration, new LocalLibGit2Client(remoteConfiguration, new NoTelemetryRecorder(), processManager, new FileSystem(), logger), logger) { - TemporaryRepositoryPath = temporaryRepositoryPath; - GitExecutable = gitExecutable; + TemporaryRepositoryPath = temporaryRepositoryPath ?? Path.GetTempPath(); Cache = cache; _logger = logger; _processManager = processManager; } - /// - /// Location of the git executable. Can be "git" or full path to temporary download location - /// used in the Maestro context. - /// - protected string GitExecutable { get; set; } - /// /// Location where repositories should be cloned. /// protected string TemporaryRepositoryPath { get; set; } - + /// /// Generic memory cache that may be supplied by the creator of the /// Remote for the purposes of caching remote responses. /// - protected IMemoryCache Cache { get; set;} + protected IMemoryCache Cache { get; set; } /// /// Cloning big repos takes a considerable amount of time when checking out the files. When diff --git a/src/Microsoft.DotNet.Darc/DarcLib/RemoteTokenProvider.cs b/src/Microsoft.DotNet.Darc/DarcLib/RemoteTokenProvider.cs index f7270300e9..5e586ca3df 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/RemoteTokenProvider.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/RemoteTokenProvider.cs @@ -1,35 +1,73 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#nullable enable using System; +using System.Threading.Tasks; +using Maestro.Common; +using Maestro.Common.AzureDevOpsTokens; using Microsoft.DotNet.DarcLib.Helpers; +#nullable enable namespace Microsoft.DotNet.DarcLib; -public class RemoteTokenProvider +public class RemoteTokenProvider : IRemoteTokenProvider { - public RemoteTokenProvider(string? gitHubToken = null, string? azureDevOpsToken = null) + private readonly IRemoteTokenProvider _azdoTokenProvider; + private readonly IRemoteTokenProvider _gitHubTokenProvider; + + public RemoteTokenProvider() { - GitHubToken = gitHubToken; - AzureDevOpsToken = azureDevOpsToken; + _azdoTokenProvider = new ResolvedTokenProvider(null); + _gitHubTokenProvider = new ResolvedTokenProvider(null); } - public static string GitRemoteUser => Constants.GitHubBotUserName; + public RemoteTokenProvider( + IRemoteTokenProvider azdoTokenProvider, + IRemoteTokenProvider gitHubTokenProvider) + { + _azdoTokenProvider = azdoTokenProvider; + _gitHubTokenProvider = gitHubTokenProvider; + } - public string? GitHubToken { get; } + public RemoteTokenProvider( + IAzureDevOpsTokenProvider azdoTokenProvider, + string? gitHubToken) + { + _azdoTokenProvider = azdoTokenProvider; + _gitHubTokenProvider = new ResolvedTokenProvider(gitHubToken); + } + public RemoteTokenProvider( + string? azdoToken, + string? gitHubToken) + { + _azdoTokenProvider = new ResolvedTokenProvider(azdoToken); + _gitHubTokenProvider = new ResolvedTokenProvider(gitHubToken); + } - public string? AzureDevOpsToken { get; } + public static string GitRemoteUser => Constants.GitHubBotUserName; + + public async Task GetTokenForRepositoryAsync(string repoUri) + { + var repoType = GitRepoUrlParser.ParseTypeFromUri(repoUri); + + return repoType switch + { + GitRepoType.GitHub => _gitHubTokenProvider.GetTokenForRepository(repoUri), + GitRepoType.AzureDevOps => await _azdoTokenProvider.GetTokenForRepositoryAsync(repoUri), + GitRepoType.Local => null, + _ => throw new NotImplementedException($"Unsupported repository remote {repoUri}"), + }; + } - public string? GetTokenForUri(string repoUri) + public string? GetTokenForRepository(string repoUri) { var repoType = GitRepoUrlParser.ParseTypeFromUri(repoUri); return repoType switch { - GitRepoType.GitHub => GitHubToken, - GitRepoType.AzureDevOps => AzureDevOpsToken, + GitRepoType.GitHub => _gitHubTokenProvider.GetTokenForRepository(repoUri), + GitRepoType.AzureDevOps => _azdoTokenProvider.GetTokenForRepositoryAsync(repoUri).GetAwaiter().GetResult(), GitRepoType.Local => null, _ => throw new NotImplementedException($"Unsupported repository remote {repoUri}"), }; diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs index 4e785e2ddb..fa4b0f1d86 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrRegistrations.cs @@ -5,6 +5,8 @@ using System.IO; using System.Net.Http; using System.Security.Cryptography.X509Certificates; +using Maestro.Common; +using Maestro.Common.AzureDevOpsTokens; using Microsoft.DotNet.Darc.Models.VirtualMonoRepo; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.Extensions.DependencyInjection; @@ -27,7 +29,16 @@ public static IServiceCollection AddVmrManagers( { // Configuration based registrations services.TryAddSingleton(new VmrInfo(Path.GetFullPath(vmrPath), Path.GetFullPath(tmpPath))); - services.TryAddSingleton(new RemoteTokenProvider(gitHubToken, azureDevOpsToken)); + services.TryAddSingleton(sp => + { + if (!string.IsNullOrEmpty(azureDevOpsToken)) + { + return new RemoteTokenProvider(azureDevOpsToken, gitHubToken); + } + + var azdoTokenProvider = sp.GetRequiredService(); + return new RemoteTokenProvider(azdoTokenProvider, gitHubToken); + }); services.TryAddTransient(sp => ActivatorUtilities.CreateInstance(sp, gitLocation)); services.TryAddTransient(sp => sp.GetRequiredService>()); @@ -36,6 +47,7 @@ public static IServiceCollection AddVmrManagers( services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); + services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); services.TryAddTransient(); @@ -60,6 +72,7 @@ public static IServiceCollection AddVmrManagers( services.TryAddTransient(); services.TryAddScoped(); services.TryAddScoped(); + services.TryAddSingleton(); services.AddHttpClient("GraphQL", httpClient => { diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Configuration/PcsConfiguration.cs b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/PcsConfiguration.cs index c1fdf1d219..c9ef796199 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Configuration/PcsConfiguration.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Configuration/PcsConfiguration.cs @@ -7,6 +7,7 @@ using ProductConstructionService.Api.Queue; using ProductConstructionService.Api.Controllers; using Microsoft.DotNet.Maestro.Client; +using Maestro.Common.AzureDevOpsTokens; namespace ProductConstructionService.Api.Configuration; @@ -34,9 +35,9 @@ public static string GetRequiredValue(this IConfiguration configuration, string /// Path to the VMR tmp folder /// Uri of the VMR /// Credentials used to authenticate to Azure Resources - /// ConnectionString to the BAR database /// Run service initialization? Currently this just means cloning the VMR /// Add endpoint authentication? + /// Add Swagger UI? /// Uri to used KeyVault public static void ConfigurePcs( this WebApplicationBuilder builder, @@ -68,6 +69,7 @@ public static void ConfigurePcs( }); builder.AddTelemetry(); + builder.Services.Configure("AzureDevOps", (o, s) => s.Bind(o)); builder.AddVmrRegistrations(vmrPath, tmpPath); builder.AddGitHubClientFactory(); builder.AddWorkitemQueues(azureCredential, waitForInitialization: initializeService); diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json index 60b13d1ce7..e13aeebd70 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Staging.json @@ -21,5 +21,10 @@ "ClientId": "baf98f1b-374e-487d-af42-aa33807f11e4", "UserRole": "User", "RedirectUri": "https://product-construction-int.wittytree-28a89311.westus2.azurecontainerapps.io/signin-oidc" + }, + "AzureDevOps": { + "default": { + "ManagedIdentityId": "system" + } } } diff --git a/test/FeedCleaner.Tests/FeedCleanerServiceTests.cs b/test/FeedCleaner.Tests/FeedCleanerServiceTests.cs index 02bfcdd866..96d19266a8 100644 --- a/test/FeedCleaner.Tests/FeedCleanerServiceTests.cs +++ b/test/FeedCleaner.Tests/FeedCleanerServiceTests.cs @@ -14,6 +14,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Moq; +using NuGet.Common; using NUnit.Framework; namespace FeedCleanerService.Tests; @@ -66,7 +67,10 @@ public void FeedCleanerServiceTests_SetUp() services.Configure( (options) => { - options.Tokens.Add(SomeAccount, "someToken"); + options[SomeAccount] = new() + { + Token = "someToken" + }; }); _provider = services.BuildServiceProvider(); @@ -157,12 +161,7 @@ private Mock SetupAzdoMock() private FeedCleanerService ConfigureFeedCleaner() { - FeedCleanerService cleaner = ActivatorUtilities.CreateInstance(_scope.ServiceProvider); - cleaner.AzureDevOpsClients = new Dictionary - { - { SomeAccount, _azdoMock.Object } - }; - return cleaner; + return ActivatorUtilities.CreateInstance(_scope.ServiceProvider, _azdoMock.Object); } private static void MarkVersionAsDeleted(List packages, string packageName, string version) diff --git a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs index da6c7496e3..c8e8c4876a 100644 --- a/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs +++ b/test/Maestro.ScenarioTests/MaestroScenarioTestBase.cs @@ -13,6 +13,7 @@ using FluentAssertions; using Maestro.MergePolicyEvaluation; using Maestro.ScenarioTests.ObjectHelpers; +using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.Internal.Testing.Utility; using Microsoft.DotNet.Maestro.Client; using Microsoft.DotNet.Maestro.Client.Models; @@ -20,7 +21,6 @@ using Newtonsoft.Json.Linq; using NuGet.Configuration; using NUnit.Framework; -using Octokit; [assembly: Parallelizable(ParallelScope.Fixtures)] @@ -30,13 +30,13 @@ namespace Maestro.ScenarioTests; internal abstract class MaestroScenarioTestBase { private TestParameters _parameters = null!; - private List _baseDarcRunArgs = new List(); + private List _baseDarcRunArgs = []; protected IMaestroApi MaestroApi => _parameters.MaestroApi; - protected GitHubClient GitHubApi => _parameters.GitHubApi; + protected Octokit.GitHubClient GitHubApi => _parameters.GitHubApi; - protected Microsoft.DotNet.DarcLib.AzureDevOpsClient AzDoClient => _parameters.AzDoClient; + protected AzureDevOpsClient AzDoClient => _parameters.AzDoClient; public void SetTestParameters(TestParameters parameters) { @@ -54,14 +54,14 @@ public void SetTestParameters(TestParameters parameters) } } - protected async Task WaitForPullRequestAsync(string targetRepo, string targetBranch) + protected async Task WaitForPullRequestAsync(string targetRepo, string targetBranch) { - Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); + Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); var attempts = 10; while (attempts-- > 0) { - IReadOnlyList prs = await GitHubApi.PullRequest.GetAllForRepository(repo.Id, new PullRequestRequest + IReadOnlyList prs = await GitHubApi.PullRequest.GetAllForRepository(repo.Id, new Octokit.PullRequestRequest { Base = targetBranch, }); @@ -82,10 +82,10 @@ protected async Task WaitForPullRequestAsync(string targetRepo, str throw new MaestroTestException($"No pull request was created in {targetRepo} targeting {targetBranch}"); } - private async Task WaitForUpdatedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 7) + private async Task WaitForUpdatedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 7) { - Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); - PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); + Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); + Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); while (attempts-- > 0) { @@ -104,8 +104,8 @@ private async Task WaitForUpdatedPullRequestAsync(string targetRepo private async Task WaitForMergedPullRequestAsync(string targetRepo, string targetBranch, int attempts = 7) { - Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); - PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); + Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepo); + Octokit.PullRequest pr = await WaitForPullRequestAsync(targetRepo, targetBranch); while (attempts-- > 0) { @@ -160,10 +160,10 @@ private async Task GetAzDoPullRequestIdAsync(string targetRepoName, string private async Task> SearchPullRequestsAsync(string repoUri, string targetPullRequestBranch) { - (var accountName, var projectName, var repoName) = Microsoft.DotNet.DarcLib.AzureDevOpsClient.ParseRepoUri(repoUri); + (var accountName, var projectName, var repoName) = AzureDevOpsClient.ParseRepoUri(repoUri); var query = new StringBuilder(); - Microsoft.DotNet.DarcLib.AzureDevOpsPrStatus prStatus = Microsoft.DotNet.DarcLib.AzureDevOpsPrStatus.Active; + AzureDevOpsPrStatus prStatus = AzureDevOpsPrStatus.Active; query.Append($"searchCriteria.status={prStatus.ToString().ToLower()}"); query.Append($"&searchCriteria.targetRefName=refs/heads/{targetPullRequestBranch}"); @@ -180,10 +180,10 @@ private async Task> SearchPullRequestsAsync(string repoUri, str return prs; } - private async Task> GetAzDoPullRequestAsync(int pullRequestId, string targetRepoName, string targetBranch, bool isUpdated, string? expectedPRTitle = null) + private async Task> GetAzDoPullRequestAsync(int pullRequestId, string targetRepoName, string targetBranch, bool isUpdated, string? expectedPRTitle = null) { var repoUri = GetAzDoRepoUrl(targetRepoName); - (var accountName, var projectName, var repoName) = Microsoft.DotNet.DarcLib.AzureDevOpsClient.ParseRepoUri(repoUri); + (var accountName, var projectName, var repoName) = AzureDevOpsClient.ParseRepoUri(repoUri); var apiBaseUrl = GetAzDoApiRepoUrl(targetRepoName); if (string.IsNullOrEmpty(expectedPRTitle)) @@ -193,7 +193,7 @@ private async Task> SearchPullRequestsAsync(string repoUri, str for (var tries = 10; tries > 0; tries--) { - Microsoft.DotNet.DarcLib.PullRequest pr = await AzDoClient.GetPullRequestAsync($"{apiBaseUrl}/pullRequests/{pullRequestId}"); + PullRequest pr = await AzDoClient.GetPullRequestAsync($"{apiBaseUrl}/pullRequests/{pullRequestId}"); var trimmedTitle = Regex.Replace(pr.Title, @"\s+", " "); if (!isUpdated || trimmedTitle == expectedPRTitle) @@ -227,7 +227,7 @@ private async Task> SearchPullRequestsAsync(string repoUri, str } protected async Task CheckBatchedGitHubPullRequest(string targetBranch, string[] sourceRepoNames, - string targetRepoName, List expectedDependencies, string repoDirectory) + string targetRepoName, List expectedDependencies, string repoDirectory) { var repoNames = sourceRepoNames .Select(name => $"{_parameters.GitHubTestOrg}/{name}") @@ -238,17 +238,17 @@ protected async Task CheckBatchedGitHubPullRequest(string targetBranch, string[] } protected async Task CheckNonBatchedGitHubPullRequest(string sourceRepoName, string targetRepoName, string targetBranch, - List expectedDependencies, string repoDirectory, bool isCompleted = false, bool isUpdated = false) + List expectedDependencies, string repoDirectory, bool isCompleted = false, bool isUpdated = false) { var expectedPRTitle = $"[{targetBranch}] Update dependencies from {_parameters.GitHubTestOrg}/{sourceRepoName}"; await CheckGitHubPullRequest(expectedPRTitle, targetRepoName, targetBranch, expectedDependencies, repoDirectory, isCompleted, isUpdated); } protected async Task CheckGitHubPullRequest(string expectedPRTitle, string targetRepoName, string targetBranch, - List expectedDependencies, string repoDirectory, bool isCompleted, bool isUpdated) + List expectedDependencies, string repoDirectory, bool isCompleted, bool isUpdated) { TestContext.WriteLine($"Checking opened PR in {targetBranch} {targetRepoName}"); - PullRequest pullRequest = isUpdated + Octokit.PullRequest pullRequest = isUpdated ? await WaitForUpdatedPullRequestAsync(targetRepoName, targetBranch) : await WaitForPullRequestAsync(targetRepoName, targetBranch); @@ -271,7 +271,7 @@ protected async Task CheckBatchedAzDoPullRequest( string[] sourceRepoNames, string targetRepoName, string targetBranch, - List expectedDependencies, + List expectedDependencies, string repoDirectory, bool complete = false) { @@ -287,7 +287,7 @@ protected async Task CheckNonBatchedAzDoPullRequest( string sourceRepoName, string targetRepoName, string targetBranch, - List expectedDependencies, + List expectedDependencies, string repoDirectory, bool isCompleted = false, bool isUpdated = false, @@ -303,7 +303,7 @@ protected async Task CheckAzDoPullRequest( string expectedPRTitle, string targetRepoName, string targetBranch, - List expectedDependencies, + List expectedDependencies, string repoDirectory, bool isCompleted, bool isUpdated, @@ -313,12 +313,12 @@ protected async Task CheckAzDoPullRequest( var targetRepoUri = GetAzDoApiRepoUrl(targetRepoName); TestContext.WriteLine($"Checking Opened PR in {targetBranch} {targetRepoUri} ..."); var pullRequestId = await GetAzDoPullRequestIdAsync(targetRepoName, targetBranch); - await using AsyncDisposableValue pullRequest = await GetAzDoPullRequestAsync(pullRequestId, targetRepoName, targetBranch, isUpdated, expectedPRTitle); + await using AsyncDisposableValue pullRequest = await GetAzDoPullRequestAsync(pullRequestId, targetRepoName, targetBranch, isUpdated, expectedPRTitle); var trimmedTitle = Regex.Replace(pullRequest.Value.Title, @"\s+", " "); trimmedTitle.Should().Be(expectedPRTitle); - Microsoft.DotNet.DarcLib.PrStatus expectedPRState = isCompleted ? Microsoft.DotNet.DarcLib.PrStatus.Closed : Microsoft.DotNet.DarcLib.PrStatus.Open; + PrStatus expectedPRState = isCompleted ? PrStatus.Closed : PrStatus.Open; var prStatus = await AzDoClient.GetPullRequestStatusAsync(GetAzDoApiRepoUrl(targetRepoName) + $"/pullRequests/{pullRequestId}"); prStatus.Should().Be(expectedPRState); @@ -340,7 +340,7 @@ protected async Task CheckAzDoPullRequest( } } - private async Task ValidatePullRequestDependencies(string pullRequestBaseBranch, List expectedDependencies, int tries = 1) + private async Task ValidatePullRequestDependencies(string pullRequestBaseBranch, List expectedDependencies, int tries = 1) { var triesRemaining = tries; while (triesRemaining-- > 0) @@ -885,14 +885,14 @@ protected async Task GetRepositoryPolicies(string repoUri, string branch return await RunDarcAsync("get-repository-policies", "--all", "--repo", repoUri, "--branch", branchName); } - protected async Task WaitForMergedPullRequestAsync(string targetRepo, string targetBranch, PullRequest pr, Repository repo, int attempts = 7) + protected async Task WaitForMergedPullRequestAsync(string targetRepo, string targetBranch, Octokit.PullRequest pr, Octokit.Repository repo, int attempts = 7) { while (attempts-- > 0) { TestContext.WriteLine($"Starting check for merge, attempts remaining {attempts}"); pr = await GitHubApi.PullRequest.Get(repo.Id, pr.Number); - if (pr.State == ItemState.Closed) + if (pr.State == Octokit.ItemState.Closed) { return; } @@ -906,18 +906,18 @@ protected async Task WaitForMergedPullRequestAsync(string targetRepo, string tar protected async Task CheckGithubPullRequestChecks(string targetRepoName, string targetBranch) { TestContext.WriteLine($"Checking opened PR in {targetBranch} {targetRepoName}"); - PullRequest pullRequest = await WaitForPullRequestAsync(targetRepoName, targetBranch); - Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepoName); + Octokit.PullRequest pullRequest = await WaitForPullRequestAsync(targetRepoName, targetBranch); + Octokit.Repository repo = await GitHubApi.Repository.Get(_parameters.GitHubTestOrg, targetRepoName); return await ValidateGithubMaestroCheckRunsSuccessful(targetRepoName, targetBranch, pullRequest, repo); } - protected async Task ValidateGithubMaestroCheckRunsSuccessful(string targetRepoName, string targetBranch, PullRequest pullRequest, Repository repo) + protected async Task ValidateGithubMaestroCheckRunsSuccessful(string targetRepoName, string targetBranch, Octokit.PullRequest pullRequest, Octokit.Repository repo) { // Waiting 5 minutes 30 seconds for maestro to add the checks to the PR (it takes 5 minutes for the checks to be added) await Task.Delay(TimeSpan.FromSeconds(5 * 60 + 30)); TestContext.WriteLine($"Checking maestro merge policies check in {targetBranch} {targetRepoName}"); - CheckRunsResponse existingCheckRuns = await GitHubApi.Check.Run.GetAllForReference(repo.Id, pullRequest.Head.Sha); + Octokit.CheckRunsResponse existingCheckRuns = await GitHubApi.Check.Run.GetAllForReference(repo.Id, pullRequest.Head.Sha); var cnt = 0; foreach (var checkRun in existingCheckRuns.CheckRuns) { diff --git a/test/Maestro.ScenarioTests/TestParameters.cs b/test/Maestro.ScenarioTests/TestParameters.cs index cbae51745e..1717cd5f34 100644 --- a/test/Maestro.ScenarioTests/TestParameters.cs +++ b/test/Maestro.ScenarioTests/TestParameters.cs @@ -8,10 +8,12 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Threading.Tasks; +using Maestro.Common.AzureDevOpsTokens; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.Internal.Testing.Utility; using Microsoft.DotNet.Maestro.Client; using Microsoft.Extensions.Configuration; -using Octokit; using Octokit.Internal; #nullable enable @@ -67,7 +69,7 @@ public static async Task GetAsync(bool useNonPrimaryEndpoint = f federatedToken: null, disableInteractiveAuth: isCI); - string darcRootDir = darcDir ?? ""; + string? darcRootDir = darcDir; if (string.IsNullOrEmpty(darcRootDir)) { await InstallDarc(maestroApi, testDirSharedWrapper); @@ -75,17 +77,29 @@ public static async Task GetAsync(bool useNonPrimaryEndpoint = f } string darcExe = Path.Join(darcRootDir, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "darc.exe" : "darc"); + string git = await TestHelpers.Which("git"); + var tokenProvider = AzureDevOpsTokenProvider.FromStaticOptions(new() + { + ["default"] = new() + { + Token = azdoToken + } + }); Assembly assembly = typeof(TestParameters).Assembly; var githubApi = - new GitHubClient( - new ProductHeaderValue(assembly.GetName().Name, assembly.GetCustomAttribute()?.InformationalVersion), - new InMemoryCredentialStore(new Credentials(githubToken))); + new Octokit.GitHubClient( + new Octokit.ProductHeaderValue(assembly.GetName().Name, assembly.GetCustomAttribute()?.InformationalVersion), + new InMemoryCredentialStore(new Octokit.Credentials(githubToken))); var azDoClient = - new Microsoft.DotNet.DarcLib.AzureDevOpsClient(await TestHelpers.Which("git"), azdoToken, new NUnitLogger(), testDirSharedWrapper.TryTake()!.Directory); + new AzureDevOpsClient( + tokenProvider, + new ProcessManager(new NUnitLogger(), git), + new NUnitLogger(), + testDirSharedWrapper.TryTake()!.Directory); return new TestParameters( - darcExe, await TestHelpers.Which("git"), maestroBaseUri, maestroToken, githubToken, maestroApi, githubApi, azDoClient, testDir, azdoToken, isCI); + darcExe, git, maestroBaseUri, maestroToken, githubToken, maestroApi, githubApi, azDoClient, testDir, azdoToken, isCI); } private static async Task InstallDarc(IMaestroApi maestroApi, Shareable toolPath) @@ -110,8 +124,18 @@ private static async Task InstallDarc(IMaestroApi maestroApi, Shareable