From 01b38504640f4bacc41dca3eb9c8ddbf00e1bd58 Mon Sep 17 00:00:00 2001 From: Nathan Mytelka Date: Thu, 28 Sep 2023 15:13:08 -0700 Subject: [PATCH 01/93] Switch to 8.0.2xx branding --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 209c35a196..a734d6d89b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.100 + 8.0.200 false release From a724e419d4f09de512bac9bf5bd2f8f113c640e5 Mon Sep 17 00:00:00 2001 From: Forgind <12969783+Forgind@users.noreply.github.com> Date: Wed, 4 Oct 2023 10:49:25 -0700 Subject: [PATCH 02/93] preview branding --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index a734d6d89b..f93c27e84d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -10,7 +10,7 @@ release true - rtm + preview rtm servicing From 19459588c0e1bac3f312cbcc5c482fd9a971fc35 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 29 Sep 2023 13:55:10 -0700 Subject: [PATCH 03/93] When determining the choices of a symbol, it was accidentally using all strings instead of the grouping for that particular symbol. --- .../LocalizationModelDeserializer.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/LocalizationModelDeserializer.cs b/src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/LocalizationModelDeserializer.cs index c30dc9a7a2..411f80c77e 100644 --- a/src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/LocalizationModelDeserializer.cs +++ b/src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/LocalizationModelDeserializer.cs @@ -74,9 +74,9 @@ private static IReadOnlyDictionary Lo string? displayName = parameterParts.SingleOrDefault(p => p.NameParts.Skip(1).FirstOrDefault() == "displayName").LocalizedString; string? description = parameterParts.SingleOrDefault(p => p.NameParts.Skip(1).FirstOrDefault() == "description").LocalizedString; - IReadOnlyDictionary? choiceModels = LoadChoiceModels(strings - .Where(s => s.NameParts.Skip(1).FirstOrDefault() == "choices") - .Select(s => (s.NameParts.Skip(2), s.LocalizedString))); + IReadOnlyDictionary? choiceModels = LoadChoiceModels(parameterParts + .Where(p => p.NameParts.Skip(1).FirstOrDefault() == "choices") + .Select(p => (p.NameParts.Skip(2), p.LocalizedString))); ParameterSymbolLocalizationModel paramLoc = new ParameterSymbolLocalizationModel( symbolName, From 1ea86362de5c39350512f6897aca813d428938f4 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 29 Sep 2023 15:16:27 -0700 Subject: [PATCH 04/93] Added entries to the preexisting localization validation tests that would fail if the fix wasn't implemented. --- .../.template.config/localize/templatestrings.de-DE.json | 8 ++++++-- .../.template.config/localize/templatestrings.tr.json | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.TemplateEngine.TestTemplates/test_templates/Invalid/Localization/ValidationFailure/.template.config/localize/templatestrings.de-DE.json b/test/Microsoft.TemplateEngine.TestTemplates/test_templates/Invalid/Localization/ValidationFailure/.template.config/localize/templatestrings.de-DE.json index 28a909b0b2..5d54fc6aa7 100644 --- a/test/Microsoft.TemplateEngine.TestTemplates/test_templates/Invalid/Localization/ValidationFailure/.template.config/localize/templatestrings.de-DE.json +++ b/test/Microsoft.TemplateEngine.TestTemplates/test_templates/Invalid/Localization/ValidationFailure/.template.config/localize/templatestrings.de-DE.json @@ -4,8 +4,12 @@ "symbols/someSymbol/displayName": "sym0_displayName_de-DE:äÄßöÖüÜ", "symbols/someSymbol/description": "sym0_desc_de-DE:äÄßöÖüÜ", "symbols/someChoice/description": "sym1_desc_de-DE:äÄßöÖüÜ", - "symbols/someChoice/choices/choice0/description": "sym1_choice0_de-DE:äÄßöÖüÜ", - "symbols/someChoice/choices/choice1/description": "sym1_choice1_de-DE:äÄßöÖüÜ", + "symbols/someChoice/choices/choice0/displayName": "sym1_choice0_de-DE_dn:äÄßöÖüÜ", + "symbols/someChoice/choices/choice0/description": "sym1_choice0_de-DE_d:äÄßöÖüÜ", + "symbols/someChoice/choices/choice1/displayName": "sym1_choice1_de-DE_dn:äÄßöÖüÜ", + "symbols/someChoice/choices/choice1/description": "sym1_choice1_de-DE_d:äÄßöÖüÜ", + "symbols/someOtherChoice/choices/choice0/displayName": "sym1_choice0_de-DE_dn:äÄßöÖüÜ", + "symbols/someOtherChoice/choices/choice1/description": "sym1_choice1_de-DE_d:äÄßöÖüÜ", "postActions/pa0/description": "pa0_desc_de-DE:äÄßöÖüÜ", "postActions/pa0/manualInstructions/first_instruction/text": "pa0_manualInstructions_de-DE:äÄßöÖüÜ", "postActions/pa1/description": "pa1_desc_de-DE:äÄßöÖüÜ", diff --git a/test/Microsoft.TemplateEngine.TestTemplates/test_templates/Invalid/Localization/ValidationFailure/.template.config/localize/templatestrings.tr.json b/test/Microsoft.TemplateEngine.TestTemplates/test_templates/Invalid/Localization/ValidationFailure/.template.config/localize/templatestrings.tr.json index 083c9e3585..5206e03692 100644 --- a/test/Microsoft.TemplateEngine.TestTemplates/test_templates/Invalid/Localization/ValidationFailure/.template.config/localize/templatestrings.tr.json +++ b/test/Microsoft.TemplateEngine.TestTemplates/test_templates/Invalid/Localization/ValidationFailure/.template.config/localize/templatestrings.tr.json @@ -7,6 +7,8 @@ "symbols/someChoice/displayName": "sym1_displayName_tr-TR:çÇğĞıIİöÖşŞüÜ", "symbols/someChoice/choices/choice0/description": "sym1_choice0_tr-TR:çÇğĞıIİöÖşŞüÜ", "symbols/someChoice/choices/choice2/description": "sym1_choice2_tr-TR:çÇğĞıIİöÖşŞüÜ", + "symbols/someOtherChoice/choices/choice0/description": "sym1_choice0_tr-TR:çÇğĞıIİöÖşŞüÜ", + "symbols/someOtherChoice/choices/choice2/description": "sym1_choice2_tr-TR:çÇğĞıIİöÖşŞüÜ", "postActions/pa6/description": "pa0_desc_tr-TR:çÇğĞıIİöÖşŞüÜ", "postActions/pa6/manualInstructions/first_instruction/text": "pa0_manualInstructions_tr-TR:çÇğĞıIİöÖşŞüÜ" } From cd2ff13129d995e848f934e0d911ae360c692552 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 28 Oct 2023 12:41:39 +0000 Subject: [PATCH 05/93] Update dependencies from https://github.com/dotnet/arcade build 20231025.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.23516.4 -> To Version 8.0.0-beta.23525.4 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 27d79d0359..2a11b1bcff 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -29,9 +29,9 @@ - + https://github.com/dotnet/arcade - 39042b4048580366d35a7c1c4f4ce8fc0dbea4b4 + a57022b44f3ff23de925530ea1d27da9701aed57 diff --git a/global.json b/global.json index ee1a484810..ed219c9d22 100644 --- a/global.json +++ b/global.json @@ -8,6 +8,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23516.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23525.4" } } From 5613804b811e7b0f5fb5f33f0d4309b5d4fa96a6 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sun, 29 Oct 2023 12:39:56 +0000 Subject: [PATCH 06/93] Update dependencies from https://github.com/dotnet/arcade build 20231025.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.23516.4 -> To Version 8.0.0-beta.23525.4 From 6172a1e7de455249d1674ec7569b578f41025a26 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Tue, 31 Oct 2023 09:51:37 -0700 Subject: [PATCH 07/93] Switch back to the shipped RC2 version. We should explore removing this dependency entirely. --- NuGet.config | 1 - eng/Version.Details.xml | 18 +++++++++--------- eng/Versions.props | 6 +++--- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/NuGet.config b/NuGet.config index f8e8fdfd00..d0ff87cda9 100644 --- a/NuGet.config +++ b/NuGet.config @@ -4,7 +4,6 @@ - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2a11b1bcff..6099e4cbd0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -11,17 +11,17 @@ b4fa7f2e1e65ef49881be2ab2df27624280a8c55 - - https://github.com/dotnet/runtime - 59edaad404d1b8e47080015ae8d0787f94c970df + + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime + 0b25e38ad32a69cd83ae246104b32449203cc71c - - https://github.com/dotnet/runtime - 59edaad404d1b8e47080015ae8d0787f94c970df + + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime + 0b25e38ad32a69cd83ae246104b32449203cc71c - - https://github.com/dotnet/runtime - 59edaad404d1b8e47080015ae8d0787f94c970df + + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime + 0b25e38ad32a69cd83ae246104b32449203cc71c https://github.com/dotnet/command-line-api diff --git a/eng/Versions.props b/eng/Versions.props index 36e0b12f28..1521ae1321 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -22,8 +22,8 @@ 2.0.0-beta4.23407.1 - 8.0.0 - 8.0.0 - 8.0.0-rtm.23524.7 + 8.0.0-rc.2.23479.6 + 8.0.0-rc.2.23479.6 + 8.0.0-rc.2.23479.6 From 7cf6402632f0b5200d04e750d4796ea2759d1aab Mon Sep 17 00:00:00 2001 From: Jacques Eloff Date: Tue, 31 Oct 2023 09:40:30 -0700 Subject: [PATCH 08/93] Turn of stable versions --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 1521ae1321..a88b1fe2b8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -6,7 +6,7 @@ true 8.0.200 - true + false release true From b0486af2117b1294d74a143224ebfcc8b16ebf54 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Thu, 2 Nov 2023 13:14:52 -0700 Subject: [PATCH 09/93] Remove the blob feed per https://github.com/dotnet/arcade/issues/13963 --- azure-pipelines.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4f50f3fc0d..f8cf65dbfe 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -80,10 +80,8 @@ stages: # Only enable publishing in non-public, non PR scenarios. - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1 # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat, symweb-symbol-server-pat # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT - - group: DotNet-Blob-Feed - group: Publish-Build-Assets - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) @@ -96,10 +94,8 @@ stages: # PRs or external builds are not signed. ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: _SignType: test - _DotNetPublishToBlobFeed : false ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: _SignType: real - _DotNetPublishToBlobFeed : true ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: Build_Debug: _BuildConfig: Debug From bfe4f83da1a2e62b1b0896bbf0f838f727ce4f43 Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Fri, 3 Nov 2023 14:35:41 -0700 Subject: [PATCH 10/93] Removed the dependency on dotnet/runtime from the repo. --- Directory.Build.targets | 8 -------- eng/Version.Details.xml | 12 ------------ eng/Versions.props | 4 ---- global.json | 7 +------ 4 files changed, 1 insertion(+), 30 deletions(-) diff --git a/Directory.Build.targets b/Directory.Build.targets index d70f9ae4cc..2ec401282f 100644 --- a/Directory.Build.targets +++ b/Directory.Build.targets @@ -4,14 +4,6 @@ - - - - - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6099e4cbd0..3a5a22a16e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -11,18 +11,6 @@ b4fa7f2e1e65ef49881be2ab2df27624280a8c55 - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0b25e38ad32a69cd83ae246104b32449203cc71c - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0b25e38ad32a69cd83ae246104b32449203cc71c - - - https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - 0b25e38ad32a69cd83ae246104b32449203cc71c - https://github.com/dotnet/command-line-api a045dd54a4c44723c215d992288160eb1401bb7f diff --git a/eng/Versions.props b/eng/Versions.props index a88b1fe2b8..1908fcbe0e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -21,9 +21,5 @@ 2.0.0-beta4.23407.1 - - 8.0.0-rc.2.23479.6 - 8.0.0-rc.2.23479.6 - 8.0.0-rc.2.23479.6 diff --git a/global.json b/global.json index ed219c9d22..3890b72bec 100644 --- a/global.json +++ b/global.json @@ -1,11 +1,6 @@ { "tools": { - "dotnet": "8.0.100-rtm.23506.1", - "runtimes": { - "dotnet": [ - "$(VSRedistCommonNetCoreSharedFrameworkx6480PackageVersion)" - ] - } + "dotnet": "8.0.100-rtm.23506.1" }, "msbuild-sdks": { "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23525.4" From 185f6dce90f0766a9f8310548bf621402459ea98 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 8 Nov 2023 13:46:06 +0000 Subject: [PATCH 11/93] Update dependencies from https://github.com/dotnet/arcade build 20231106.5 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.23525.4 -> To Version 8.0.0-beta.23556.5 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3a5a22a16e..7554db9a5f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - a57022b44f3ff23de925530ea1d27da9701aed57 + 080141bf0f9f15408bb6eb8e301b23bddf81d054 diff --git a/global.json b/global.json index 3890b72bec..5a7488a676 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.100-rtm.23506.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23525.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23556.5" } } From e8f176ec51b3ccd8aad99c3fb42d7930a23ce20b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 15 Nov 2023 16:33:24 +0000 Subject: [PATCH 12/93] Update dependencies from https://github.com/dotnet/arcade build 20231114.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.23556.5 -> To Version 8.0.0-beta.23564.4 --- eng/Version.Details.xml | 4 ++-- global.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7554db9a5f..d71ba8ab3e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 080141bf0f9f15408bb6eb8e301b23bddf81d054 + 0aaeafef60933f87b0b50350313bb2fd77defb5d diff --git a/global.json b/global.json index 5a7488a676..914ede0eac 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "8.0.100-rtm.23506.1" + "dotnet": "8.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23556.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23564.4" } } From 37d98e40bacd74eb1e78a234e06efde42dc14174 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 10 Jan 2024 17:02:42 +0000 Subject: [PATCH 13/93] Update dependencies from https://github.com/dotnet/arcade build 20240109.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.23564.4 -> To Version 8.0.0-beta.24059.4 --- eng/Version.Details.xml | 4 ++-- eng/common/darc-init.ps1 | 2 +- eng/common/darc-init.sh | 2 +- eng/common/post-build/add-build-to-channel.ps1 | 2 +- eng/common/post-build/publish-using-darc.ps1 | 2 +- eng/common/post-build/trigger-subscriptions.ps1 | 2 +- eng/common/templates/job/job.yml | 2 +- eng/common/templates/job/publish-build-assets.yml | 4 ++-- eng/common/templates/post-build/common-variables.yml | 2 +- eng/common/templates/post-build/post-build.yml | 4 ++-- eng/common/tools.ps1 | 10 +++++++++- eng/common/tools.sh | 7 ++++++- global.json | 4 ++-- 13 files changed, 30 insertions(+), 17 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d71ba8ab3e..1d897bff15 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 0aaeafef60933f87b0b50350313bb2fd77defb5d + 61ae141d2bf3534619265c8f691fd55dc3e75147 diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 435e764134..8fda30bdce 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -1,6 +1,6 @@ param ( $darcVersion = $null, - $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16', + $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16', $verbosity = 'minimal', $toolpath = $null ) diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 84c1d0cc2e..c305ae6bd7 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -2,7 +2,7 @@ source="${BASH_SOURCE[0]}" darcVersion='' -versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16' +versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16' verbosity='minimal' while [[ $# > 0 ]]; do diff --git a/eng/common/post-build/add-build-to-channel.ps1 b/eng/common/post-build/add-build-to-channel.ps1 index de2d957922..49938f0c89 100644 --- a/eng/common/post-build/add-build-to-channel.ps1 +++ b/eng/common/post-build/add-build-to-channel.ps1 @@ -2,7 +2,7 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' ) diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 8508397d77..1e779fec4d 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -3,7 +3,7 @@ param( [Parameter(Mandatory=$true)][int] $PublishingInfraVersion, [Parameter(Mandatory=$true)][string] $AzdoToken, [Parameter(Mandatory=$true)][string] $MaestroToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters diff --git a/eng/common/post-build/trigger-subscriptions.ps1 b/eng/common/post-build/trigger-subscriptions.ps1 index 55dea518ac..ac9a95778f 100644 --- a/eng/common/post-build/trigger-subscriptions.ps1 +++ b/eng/common/post-build/trigger-subscriptions.ps1 @@ -2,7 +2,7 @@ param( [Parameter(Mandatory=$true)][string] $SourceRepo, [Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, - [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16' ) diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index e20ee3a983..e24ca2f46f 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -136,7 +136,7 @@ jobs: condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@1 - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - task: DownloadPipelineArtifact@2 diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 42017109f3..fa5446c093 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -72,7 +72,7 @@ jobs: condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@1 - task: PowerShell@2 displayName: Publish Build Assets @@ -81,7 +81,7 @@ jobs: arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index c24193acfc..173914f236 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -7,7 +7,7 @@ variables: # Default Maestro++ API Endpoint and API Version - name: MaestroApiEndPoint - value: "https://maestro-prod.westus2.cloudapp.azure.com" + value: "https://maestro.dot.net" - name: MaestroApiAccessToken value: $(MaestroAccessToken) - name: MaestroApiVersion diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index ef720f9d78..3f74abf7ce 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -169,7 +169,7 @@ stages: # This is necessary whenever we want to publish/restore to an AzDO private feed # Since sdk-task.ps1 tries to restore packages we need to do this authentication here # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@1 displayName: 'Authenticate to AzDO Feeds' # Signing validation will optionally work with the buildmanifest file which is downloaded from @@ -266,7 +266,7 @@ stages: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: NuGetAuthenticate@0 + - task: NuGetAuthenticate@1 - task: PowerShell@2 displayName: Publish Using Darc diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index fdd0cbb91f..eb188cfda4 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -601,7 +601,15 @@ function InitializeBuildTool() { ExitWithExitCode 1 } $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'net8.0' } + + # Use override if it exists - commonly set by source-build + if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) { + $initializeBuildToolFramework="net8.0" + } else { + $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework + } + + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore diff --git a/eng/common/tools.sh b/eng/common/tools.sh index e8d4789433..3392e3a999 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -341,7 +341,12 @@ function InitializeBuildTool { # return values _InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildToolCommand="msbuild" - _InitializeBuildToolFramework="net8.0" + # use override if it exists - commonly set by source-build + if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then + _InitializeBuildToolFramework="net8.0" + else + _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}" + fi } # Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 diff --git a/global.json b/global.json index 914ede0eac..eb070263e0 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "8.0.100" + "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23564.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24059.4" } } From 9a418d1ad8fa0e8fd02e3b5ec77a77b49cbe281a Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Fri, 12 Jan 2024 11:47:02 -0800 Subject: [PATCH 14/93] Stabilize branding --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 1908fcbe0e..6243249b5b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -6,7 +6,7 @@ true 8.0.200 - false + true release true From 58bb944a4856f3f36753bff1fa581fc7cfbb8cda Mon Sep 17 00:00:00 2001 From: Forgind <12969783+Forgind@users.noreply.github.com> Date: Fri, 12 Jan 2024 14:27:06 -0800 Subject: [PATCH 15/93] 3xx branding (#7449) --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 1908fcbe0e..fba21eb049 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.200 + 8.0.300 false release From 968cf1235b7a34a078cd0f0e4b5eeb1c5358cf43 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 12 Jan 2024 23:40:42 +0000 Subject: [PATCH 16/93] Update dependencies from https://github.com/dotnet/arcade build 20240110.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24059.4 -> To Version 8.0.0-beta.24060.4 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1d897bff15..beb7cbacfb 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + 888985fb9a9ae4cb30bca75f98af9126c839e660 diff --git a/global.json b/global.json index eb070263e0..0c13a5e3cd 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24059.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24060.4" } } From 476f5b2383ce7d86e4ceb2b9adfc05b3130dab54 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 13 Jan 2024 14:12:10 +0000 Subject: [PATCH 17/93] Update dependencies from https://github.com/dotnet/arcade build 20240110.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24059.4 -> To Version 8.0.0-beta.24060.4 From 3561475bf2be0eafc8a61670a15e586ce4b20127 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sun, 14 Jan 2024 14:03:35 +0000 Subject: [PATCH 18/93] Update dependencies from https://github.com/dotnet/arcade build 20240110.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24059.4 -> To Version 8.0.0-beta.24060.4 From 79e628df318fefe46105c2d9b1427ab2b94f9323 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 24 Jan 2024 14:06:12 +0000 Subject: [PATCH 19/93] Update dependencies from https://github.com/dotnet/arcade build 20240124.2 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24060.4 -> To Version 8.0.0-beta.24074.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index beb7cbacfb..16745f7681 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 888985fb9a9ae4cb30bca75f98af9126c839e660 + 96c2cee493aa1542c0b06a6498e0379eb11e005f diff --git a/global.json b/global.json index 0c13a5e3cd..4d640c9349 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24060.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24074.2" } } From e3a3796b2de947e572d094a8b38245a50ce65e8c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 27 Jan 2024 14:02:43 +0000 Subject: [PATCH 20/93] Update dependencies from https://github.com/dotnet/arcade build 20240125.5 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24074.2 -> To Version 8.0.0-beta.24075.5 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 16745f7681..c3dc0f4ef7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 96c2cee493aa1542c0b06a6498e0379eb11e005f + 07cf24f27ee58b5d1a9662334a101d84bd1e07e5 diff --git a/global.json b/global.json index 4d640c9349..bca72c96f1 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24074.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24075.5" } } From 1f2e6f85a108d4fa362edf1cefa943e4fd44eebb Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sun, 28 Jan 2024 13:45:53 +0000 Subject: [PATCH 21/93] Update dependencies from https://github.com/dotnet/arcade build 20240125.5 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24074.2 -> To Version 8.0.0-beta.24075.5 From ae06d0103fe772dccdbb2e8d493261e06443a262 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 1 Feb 2024 14:05:12 +0000 Subject: [PATCH 22/93] Update dependencies from https://github.com/dotnet/arcade build 20240131.5 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24075.5 -> To Version 8.0.0-beta.24081.5 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c3dc0f4ef7..09738d9c30 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 07cf24f27ee58b5d1a9662334a101d84bd1e07e5 + be88b08c41971b52ec11aec05ef31e72185d4a1f diff --git a/global.json b/global.json index bca72c96f1..13ae146aec 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24075.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24081.5" } } From 4306af3aa96b2404792723e17f7c93898df19b8f Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Tue, 6 Feb 2024 11:08:55 -0800 Subject: [PATCH 23/93] Update branding to 8.0.201 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 6243249b5b..31d5e1e566 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.200 + 8.0.201 true release From 0e2e820ac6ae24f8f1f29a069cd88266969e833d Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Wed, 14 Feb 2024 15:53:42 -0800 Subject: [PATCH 24/93] Update branding --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 31d5e1e566..ebb323cdec 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.201 + 8.0.202 true release From d28ef1a8222095d1571548c9f5599bd52d74c61a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 16 Feb 2024 07:07:50 +0000 Subject: [PATCH 25/93] Update dependencies from https://github.com/dotnet/arcade build 20240213.2 (#7563) [release/8.0.2xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/post-build/publish-using-darc.ps1 | 4 ++-- .../templates/job/publish-build-assets.yml | 14 +++++++------- eng/common/templates/post-build/post-build.yml | 16 ++++++++-------- .../templates/variables/pool-providers.yml | 12 ++++++------ global.json | 2 +- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1d897bff15..19d7f6abf7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 61ae141d2bf3534619265c8f691fd55dc3e75147 + da98edc4c3ea539f109ea320672136ceb32591a7 diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 1e779fec4d..5a3a32ea8d 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -12,7 +12,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - $darc = Get-Darc + $darc = Get-Darc $optionalParams = [System.Collections.ArrayList]::new() @@ -46,7 +46,7 @@ try { } Write-Host 'done.' -} +} catch { Write-Host $_ Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels." diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index fa5446c093..8ec0151def 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -58,7 +58,7 @@ jobs: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2019.amd64 steps: @@ -71,7 +71,7 @@ jobs: checkDownloadedFiles: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: NuGetAuthenticate@1 - task: PowerShell@2 @@ -86,7 +86,7 @@ jobs: /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -95,7 +95,7 @@ jobs: Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - + - task: PublishBuildArtifacts@1 displayName: Publish ReleaseConfigs Artifact inputs: @@ -121,7 +121,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') + condition: eq(variables['SymbolExclusionFile'], 'true') inputs: PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' PublishLocation: Container @@ -137,7 +137,7 @@ jobs: displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' @@ -148,4 +148,4 @@ jobs: - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - template: /eng/common/templates/steps/publish-logs.yml parameters: - JobLabel: 'Publish_Artifacts_Logs' + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 3f74abf7ce..aba44a25a3 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -39,7 +39,7 @@ parameters: displayName: Enable NuGet validation type: boolean default: true - + - name: publishInstallersAndChecksums displayName: Publish installers and checksums type: boolean @@ -131,8 +131,8 @@ stages: displayName: Validate inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - job: displayName: Signing Validation @@ -221,9 +221,9 @@ stages: displayName: Validate inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) -GHCommit $(Build.SourceVersion) -SourcelinkCliVersion $(SourceLinkCLIVersion) continueOnError: true @@ -258,7 +258,7 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml @@ -272,7 +272,7 @@ stages: displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index 9cc5c550d3..d236f9fdbb 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -1,15 +1,15 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, # otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. -# Motivation: +# Motivation: # Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS # (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing # (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. -# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services -# team needs to move resources around and create new and potentially differently-named pools. Using this template +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template # file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. -# How to use: +# How to use: # This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). # If we find alternate naming conventions in broad usage it can be added to the condition below. # @@ -54,4 +54,4 @@ variables: False, 'NetCore1ESPool-Internal' ) - ] \ No newline at end of file + ] diff --git a/global.json b/global.json index eb070263e0..412cda8d3e 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24059.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24113.2" } } From 82f5e89616d0c7a6f2f6f21d02181ef49de26b52 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 16 Feb 2024 07:08:07 +0000 Subject: [PATCH 26/93] [release/8.0.3xx] Update dependencies from dotnet/arcade (#7561) [release/8.0.3xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/post-build/publish-using-darc.ps1 | 4 ++-- .../templates/job/publish-build-assets.yml | 14 +++++++------- eng/common/templates/post-build/post-build.yml | 16 ++++++++-------- .../templates/variables/pool-providers.yml | 12 ++++++------ global.json | 2 +- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 09738d9c30..19d7f6abf7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - be88b08c41971b52ec11aec05ef31e72185d4a1f + da98edc4c3ea539f109ea320672136ceb32591a7 diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 1e779fec4d..5a3a32ea8d 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -12,7 +12,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 - $darc = Get-Darc + $darc = Get-Darc $optionalParams = [System.Collections.ArrayList]::new() @@ -46,7 +46,7 @@ try { } Write-Host 'done.' -} +} catch { Write-Host $_ Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels." diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index fa5446c093..8ec0151def 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -58,7 +58,7 @@ jobs: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2019.amd64 steps: @@ -71,7 +71,7 @@ jobs: checkDownloadedFiles: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: NuGetAuthenticate@1 - task: PowerShell@2 @@ -86,7 +86,7 @@ jobs: /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -95,7 +95,7 @@ jobs: Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - + - task: PublishBuildArtifacts@1 displayName: Publish ReleaseConfigs Artifact inputs: @@ -121,7 +121,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') + condition: eq(variables['SymbolExclusionFile'], 'true') inputs: PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' PublishLocation: Container @@ -137,7 +137,7 @@ jobs: displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' @@ -148,4 +148,4 @@ jobs: - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - template: /eng/common/templates/steps/publish-logs.yml parameters: - JobLabel: 'Publish_Artifacts_Logs' + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 3f74abf7ce..aba44a25a3 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -39,7 +39,7 @@ parameters: displayName: Enable NuGet validation type: boolean default: true - + - name: publishInstallersAndChecksums displayName: Publish installers and checksums type: boolean @@ -131,8 +131,8 @@ stages: displayName: Validate inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - job: displayName: Signing Validation @@ -221,9 +221,9 @@ stages: displayName: Validate inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) -GHCommit $(Build.SourceVersion) -SourcelinkCliVersion $(SourceLinkCLIVersion) continueOnError: true @@ -258,7 +258,7 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Publishing-Internal demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml @@ -272,7 +272,7 @@ stages: displayName: Publish Using Darc inputs: filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -MaestroToken '$(MaestroApiAccessToken)' diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index 9cc5c550d3..d236f9fdbb 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -1,15 +1,15 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, # otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. -# Motivation: +# Motivation: # Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS # (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing # (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. -# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services -# team needs to move resources around and create new and potentially differently-named pools. Using this template +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template # file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. -# How to use: +# How to use: # This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). # If we find alternate naming conventions in broad usage it can be added to the condition below. # @@ -54,4 +54,4 @@ variables: False, 'NetCore1ESPool-Internal' ) - ] \ No newline at end of file + ] diff --git a/global.json b/global.json index 13ae146aec..412cda8d3e 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24081.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24113.2" } } From a93dc85d8bc08a8a2aebdb86c7a78b5652a8d53e Mon Sep 17 00:00:00 2001 From: MilenaHristova Date: Wed, 21 Feb 2024 12:04:27 +0100 Subject: [PATCH 27/93] add ProducesDotNetReleaseShippingAssets property --- eng/Publishing.props | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/Publishing.props b/eng/Publishing.props index 10bc8c8684..579a1360d9 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -2,5 +2,6 @@ 3 + true From 0f1dbcc2353293e9ec761dc8c96b9de5df799dc1 Mon Sep 17 00:00:00 2001 From: MilenaHristova Date: Wed, 21 Feb 2024 12:04:27 +0100 Subject: [PATCH 28/93] add ProducesDotNetReleaseShippingAssets property --- eng/Publishing.props | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/Publishing.props b/eng/Publishing.props index 10bc8c8684..579a1360d9 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -2,5 +2,6 @@ 3 + true From 142d2b7e8679f4ee6609a803229293fce42c8cd0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 27 Feb 2024 13:55:13 +0000 Subject: [PATCH 29/93] Update dependencies from https://github.com/dotnet/arcade build 20240223.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24113.2 -> To Version 8.0.0-beta.24123.1 --- eng/Version.Details.xml | 4 ++-- eng/common/templates/steps/generate-sbom.yml | 2 +- global.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 19d7f6abf7..986bec74cc 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + 042763a811fd94dc3556253d4c64118dd665216e diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38f..2b21eae427 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -5,7 +5,7 @@ # IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. parameters: - PackageVersion: 7.0.0 + PackageVersion: 8.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' PackageName: '.NET' ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom diff --git a/global.json b/global.json index 412cda8d3e..8a39779fd5 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24113.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24123.1" } } From 9e0064703984e04d7f81e73ad1827ca547d95e70 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 2 Mar 2024 14:03:00 +0000 Subject: [PATCH 30/93] Update dependencies from https://github.com/dotnet/arcade build 20240301.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24123.1 -> To Version 8.0.0-beta.24151.4 --- eng/Version.Details.xml | 4 +- eng/common/templates-official/job/job.yml | 255 ++++++++++++++++ .../templates-official/job/onelocbuild.yml | 112 +++++++ .../job/publish-build-assets.yml | 153 ++++++++++ .../templates-official/job/source-build.yml | 67 ++++ .../job/source-index-stage1.yml | 68 +++++ .../templates-official/jobs/codeql-build.yml | 31 ++ eng/common/templates-official/jobs/jobs.yml | 97 ++++++ .../templates-official/jobs/source-build.yml | 46 +++ .../post-build/common-variables.yml | 22 ++ .../post-build/post-build.yml | 285 ++++++++++++++++++ .../post-build/setup-maestro-vars.yml | 70 +++++ .../post-build/trigger-subscription.yml | 13 + .../steps/add-build-to-channel.yml | 13 + .../templates-official/steps/build-reason.yml | 12 + .../steps/component-governance.yml | 13 + .../steps/execute-codeql.yml | 32 ++ .../templates-official/steps/execute-sdl.yml | 88 ++++++ .../steps/generate-sbom.yml | 48 +++ .../templates-official/steps/publish-logs.yml | 23 ++ .../templates-official/steps/retain-build.yml | 28 ++ .../steps/send-to-helix.yml | 91 ++++++ .../templates-official/steps/source-build.yml | 129 ++++++++ .../variables/pool-providers.yml | 45 +++ .../variables/sdl-variables.yml | 7 + global.json | 2 +- 26 files changed, 1751 insertions(+), 3 deletions(-) create mode 100644 eng/common/templates-official/job/job.yml create mode 100644 eng/common/templates-official/job/onelocbuild.yml create mode 100644 eng/common/templates-official/job/publish-build-assets.yml create mode 100644 eng/common/templates-official/job/source-build.yml create mode 100644 eng/common/templates-official/job/source-index-stage1.yml create mode 100644 eng/common/templates-official/jobs/codeql-build.yml create mode 100644 eng/common/templates-official/jobs/jobs.yml create mode 100644 eng/common/templates-official/jobs/source-build.yml create mode 100644 eng/common/templates-official/post-build/common-variables.yml create mode 100644 eng/common/templates-official/post-build/post-build.yml create mode 100644 eng/common/templates-official/post-build/setup-maestro-vars.yml create mode 100644 eng/common/templates-official/post-build/trigger-subscription.yml create mode 100644 eng/common/templates-official/steps/add-build-to-channel.yml create mode 100644 eng/common/templates-official/steps/build-reason.yml create mode 100644 eng/common/templates-official/steps/component-governance.yml create mode 100644 eng/common/templates-official/steps/execute-codeql.yml create mode 100644 eng/common/templates-official/steps/execute-sdl.yml create mode 100644 eng/common/templates-official/steps/generate-sbom.yml create mode 100644 eng/common/templates-official/steps/publish-logs.yml create mode 100644 eng/common/templates-official/steps/retain-build.yml create mode 100644 eng/common/templates-official/steps/send-to-helix.yml create mode 100644 eng/common/templates-official/steps/source-build.yml create mode 100644 eng/common/templates-official/variables/pool-providers.yml create mode 100644 eng/common/templates-official/variables/sdl-variables.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 986bec74cc..2a1458b94e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 042763a811fd94dc3556253d4c64118dd665216e + cbb61c3a9a42e7c3cce17ee453ff5ecdc7f69282 diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml new file mode 100644 index 0000000000..9e7bebe9af --- /dev/null +++ b/eng/common/templates-official/job/job.yml @@ -0,0 +1,255 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + disableComponentGovernance: '' + componentGovernanceIgnoreDirectories: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + runAsPublic: false +# Sbom related params + enableSbom: true + PackageVersion: 7.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@3 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - template: /eng/common/templates-official/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - publish: artifacts/log + artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: Publish logs + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/templates-official/steps/generate-sbom.yml + parameters: + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} + IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration + artifact: BuildConfiguration + displayName: Publish build retry configuration + continueOnError: true diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml new file mode 100644 index 0000000000..ba9ba49303 --- /dev/null +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -0,0 +1,112 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml new file mode 100644 index 0000000000..ea5104625f --- /dev/null +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -0,0 +1,153 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" + Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/templates-official/steps/publish-logs.yml + parameters: + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml new file mode 100644 index 0000000000..8aba3b44bb --- /dev/null +++ b/eng/common/templates-official/job/source-build.yml @@ -0,0 +1,67 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: 1es-mariner-2-pt + os: linux + + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates-official/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml new file mode 100644 index 0000000000..4b63373917 --- /dev/null +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -0,0 +1,68 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: '' + dependsOn: '' + pool: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET Core SDK 6 + inputs: + packageType: sdk + version: 6.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml new file mode 100644 index 0000000000..b68d3c2f31 --- /dev/null +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -0,0 +1,31 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + +jobs: +- template: /eng/common/templates-official/jobs/jobs.yml + parameters: + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml new file mode 100644 index 0000000000..857a0f8ba4 --- /dev/null +++ b/eng/common/templates-official/jobs/jobs.yml @@ -0,0 +1,97 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates-official/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - template: ../job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/templates-official/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml new file mode 100644 index 0000000000..08e5db9bb1 --- /dev/null +++ b/eng/common/templates-official/jobs/source-build.yml @@ -0,0 +1,46 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates-official/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml new file mode 100644 index 0000000000..c24193acfc --- /dev/null +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -0,0 +1,22 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro-prod.westus2.cloudapp.azure.com" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml new file mode 100644 index 0000000000..5c98fe1c0f --- /dev/null +++ b/eng/common/templates-official/post-build/post-build.yml @@ -0,0 +1,285 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml new file mode 100644 index 0000000000..0c87f149a4 --- /dev/null +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -0,0 +1,70 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + +steps: + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml new file mode 100644 index 0000000000..da669030da --- /dev/null +++ b/eng/common/templates-official/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml new file mode 100644 index 0000000000..f67a210d62 --- /dev/null +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/build-reason.yml b/eng/common/templates-official/steps/build-reason.yml new file mode 100644 index 0000000000..eba58109b5 --- /dev/null +++ b/eng/common/templates-official/steps/build-reason.yml @@ -0,0 +1,12 @@ +# build-reason.yml +# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons +# to include steps (',' separated). +parameters: + conditions: '' + steps: [] + +steps: + - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: + - ${{ parameters.steps }} + - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: + - ${{ parameters.steps }} diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml new file mode 100644 index 0000000000..0ecec47b0c --- /dev/null +++ b/eng/common/templates-official/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-codeql.yml b/eng/common/templates-official/steps/execute-codeql.yml new file mode 100644 index 0000000000..9b4a5ffa30 --- /dev/null +++ b/eng/common/templates-official/steps/execute-codeql.yml @@ -0,0 +1,32 @@ +parameters: + # Language that should be analyzed. Defaults to csharp + language: csharp + # Build Commands + buildCommands: '' + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth + # diagnosis of problems with specific tool configurations. + publishGuardianDirectoryToPipeline: false + # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL + # parameters rather than relying on YAML. It may be better to use a local script, because you can + # reproduce results locally without piecing together a command based on the YAML. + executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + # optional: determines whether to continue the build if the step errors; + sdlContinueOnError: false + +steps: +- template: /eng/common/templates-official/steps/execute-sdl.yml + parameters: + overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} + executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} + overrideParameters: ${{ parameters.overrideParameters }} + additionalParameters: '${{ parameters.additionalParameters }} + -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' + publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml new file mode 100644 index 0000000000..07426fde05 --- /dev/null +++ b/eng/common/templates-official/steps/execute-sdl.yml @@ -0,0 +1,88 @@ +parameters: + overrideGuardianVersion: '' + executeAllSdlToolsScript: '' + overrideParameters: '' + additionalParameters: '' + publishGuardianDirectoryToPipeline: false + sdlContinueOnError: false + condition: '' + +steps: +- task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: GuardianConnect + +- task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian (Overridden) + +- ${{ if eq(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian + +- ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} + displayName: Execute SDL (Overridden) + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianCliLocation $(GuardianCliLocation) + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() + + # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration + # with the "SARIF SAST Scans Tab" Azure DevOps extension + - task: CopyFiles@2 + displayName: Copy SARIF files + inputs: + flattenFolders: true + sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ + contents: '**/*.sarif' + targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs + condition: succeededOrFailed() + + # Use PublishBuildArtifacts because the SARIF extension only checks this case + # see microsoft/sarif-azuredevops-extension#4 + - task: PublishBuildArtifacts@1 + displayName: Publish SARIF files to CodeAnalysisLogs container + inputs: + pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs + artifactName: CodeAnalysisLogs + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml new file mode 100644 index 0000000000..1bf43bf807 --- /dev/null +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -0,0 +1,48 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 8.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + inputs: + targetPath: '${{parameters.manifestDirPath}}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml new file mode 100644 index 0000000000..04012fed18 --- /dev/null +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -0,0 +1,23 @@ +parameters: + StageLabel: '' + JobLabel: '' + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml new file mode 100644 index 0000000000..83d97a26a0 --- /dev/null +++ b/eng/common/templates-official/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml new file mode 100644 index 0000000000..3eb7e2d5f8 --- /dev/null +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -0,0 +1,91 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml new file mode 100644 index 0000000000..829f17c34d --- /dev/null +++ b/eng/common/templates-official/steps/source-build.yml @@ -0,0 +1,129 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml new file mode 100644 index 0000000000..beab7d1bfb --- /dev/null +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -0,0 +1,45 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates-official/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) +# +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022-pt + +variables: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 0000000000..dbdd66d4a4 --- /dev/null +++ b/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/global.json b/global.json index 8a39779fd5..261f7f5d91 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24123.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24151.4" } } From eca6439c6b373e4af01f9f2ed26fcba09350c873 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sun, 3 Mar 2024 13:54:18 +0000 Subject: [PATCH 31/93] Update dependencies from https://github.com/dotnet/arcade build 20240301.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24123.1 -> To Version 8.0.0-beta.24151.4 From 628a50ca7c0420335013c2de0a4f671fbf750fff Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Tue, 5 Mar 2024 10:51:19 -0800 Subject: [PATCH 32/93] Update branding to 8.0.203 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index ebb323cdec..2f96daf326 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.202 + 8.0.203 true release From ff64e81497feef4eb9431bf4830d086c3ad23d46 Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Tue, 5 Mar 2024 14:49:32 -0800 Subject: [PATCH 33/93] Update branding to 8.0.204 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 2f96daf326..46205bc007 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.203 + 8.0.204 true release From cb87fd4032421281685cba11f778cd5fd2da32e8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 7 Mar 2024 14:04:12 +0000 Subject: [PATCH 34/93] Update dependencies from https://github.com/dotnet/arcade build 20240305.2 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24151.4 -> To Version 8.0.0-beta.24155.2 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2a1458b94e..9d0243e7b4 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - cbb61c3a9a42e7c3cce17ee453ff5ecdc7f69282 + fb20e40883718b93d6d60eb0a2446e876ffa3d32 diff --git a/global.json b/global.json index 261f7f5d91..dbf96cc188 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24151.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24155.2" } } From 78c47b449ed5d9c585c6586bc820bbfde40d696b Mon Sep 17 00:00:00 2001 From: Jason Zhai Date: Thu, 7 Mar 2024 23:26:22 -0800 Subject: [PATCH 35/93] [release/8.0.3xx] Add -pr version of pipeline --- azure-pipelines-pr.yml | 214 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 azure-pipelines-pr.yml diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml new file mode 100644 index 0000000000..f8cf65dbfe --- /dev/null +++ b/azure-pipelines-pr.yml @@ -0,0 +1,214 @@ +trigger: + batch: true + branches: + include: + - stabilize + - main + - release/* + - internal/release/* + - feature/* + - legacy/* +pr: + branches: + include: + - stabilize + - main + - release/* + - feature/* + - legacy/* + +variables: + # Cannot use key:value syntax in root defined variables + - name: _TeamName + value: NETDevUX + - name: _PublishUsingPipelines + value: true + - name: Codeql.Enabled + value: true + - name: EnableReleaseOneLocBuild + value: false + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Templating-SDLValidation-Params + + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - group: DotNetBuilds storage account read tokens + - name: _InternalRuntimeDownloadArgs + value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal + /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - name: _InternalRuntimeDownloadArgs + value: '' + - template: /eng/common/templates/variables/pool-providers.yml + +stages: +- stage: build + displayName: Build + jobs: + - ${{ if and( ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + # The localization setup for main branch. Note difference in package ID. Should not be used with release/ branches. + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: + - template: /eng/common/templates/job/onelocbuild.yml + parameters: + MirrorRepo: templating + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-TMPLTNGMAIN' + MirrorBranch: 'main' + JobNameSuffix: '_main' + condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') + - template: /eng/common/templates/jobs/jobs.yml + parameters: + enableMicrobuild: true + enablePublishBuildArtifacts: true + enablePublishTestResults: true + enablePublishBuildAssets: true + enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} + enableSourceBuild: true + enableTelemetry: true + helixRepo: dotnet/templating + jobs: + - job: Windows_NT + timeoutInMinutes: 90 + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + vmImage: 1es-windows-2019-open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64 + variables: + - _InternalBuildArgs: '' + + # Only enable publishing in non-public, non PR scenarios. + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat, symweb-symbol-server-pat + # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT + - group: Publish-Build-Assets + - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + + strategy: + matrix: + Build_Release: + _BuildConfig: Release + # PRs or external builds are not signed. + ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + _SignType: test + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _SignType: real + ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + Build_Debug: + _BuildConfig: Debug + _SignType: test + + steps: + - checkout: self + clean: true + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + # Use utility script to run script command dependent on agent OS. + - script: eng/common/cibuild.cmd + -configuration $(_BuildConfig) + -prepareMachine + -integrationTest + $(_InternalBuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Windows Build / Publish + + - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + - job: OSX + pool: + vmImage: 'macOS-latest' + strategy: + matrix: + debug_configuration: + _BuildConfig: Debug + _SignType: none + release_configuration: + _BuildConfig: Release + _SignType: none + steps: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: eng/common/cibuild.sh + --configuration $(_BuildConfig) + --prepareMachine + --integrationTest + $(_InternalRuntimeDownloadArgs) + name: Build + displayName: Build + + - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + - job: Linux + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + vmImage: ubuntu-latest + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + strategy: + matrix: + debug_configuration: + _BuildConfig: Debug + _SignType: none + release_configuration: + _BuildConfig: Release + _SignType: none + steps: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: eng/common/cibuild.sh + --configuration $(_BuildConfig) + --prepareMachine + --integrationTest + $(_InternalRuntimeDownloadArgs) + name: Build + displayName: Build + condition: succeeded() + +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: eng\common\templates\post-build\post-build.yml + parameters: + publishingInfraVersion: 3 + # Symbol validation isn't being very reliable lately. This should be enabled back + # once this issue is resolved: https://github.com/dotnet/arcade/issues/2871 + enableSymbolValidation: false + # Sourcelink validation isn't passing for Arcade due to some regressions. This should be + # enabled back once this issue is resolved: https://github.com/dotnet/arcade/issues/2912 + enableSourceLinkValidation: false + publishDependsOn: + - Validate + # This is to enable SDL runs part of Post-Build Validation Stage + SDLValidationParameters: + enable: true + continueOnError: false + params: ' -SourceToolsList @("policheck","credscan") + -ArtifactToolsList @("binskim") + -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName $(_TsaRepsitoryName) + -TsaCodebaseName $(_TsaCodebaseName) + -TsaPublish $True' From d41744e4cb84316b2bba4c76eee699592c72be54 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 02:34:15 +0000 Subject: [PATCH 36/93] [release/8.0.3xx] Update dependencies from dotnet/arcade (#7658) [release/8.0.3xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/SetupNugetSources.ps1 | 26 +++++++++++------------ eng/common/templates-official/job/job.yml | 4 ++++ eng/common/templates/job/job.yml | 4 ++++ global.json | 2 +- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 9d0243e7b4..5a5517d8e1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - fb20e40883718b93d6d60eb0a2446e876ffa3d32 + 052a4b9e7a9bdb9744c86c05665f1b46e4d59b15 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6c65e81925..efa2fd72bf 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -48,12 +48,11 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } - - AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $password) { +function AddCredential($creds, $source, $username, $pwd) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -82,17 +81,18 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("key", "ClearTextPassword") $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $Password) + + $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." ForEach ($PackageSource in $maestroPrivateSources) { Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd } } @@ -144,13 +144,13 @@ if ($disabledSources -ne $null) { $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnetVersions = @('5','6','7','8') @@ -159,9 +159,9 @@ foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } } -$doc.Save($filename) +$doc.Save($filename) \ No newline at end of file diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 9e7bebe9af..647e3f92e5 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -15,6 +15,7 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -68,6 +69,9 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index e24ca2f46f..8ec5c4f2d9 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -15,6 +15,7 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -68,6 +69,9 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/global.json b/global.json index dbf96cc188..ff6fd14602 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24155.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24158.4" } } From 26e487ef2d61fe665439ae9c583f05e178e60cf8 Mon Sep 17 00:00:00 2001 From: Jason Zhai Date: Sun, 10 Mar 2024 22:41:38 -0700 Subject: [PATCH 37/93] [release/8.0.2xx] Add -pr version of pipeline --- azure-pipelines-pr.yml | 214 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 azure-pipelines-pr.yml diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml new file mode 100644 index 0000000000..f8cf65dbfe --- /dev/null +++ b/azure-pipelines-pr.yml @@ -0,0 +1,214 @@ +trigger: + batch: true + branches: + include: + - stabilize + - main + - release/* + - internal/release/* + - feature/* + - legacy/* +pr: + branches: + include: + - stabilize + - main + - release/* + - feature/* + - legacy/* + +variables: + # Cannot use key:value syntax in root defined variables + - name: _TeamName + value: NETDevUX + - name: _PublishUsingPipelines + value: true + - name: Codeql.Enabled + value: true + - name: EnableReleaseOneLocBuild + value: false + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Templating-SDLValidation-Params + + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - group: DotNetBuilds storage account read tokens + - name: _InternalRuntimeDownloadArgs + value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal + /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - name: _InternalRuntimeDownloadArgs + value: '' + - template: /eng/common/templates/variables/pool-providers.yml + +stages: +- stage: build + displayName: Build + jobs: + - ${{ if and( ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + # The localization setup for main branch. Note difference in package ID. Should not be used with release/ branches. + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: + - template: /eng/common/templates/job/onelocbuild.yml + parameters: + MirrorRepo: templating + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-TMPLTNGMAIN' + MirrorBranch: 'main' + JobNameSuffix: '_main' + condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') + - template: /eng/common/templates/jobs/jobs.yml + parameters: + enableMicrobuild: true + enablePublishBuildArtifacts: true + enablePublishTestResults: true + enablePublishBuildAssets: true + enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} + enableSourceBuild: true + enableTelemetry: true + helixRepo: dotnet/templating + jobs: + - job: Windows_NT + timeoutInMinutes: 90 + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + vmImage: 1es-windows-2019-open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64 + variables: + - _InternalBuildArgs: '' + + # Only enable publishing in non-public, non PR scenarios. + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat, symweb-symbol-server-pat + # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT + - group: Publish-Build-Assets + - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + + strategy: + matrix: + Build_Release: + _BuildConfig: Release + # PRs or external builds are not signed. + ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + _SignType: test + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + _SignType: real + ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + Build_Debug: + _BuildConfig: Debug + _SignType: test + + steps: + - checkout: self + clean: true + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + # Use utility script to run script command dependent on agent OS. + - script: eng/common/cibuild.cmd + -configuration $(_BuildConfig) + -prepareMachine + -integrationTest + $(_InternalBuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Windows Build / Publish + + - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + - job: OSX + pool: + vmImage: 'macOS-latest' + strategy: + matrix: + debug_configuration: + _BuildConfig: Debug + _SignType: none + release_configuration: + _BuildConfig: Release + _SignType: none + steps: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: eng/common/cibuild.sh + --configuration $(_BuildConfig) + --prepareMachine + --integrationTest + $(_InternalRuntimeDownloadArgs) + name: Build + displayName: Build + + - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: + - job: Linux + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + vmImage: ubuntu-latest + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + strategy: + matrix: + debug_configuration: + _BuildConfig: Debug + _SignType: none + release_configuration: + _BuildConfig: Release + _SignType: none + steps: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: eng/common/cibuild.sh + --configuration $(_BuildConfig) + --prepareMachine + --integrationTest + $(_InternalRuntimeDownloadArgs) + name: Build + displayName: Build + condition: succeeded() + +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: eng\common\templates\post-build\post-build.yml + parameters: + publishingInfraVersion: 3 + # Symbol validation isn't being very reliable lately. This should be enabled back + # once this issue is resolved: https://github.com/dotnet/arcade/issues/2871 + enableSymbolValidation: false + # Sourcelink validation isn't passing for Arcade due to some regressions. This should be + # enabled back once this issue is resolved: https://github.com/dotnet/arcade/issues/2912 + enableSourceLinkValidation: false + publishDependsOn: + - Validate + # This is to enable SDL runs part of Post-Build Validation Stage + SDLValidationParameters: + enable: true + continueOnError: false + params: ' -SourceToolsList @("policheck","credscan") + -ArtifactToolsList @("binskim") + -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName $(_TsaRepsitoryName) + -TsaCodebaseName $(_TsaCodebaseName) + -TsaPublish $True' From ea46d8d5ae64ec9cbbd22f26bf1478498cd1cead Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 12 Mar 2024 12:59:02 +0000 Subject: [PATCH 38/93] Update dependencies from https://github.com/dotnet/arcade build 20240311.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24158.4 -> To Version 8.0.0-beta.24161.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 5a5517d8e1..33239a0c23 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 052a4b9e7a9bdb9744c86c05665f1b46e4d59b15 + 5c3fdd3b5aaaa32b24383ec12a60b37ebff13079 diff --git a/global.json b/global.json index ff6fd14602..9567821a80 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24158.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24161.1" } } From 5a3dcbc8022122914aee42fbe0f5143576b2d848 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:34:08 +0000 Subject: [PATCH 39/93] Update dependencies from https://github.com/dotnet/arcade build (#7703) [release/8.0.3xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/templates-official/job/job.yml | 18 +++++++++++------- global.json | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 33239a0c23..7f7c979de1 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 5c3fdd3b5aaaa32b24383ec12a60b37ebff13079 + cd10e5d3748676d70ae2b4d9c84f073eeb203cac diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 647e3f92e5..a2709d1056 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -206,9 +206,11 @@ jobs: continueOnError: true condition: always() - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - publish: artifacts/log - artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - displayName: Publish logs + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: 'artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: 'Publish logs' continueOnError: true condition: always() @@ -253,7 +255,9 @@ jobs: IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration - artifact: BuildConfiguration - displayName: Publish build retry configuration - continueOnError: true + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true \ No newline at end of file diff --git a/global.json b/global.json index 9567821a80..9a169ff4ac 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24161.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24161.7" } } From 7dce0757254da6676d84de85ec189c7cc08cce6b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 16 Mar 2024 12:31:31 +0000 Subject: [PATCH 40/93] Update dependencies from https://github.com/dotnet/arcade build Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24161.7 -> To Version 8.0.0-beta.24165.4 --- eng/Version.Details.xml | 4 ++-- .../templates-official/job/publish-build-assets.yml | 10 ++++++---- global.json | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7f7c979de1..8a9769f47d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - cd10e5d3748676d70ae2b4d9c84f073eeb203cac + f311667e0587f19c3fa9553a909975662107a351 diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index ea5104625f..53138622fe 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -94,14 +94,16 @@ jobs: inputs: targetType: inline script: | - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force + $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + Add-Content -Path $filePath -Value $(BARBuildId) + Add-Content -Path $filePath -Value "$(DefaultChannels)" + Add-Content -Path $filePath -Value $(IsStableBuild) - task: 1ES.PublishBuildArtifacts@1 displayName: Publish ReleaseConfigs Artifact inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs' PublishLocation: Container ArtifactName: ReleaseConfigs diff --git a/global.json b/global.json index 9a169ff4ac..d719bb08ca 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24161.7" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24165.4" } } From 344a9d4a936420fb0b323a9cd98c7e88990170be Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sun, 17 Mar 2024 12:24:13 +0000 Subject: [PATCH 41/93] Update dependencies from https://github.com/dotnet/arcade build Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24161.7 -> To Version 8.0.0-beta.24165.4 From 88ca31ebf23085e3dfaeceb6ca9939f8a8a6da3b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 19 Mar 2024 23:01:21 +0000 Subject: [PATCH 42/93] Update dependencies from https://github.com/dotnet/arcade build Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24113.2 -> To Version 8.0.0-beta.24165.4 --- eng/Version.Details.xml | 4 +- eng/common/SetupNugetSources.ps1 | 26 +- eng/common/templates-official/job/job.yml | 263 ++++++++++++++++ .../templates-official/job/onelocbuild.yml | 112 +++++++ .../job/publish-build-assets.yml | 155 ++++++++++ .../templates-official/job/source-build.yml | 67 ++++ .../job/source-index-stage1.yml | 68 +++++ .../templates-official/jobs/codeql-build.yml | 31 ++ eng/common/templates-official/jobs/jobs.yml | 97 ++++++ .../templates-official/jobs/source-build.yml | 46 +++ .../post-build/common-variables.yml | 22 ++ .../post-build/post-build.yml | 285 ++++++++++++++++++ .../post-build/setup-maestro-vars.yml | 70 +++++ .../post-build/trigger-subscription.yml | 13 + .../steps/add-build-to-channel.yml | 13 + .../templates-official/steps/build-reason.yml | 12 + .../steps/component-governance.yml | 13 + .../steps/execute-codeql.yml | 32 ++ .../templates-official/steps/execute-sdl.yml | 88 ++++++ .../steps/generate-sbom.yml | 48 +++ .../templates-official/steps/publish-logs.yml | 23 ++ .../templates-official/steps/retain-build.yml | 28 ++ .../steps/send-to-helix.yml | 91 ++++++ .../templates-official/steps/source-build.yml | 129 ++++++++ .../variables/pool-providers.yml | 45 +++ .../variables/sdl-variables.yml | 7 + eng/common/templates/job/job.yml | 4 + eng/common/templates/steps/generate-sbom.yml | 2 +- global.json | 2 +- 29 files changed, 1779 insertions(+), 17 deletions(-) create mode 100644 eng/common/templates-official/job/job.yml create mode 100644 eng/common/templates-official/job/onelocbuild.yml create mode 100644 eng/common/templates-official/job/publish-build-assets.yml create mode 100644 eng/common/templates-official/job/source-build.yml create mode 100644 eng/common/templates-official/job/source-index-stage1.yml create mode 100644 eng/common/templates-official/jobs/codeql-build.yml create mode 100644 eng/common/templates-official/jobs/jobs.yml create mode 100644 eng/common/templates-official/jobs/source-build.yml create mode 100644 eng/common/templates-official/post-build/common-variables.yml create mode 100644 eng/common/templates-official/post-build/post-build.yml create mode 100644 eng/common/templates-official/post-build/setup-maestro-vars.yml create mode 100644 eng/common/templates-official/post-build/trigger-subscription.yml create mode 100644 eng/common/templates-official/steps/add-build-to-channel.yml create mode 100644 eng/common/templates-official/steps/build-reason.yml create mode 100644 eng/common/templates-official/steps/component-governance.yml create mode 100644 eng/common/templates-official/steps/execute-codeql.yml create mode 100644 eng/common/templates-official/steps/execute-sdl.yml create mode 100644 eng/common/templates-official/steps/generate-sbom.yml create mode 100644 eng/common/templates-official/steps/publish-logs.yml create mode 100644 eng/common/templates-official/steps/retain-build.yml create mode 100644 eng/common/templates-official/steps/send-to-helix.yml create mode 100644 eng/common/templates-official/steps/source-build.yml create mode 100644 eng/common/templates-official/variables/pool-providers.yml create mode 100644 eng/common/templates-official/variables/sdl-variables.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 19d7f6abf7..8a9769f47d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - da98edc4c3ea539f109ea320672136ceb32591a7 + f311667e0587f19c3fa9553a909975662107a351 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6c65e81925..efa2fd72bf 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -48,12 +48,11 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } - - AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $password) { +function AddCredential($creds, $source, $username, $pwd) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -82,17 +81,18 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("key", "ClearTextPassword") $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $Password) + + $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." ForEach ($PackageSource in $maestroPrivateSources) { Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd } } @@ -144,13 +144,13 @@ if ($disabledSources -ne $null) { $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnetVersions = @('5','6','7','8') @@ -159,9 +159,9 @@ foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } } -$doc.Save($filename) +$doc.Save($filename) \ No newline at end of file diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml new file mode 100644 index 0000000000..a2709d1056 --- /dev/null +++ b/eng/common/templates-official/job/job.yml @@ -0,0 +1,263 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + templateContext: '' + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + disableComponentGovernance: '' + componentGovernanceIgnoreDirectories: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + runAsPublic: false +# Sbom related params + enableSbom: true + PackageVersion: 7.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@3 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - template: /eng/common/templates-official/steps/component-governance.yml + parameters: + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: 'artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/templates-official/steps/generate-sbom.yml + parameters: + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} + IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - task: 1ES.PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true \ No newline at end of file diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml new file mode 100644 index 0000000000..ba9ba49303 --- /dev/null +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -0,0 +1,112 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml new file mode 100644 index 0000000000..53138622fe --- /dev/null +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -0,0 +1,155 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force + $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + Add-Content -Path $filePath -Value $(BARBuildId) + Add-Content -Path $filePath -Value "$(DefaultChannels)" + Add-Content -Path $filePath -Value $(IsStableBuild) + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish ReleaseConfigs Artifact + inputs: + PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - task: 1ES.PublishBuildArtifacts@1 + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + PublishLocation: Container + ArtifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/templates-official/steps/publish-logs.yml + parameters: + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml new file mode 100644 index 0000000000..8aba3b44bb --- /dev/null +++ b/eng/common/templates-official/job/source-build.yml @@ -0,0 +1,67 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: 1es-mariner-2-pt + os: linux + + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates-official/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml new file mode 100644 index 0000000000..4b63373917 --- /dev/null +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -0,0 +1,68 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: '' + dependsOn: '' + pool: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + - template: /eng/common/templates-official/variables/pool-providers.yml + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET Core SDK 6 + inputs: + packageType: sdk + version: 6.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml new file mode 100644 index 0000000000..b68d3c2f31 --- /dev/null +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -0,0 +1,31 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + +jobs: +- template: /eng/common/templates-official/jobs/jobs.yml + parameters: + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml new file mode 100644 index 0000000000..857a0f8ba4 --- /dev/null +++ b/eng/common/templates-official/jobs/jobs.yml @@ -0,0 +1,97 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates-official/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - template: ../job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/templates-official/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml new file mode 100644 index 0000000000..08e5db9bb1 --- /dev/null +++ b/eng/common/templates-official/jobs/source-build.yml @@ -0,0 +1,46 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates-official/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/templates-official/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml new file mode 100644 index 0000000000..c24193acfc --- /dev/null +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -0,0 +1,22 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro-prod.westus2.cloudapp.azure.com" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml new file mode 100644 index 0000000000..5c98fe1c0f --- /dev/null +++ b/eng/common/templates-official/post-build/post-build.yml @@ -0,0 +1,285 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: common-variables.yml + - template: /eng/common/templates-official/variables/pool-providers.yml + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + steps: + - template: setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml new file mode 100644 index 0000000000..0c87f149a4 --- /dev/null +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -0,0 +1,70 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + +steps: + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml new file mode 100644 index 0000000000..da669030da --- /dev/null +++ b/eng/common/templates-official/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml new file mode 100644 index 0000000000..f67a210d62 --- /dev/null +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/build-reason.yml b/eng/common/templates-official/steps/build-reason.yml new file mode 100644 index 0000000000..eba58109b5 --- /dev/null +++ b/eng/common/templates-official/steps/build-reason.yml @@ -0,0 +1,12 @@ +# build-reason.yml +# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons +# to include steps (',' separated). +parameters: + conditions: '' + steps: [] + +steps: + - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: + - ${{ parameters.steps }} + - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: + - ${{ parameters.steps }} diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml new file mode 100644 index 0000000000..0ecec47b0c --- /dev/null +++ b/eng/common/templates-official/steps/component-governance.yml @@ -0,0 +1,13 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-codeql.yml b/eng/common/templates-official/steps/execute-codeql.yml new file mode 100644 index 0000000000..9b4a5ffa30 --- /dev/null +++ b/eng/common/templates-official/steps/execute-codeql.yml @@ -0,0 +1,32 @@ +parameters: + # Language that should be analyzed. Defaults to csharp + language: csharp + # Build Commands + buildCommands: '' + overrideParameters: '' # Optional: to override values for parameters. + additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth + # diagnosis of problems with specific tool configurations. + publishGuardianDirectoryToPipeline: false + # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL + # parameters rather than relying on YAML. It may be better to use a local script, because you can + # reproduce results locally without piecing together a command based on the YAML. + executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + # optional: determines whether to continue the build if the step errors; + sdlContinueOnError: false + +steps: +- template: /eng/common/templates-official/steps/execute-sdl.yml + parameters: + overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} + executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} + overrideParameters: ${{ parameters.overrideParameters }} + additionalParameters: '${{ parameters.additionalParameters }} + -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' + publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml new file mode 100644 index 0000000000..07426fde05 --- /dev/null +++ b/eng/common/templates-official/steps/execute-sdl.yml @@ -0,0 +1,88 @@ +parameters: + overrideGuardianVersion: '' + executeAllSdlToolsScript: '' + overrideParameters: '' + additionalParameters: '' + publishGuardianDirectoryToPipeline: false + sdlContinueOnError: false + condition: '' + +steps: +- task: NuGetAuthenticate@1 + inputs: + nuGetServiceConnections: GuardianConnect + +- task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian (Overridden) + +- ${{ if eq(parameters.overrideGuardianVersion, '') }}: + - pwsh: | + Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl + . .\sdl.ps1 + $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts + Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" + displayName: Install Guardian + +- ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} + displayName: Execute SDL (Overridden) + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianCliLocation $(GuardianCliLocation) + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() + + # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration + # with the "SARIF SAST Scans Tab" Azure DevOps extension + - task: CopyFiles@2 + displayName: Copy SARIF files + inputs: + flattenFolders: true + sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ + contents: '**/*.sarif' + targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs + condition: succeededOrFailed() + + # Use PublishBuildArtifacts because the SARIF extension only checks this case + # see microsoft/sarif-azuredevops-extension#4 + - task: PublishBuildArtifacts@1 + displayName: Publish SARIF files to CodeAnalysisLogs container + inputs: + pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs + artifactName: CodeAnalysisLogs + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml new file mode 100644 index 0000000000..1bf43bf807 --- /dev/null +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -0,0 +1,48 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 8.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + inputs: + targetPath: '${{parameters.manifestDirPath}}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml new file mode 100644 index 0000000000..04012fed18 --- /dev/null +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -0,0 +1,23 @@ +parameters: + StageLabel: '' + JobLabel: '' + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: 1ES.PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml new file mode 100644 index 0000000000..83d97a26a0 --- /dev/null +++ b/eng/common/templates-official/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml new file mode 100644 index 0000000000..3eb7e2d5f8 --- /dev/null +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -0,0 +1,91 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml new file mode 100644 index 0000000000..829f17c34d --- /dev/null +++ b/eng/common/templates-official/steps/source-build.yml @@ -0,0 +1,129 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: 1ES.PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml new file mode 100644 index 0000000000..beab7d1bfb --- /dev/null +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -0,0 +1,45 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: /eng/common/templates-official/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) +# +# pool: +# name: $(DncEngInternalBuildPool) +# image: 1es-windows-2022-pt + +variables: + # Coalesce the target and source branches so we know when a PR targets a release branch + # If these variables are somehow missing, fall back to main (tends to have more capacity) + + # Any new -Svc alternative pools should have variables added here to allow for splitting work + + - name: DncEngInternalBuildPool + value: $[ + replace( + replace( + eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), + True, + 'NetCore1ESPool-Svc-Internal' + ), + False, + 'NetCore1ESPool-Internal' + ) + ] \ No newline at end of file diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml new file mode 100644 index 0000000000..dbdd66d4a4 --- /dev/null +++ b/eng/common/templates-official/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index e24ca2f46f..8ec5c4f2d9 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -15,6 +15,7 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' + templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -68,6 +69,9 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38f..2b21eae427 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -5,7 +5,7 @@ # IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. parameters: - PackageVersion: 7.0.0 + PackageVersion: 8.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' PackageName: '.NET' ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom diff --git a/global.json b/global.json index 412cda8d3e..d719bb08ca 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24113.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24165.4" } } From b3c1f1c59bbac7cd3b66678c6586bbef9dd36935 Mon Sep 17 00:00:00 2001 From: Jacques Eloff Date: Wed, 20 Mar 2024 10:51:46 -0700 Subject: [PATCH 43/93] Merge pull request #7734 from joeloff/1es Migrate to 1ES templates --- azure-pipelines-codeql.yml | 48 ----- azure-pipelines.yml | 349 +++++++++++++++++++------------------ 2 files changed, 179 insertions(+), 218 deletions(-) delete mode 100644 azure-pipelines-codeql.yml diff --git a/azure-pipelines-codeql.yml b/azure-pipelines-codeql.yml deleted file mode 100644 index 94dc81ba8c..0000000000 --- a/azure-pipelines-codeql.yml +++ /dev/null @@ -1,48 +0,0 @@ -variables: - - name: _TeamName - value: DotNetCore - - group: SDL_Settings - -trigger: none - -schedules: - - cron: 17 12 * * 0 - displayName: Weekly Sunday at 12:17 CodeQL/Semmle run - branches: - include: - - main - always: true - -stages: -- stage: build - displayName: Build - # Three phases for each of the three OSes we want to run on - jobs: - - template: /eng/common/templates/jobs/codeql-build.yml - parameters: - jobs: - - job: Windows_NT_CSharp - timeoutInMinutes: 90 - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - checkout: self - clean: true - - - template: /eng/common/templates/steps/execute-codeql.yml - parameters: - executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' - buildCommands: '.\build.cmd -configuration Release -prepareMachine /p:Test=false /p:Sign=false' - language: csharp - additionalParameters: '-SourceToolsList @("semmle") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "Arcade" - -TsaCodebaseName "Arcade" - -TsaPublish $False' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f8cf65dbfe..5afa5ecf75 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,7 +29,6 @@ variables: value: false - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: Templating-SDLValidation-Params - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs @@ -38,177 +37,187 @@ variables: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: _InternalRuntimeDownloadArgs value: '' - - template: /eng/common/templates/variables/pool-providers.yml - -stages: -- stage: build - displayName: Build - jobs: - - ${{ if and( ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - # The localization setup for main branch. Note difference in package ID. Should not be used with release/ branches. - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: - - template: /eng/common/templates/job/onelocbuild.yml - parameters: - MirrorRepo: templating - LclSource: lclFilesfromPackage - LclPackageId: 'LCL-JUNO-PROD-TMPLTNGMAIN' - MirrorBranch: 'main' - JobNameSuffix: '_main' - condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') - - template: /eng/common/templates/jobs/jobs.yml - parameters: - enableMicrobuild: true - enablePublishBuildArtifacts: true - enablePublishTestResults: true - enablePublishBuildAssets: true - enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} - enableSourceBuild: true - enableTelemetry: true - helixRepo: dotnet/templating - jobs: - - job: Windows_NT - timeoutInMinutes: 90 - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - vmImage: 1es-windows-2019-open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - variables: - - _InternalBuildArgs: '' - - # Only enable publishing in non-public, non PR scenarios. - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat, symweb-symbol-server-pat - # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT - - group: Publish-Build-Assets - - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) - /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + - template: /eng/common/templates-official/variables/pool-providers.yml - strategy: - matrix: - Build_Release: - _BuildConfig: Release - # PRs or external builds are not signed. - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - _SignType: test - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _SignType: real - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - Build_Debug: - _BuildConfig: Debug - _SignType: test +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release - steps: - - checkout: self - clean: true - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - # Use utility script to run script command dependent on agent OS. - - script: eng/common/cibuild.cmd - -configuration $(_BuildConfig) - -prepareMachine - -integrationTest - $(_InternalBuildArgs) - $(_InternalRuntimeDownloadArgs) - displayName: Windows Build / Publish +# This is necessary because 1ES doesn't work using strategy & matrix +parameters: +- name: buildConfigurations + type: object + default: + # Always build the Release configuration, but never sign for PRs. + - buildConfig: Release + - buildConfig: Debug - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - job: OSX - pool: - vmImage: 'macOS-latest' - strategy: - matrix: - debug_configuration: - _BuildConfig: Debug - _SignType: none - release_configuration: - _BuildConfig: Release - _SignType: none - steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - script: eng/common/cibuild.sh - --configuration $(_BuildConfig) - --prepareMachine - --integrationTest - $(_InternalRuntimeDownloadArgs) - name: Build - displayName: Build +extends: + ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1esPipelines + parameters: + sdl: + sourceAnalysisPool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: build + displayName: Build + jobs: + - ${{ if and( ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + # The localization setup for main branch. Note difference in package ID. Should not be used with release/ branches. + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: + - template: /eng/common/templates-official/job/onelocbuild.yml@self + parameters: + MirrorRepo: templating + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-TMPLTNGMAIN' + MirrorBranch: 'main' + JobNameSuffix: '_main' + condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') + - template: /eng/common/templates-official/jobs/jobs.yml@self + parameters: + enableMicrobuild: true + enablePublishBuildArtifacts: true + enablePublishTestResults: true + enablePublishBuildAssets: true + enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} + enableSourceBuild: true + enableTelemetry: true + helixRepo: dotnet/templating + jobs: + - ${{ each config in parameters.buildConfigurations }}: + - job: Windows_NT_${{ config.buildConfig }} + displayName: Windows_NT ${{ config.buildConfig }} + timeoutInMinutes: 90 + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64 + variables: + - _BuildConfig: ${{ config.buildConfig }} + - _SignType: test + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - _SignType: real + - _InternalBuildArgs: '' + # Only enable publishing in non-public, non PR scenarios. + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat, symweb-symbol-server-pat + # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT + - group: Publish-Build-Assets + - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - job: Linux - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - vmImage: ubuntu-latest - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 - strategy: - matrix: - debug_configuration: - _BuildConfig: Debug - _SignType: none - release_configuration: - _BuildConfig: Release - _SignType: none - steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - script: eng/common/cibuild.sh - --configuration $(_BuildConfig) - --prepareMachine - --integrationTest - $(_InternalRuntimeDownloadArgs) - name: Build - displayName: Build - condition: succeeded() + steps: + - checkout: self + clean: true + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + # Use utility script to run script command dependent on agent OS. + - script: eng/common/cibuild.cmd + -configuration $(_BuildConfig) + -prepareMachine + -integrationTest + $(_InternalBuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Windows Build / Publish -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: eng\common\templates\post-build\post-build.yml - parameters: - publishingInfraVersion: 3 - # Symbol validation isn't being very reliable lately. This should be enabled back - # once this issue is resolved: https://github.com/dotnet/arcade/issues/2871 - enableSymbolValidation: false - # Sourcelink validation isn't passing for Arcade due to some regressions. This should be - # enabled back once this issue is resolved: https://github.com/dotnet/arcade/issues/2912 - enableSourceLinkValidation: false - publishDependsOn: - - Validate - # This is to enable SDL runs part of Post-Build Validation Stage - SDLValidationParameters: - enable: true - continueOnError: false - params: ' -SourceToolsList @("policheck","credscan") - -ArtifactToolsList @("binskim") - -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName $(_TsaRepsitoryName) - -TsaCodebaseName $(_TsaCodebaseName) - -TsaPublish $True' + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - ${{ each config in parameters.buildConfigurations }}: + - job: OSX_${{ config.buildConfig }} + displayName: OSX ${{ config.buildConfig }} + pool: + vmImage: 'macOS-latest' + variables: + - _BuildConfig: ${{ config.buildConfig }} + - _SignType: none + steps: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: eng/common/cibuild.sh + --configuration $(_BuildConfig) + --prepareMachine + --integrationTest + $(_InternalRuntimeDownloadArgs) + name: Build + displayName: Build + + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - ${{ each config in parameters.buildConfigurations }}: + - job: Linux_${{ config.buildConfig }} + displayName: Linux ${{ config.buildConfig }} + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + vmImage: ubuntu-latest + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + variables: + - _BuildConfig: ${{ config.buildConfig }} + - _SignType: none + steps: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: eng/common/cibuild.sh + --configuration $(_BuildConfig) + --prepareMachine + --integrationTest + $(_InternalRuntimeDownloadArgs) + name: Build + displayName: Build + condition: succeeded() + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: eng/common/templates-official/post-build/post-build.yml@self + parameters: + publishingInfraVersion: 3 + # Symbol validation isn't being very reliable lately. This should be enabled back + # once this issue is resolved: https://github.com/dotnet/arcade/issues/2871 + enableSymbolValidation: false + # Sourcelink validation isn't passing for Arcade due to some regressions. This should be + # enabled back once this issue is resolved: https://github.com/dotnet/arcade/issues/2912 + enableSourceLinkValidation: false + publishDependsOn: + - Validate + # This is to enable SDL runs part of Post-Build Validation Stage + SDLValidationParameters: + enable: true + continueOnError: false + params: ' -SourceToolsList @("policheck","credscan") + -ArtifactToolsList @("binskim") + -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName $(_TsaRepsitoryName) + -TsaCodebaseName $(_TsaCodebaseName) + -TsaPublish $True' From f69286eb936ae0ec766a877cc72123ecf930347d Mon Sep 17 00:00:00 2001 From: Jacques Eloff Date: Thu, 21 Mar 2024 15:36:04 -0700 Subject: [PATCH 44/93] Update Linux pools for 1ES --- azure-pipelines.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5afa5ecf75..4f029ee5c6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -168,10 +168,13 @@ extends: displayName: Linux ${{ config.buildConfig }} pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - vmImage: ubuntu-latest + name: $(DncEngPublicBuildPool) + image: 1es-ubuntu-2204-open + os: linux ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + image: 1es-ubuntu-2204 + os: linux variables: - _BuildConfig: ${{ config.buildConfig }} - _SignType: none From ed421b531260b9ff7e90c5ca92878285f88a79b3 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 22 Mar 2024 13:01:02 +0000 Subject: [PATCH 45/93] Update dependencies from https://github.com/dotnet/arcade build Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24165.4 -> To Version 8.0.0-beta.24170.6 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8a9769f47d..a6202f9c81 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - f311667e0587f19c3fa9553a909975662107a351 + 8e3e00a76f467cc262dc14f6466ab884b2c4eb96 diff --git a/global.json b/global.json index d719bb08ca..63ede0adee 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24165.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24170.6" } } From 7c5e3eda30402898889decd48ad66420fe348dc2 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 06:01:17 +0000 Subject: [PATCH 46/93] Update dependencies from https://github.com/dotnet/arcade build (#7767) [release/8.0.3xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/templates-official/job/job.yml | 2 +- global.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a6202f9c81..34a7e77ce3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 8e3e00a76f467cc262dc14f6466ab884b2c4eb96 + ceb071c1060b8e6de404c065b4045442570caa18 diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index a2709d1056..0604277a2f 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -128,7 +128,7 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@3 + - task: MicroBuildSigningPlugin@4 displayName: Install MicroBuild plugin inputs: signType: $(_SignType) diff --git a/global.json b/global.json index 63ede0adee..372f36d934 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24170.6" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24172.5" } } From eaef194ef84f0bae7a49ac3b6d96505086130cea Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 02:24:32 +0000 Subject: [PATCH 47/93] Update dependencies from https://github.com/dotnet/arcade build 20240326.8 (#7773) [release/8.0.3xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/templates-official/job/job.yml | 1 + global.json | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 34a7e77ce3..8a09bafe5d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - ceb071c1060b8e6de404c065b4045442570caa18 + 48e9e0d2164de0535446809364724da8962123a6 diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 0604277a2f..1f035fee73 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -136,6 +136,7 @@ jobs: feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json env: TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) diff --git a/global.json b/global.json index 372f36d934..35e386f79e 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24172.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24176.8" } } From c9267dba08f5a83782f9e64e5d1618d972efae1f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 29 Mar 2024 02:27:29 +0000 Subject: [PATCH 48/93] Update dependencies from https://github.com/dotnet/arcade build 20240327.1 (#7776) [release/8.0.3xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/native/init-compiler.sh | 2 +- global.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8a09bafe5d..9485b3e187 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 48e9e0d2164de0535446809364724da8962123a6 + b17b2a1bb7da23253043dee059f374b00f3e321a diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index f5c1ec7eaf..2d5660642b 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then # Set default versions if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions="17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" + if [ "$compiler" = "clang" ]; then versions="18 17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do diff --git a/global.json b/global.json index 35e386f79e..1a5335e805 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24176.8" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24177.1" } } From c7d00de97ec2e8acba1397bc70f2309e7efe80a4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 30 Mar 2024 13:07:41 +0000 Subject: [PATCH 49/93] Update dependencies from https://github.com/dotnet/arcade build 20240329.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24177.1 -> To Version 8.0.0-beta.24179.4 --- eng/Version.Details.xml | 4 ++-- eng/common/templates-official/job/onelocbuild.yml | 2 +- .../templates-official/job/publish-build-assets.yml | 4 ++-- eng/common/templates-official/job/source-build.yml | 2 +- .../templates-official/job/source-index-stage1.yml | 2 +- .../templates-official/post-build/post-build.yml | 10 +++++----- .../templates-official/variables/pool-providers.yml | 2 +- global.json | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 9485b3e187..a7cf334b98 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - b17b2a1bb7da23253043dee059f374b00f3e321a + fc2b7849b25c4a21457feb6da5fc7c9806a80976 diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml index ba9ba49303..52b4d05d3f 100644 --- a/eng/common/templates-official/job/onelocbuild.yml +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -56,7 +56,7 @@ jobs: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 53138622fe..589ac80a18 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -60,8 +60,8 @@ jobs: os: windows # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 os: windows steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index 8aba3b44bb..f193dfbe23 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -52,7 +52,7 @@ jobs: ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-mariner-2-pt + image: 1es-mariner-2 os: linux ${{ if ne(parameters.platform.pool, '') }}: diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index 4b63373917..f0513aee5b 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -33,7 +33,7 @@ jobs: demands: ImageOverride -equals windows.vs2019.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: windows.vs2022.amd64 os: windows steps: diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index 5c98fe1c0f..da1f40958b 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -110,7 +110,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: @@ -150,7 +150,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: - template: setup-maestro-vars.yml @@ -208,7 +208,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: - template: setup-maestro-vars.yml @@ -261,8 +261,8 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 os: windows steps: - template: setup-maestro-vars.yml diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml index beab7d1bfb..1f308b24ef 100644 --- a/eng/common/templates-official/variables/pool-providers.yml +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022-pt +# image: 1es-windows-2022 variables: # Coalesce the target and source branches so we know when a PR targets a release branch diff --git a/global.json b/global.json index 1a5335e805..140b3870cd 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24177.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24179.4" } } From 37a38a5ad9d0be209cc60a4e0e263afbaba142d4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sun, 31 Mar 2024 12:59:17 +0000 Subject: [PATCH 50/93] Update dependencies from https://github.com/dotnet/arcade build 20240329.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24177.1 -> To Version 8.0.0-beta.24179.4 From 2c7783cc4245cd052aae2282b160d9d4ce1f12dc Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 1 Apr 2024 12:57:36 +0000 Subject: [PATCH 51/93] Update dependencies from https://github.com/dotnet/arcade build 20240329.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24177.1 -> To Version 8.0.0-beta.24179.4 From 1ca6b37d26e9dcb1305d87fdefc201bb465e028a Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Tue, 2 Apr 2024 11:42:50 -0700 Subject: [PATCH 52/93] Update branding to 8.0.205 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 46205bc007..24dd89d745 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.204 + 8.0.205 true release From 53bcd7a5658b2f7942c2aa75f808a8b002abf624 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 21:33:47 +0000 Subject: [PATCH 53/93] Update dependencies from https://github.com/dotnet/arcade build 20240329.4 (#7788) [release/8.0.2xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/native/init-compiler.sh | 2 +- eng/common/templates-official/job/job.yml | 3 ++- eng/common/templates-official/job/onelocbuild.yml | 2 +- .../templates-official/job/publish-build-assets.yml | 4 ++-- eng/common/templates-official/job/source-build.yml | 2 +- .../templates-official/job/source-index-stage1.yml | 2 +- .../templates-official/post-build/post-build.yml | 10 +++++----- .../templates-official/variables/pool-providers.yml | 2 +- global.json | 2 +- 10 files changed, 17 insertions(+), 16 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8a9769f47d..a7cf334b98 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - f311667e0587f19c3fa9553a909975662107a351 + fc2b7849b25c4a21457feb6da5fc7c9806a80976 diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index f5c1ec7eaf..2d5660642b 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then # Set default versions if [ -z "$majorVersion" ]; then # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. - if [ "$compiler" = "clang" ]; then versions="17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" + if [ "$compiler" = "clang" ]; then versions="18 17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5" elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi for version in $versions; do diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index a2709d1056..1f035fee73 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -128,7 +128,7 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@3 + - task: MicroBuildSigningPlugin@4 displayName: Install MicroBuild plugin inputs: signType: $(_SignType) @@ -136,6 +136,7 @@ jobs: feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json env: TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml index ba9ba49303..52b4d05d3f 100644 --- a/eng/common/templates-official/job/onelocbuild.yml +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -56,7 +56,7 @@ jobs: # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 53138622fe..589ac80a18 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -60,8 +60,8 @@ jobs: os: windows # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 os: windows steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index 8aba3b44bb..f193dfbe23 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -52,7 +52,7 @@ jobs: ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-mariner-2-pt + image: 1es-mariner-2 os: linux ${{ if ne(parameters.platform.pool, '') }}: diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index 4b63373917..f0513aee5b 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -33,7 +33,7 @@ jobs: demands: ImageOverride -equals windows.vs2019.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: windows.vs2022.amd64 os: windows steps: diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index 5c98fe1c0f..da1f40958b 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -110,7 +110,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: @@ -150,7 +150,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: - template: setup-maestro-vars.yml @@ -208,7 +208,7 @@ stages: # If it's not devdiv, it's dnceng ${{ else }}: name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + image: 1es-windows-2022 os: windows steps: - template: setup-maestro-vars.yml @@ -261,8 +261,8 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 os: windows steps: - template: setup-maestro-vars.yml diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml index beab7d1bfb..1f308b24ef 100644 --- a/eng/common/templates-official/variables/pool-providers.yml +++ b/eng/common/templates-official/variables/pool-providers.yml @@ -23,7 +23,7 @@ # # pool: # name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022-pt +# image: 1es-windows-2022 variables: # Coalesce the target and source branches so we know when a PR targets a release branch diff --git a/global.json b/global.json index d719bb08ca..140b3870cd 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24165.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24179.4" } } From 939ba04384564183579fb1ffc7c16d04ad6dab74 Mon Sep 17 00:00:00 2001 From: Jacques Eloff Date: Tue, 2 Apr 2024 17:29:38 -0700 Subject: [PATCH 54/93] Update to 1ESPT (#7762) * Merge pull request #7748 from joeloff/1ES-803xx Update to 1ESPT * Update to 1ESPT --- azure-pipelines-codeql.yml | 48 ----- azure-pipelines.yml | 352 +++++++++++++++++++------------------ 2 files changed, 182 insertions(+), 218 deletions(-) delete mode 100644 azure-pipelines-codeql.yml diff --git a/azure-pipelines-codeql.yml b/azure-pipelines-codeql.yml deleted file mode 100644 index 94dc81ba8c..0000000000 --- a/azure-pipelines-codeql.yml +++ /dev/null @@ -1,48 +0,0 @@ -variables: - - name: _TeamName - value: DotNetCore - - group: SDL_Settings - -trigger: none - -schedules: - - cron: 17 12 * * 0 - displayName: Weekly Sunday at 12:17 CodeQL/Semmle run - branches: - include: - - main - always: true - -stages: -- stage: build - displayName: Build - # Three phases for each of the three OSes we want to run on - jobs: - - template: /eng/common/templates/jobs/codeql-build.yml - parameters: - jobs: - - job: Windows_NT_CSharp - timeoutInMinutes: 90 - pool: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - checkout: self - clean: true - - - template: /eng/common/templates/steps/execute-codeql.yml - parameters: - executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' - buildCommands: '.\build.cmd -configuration Release -prepareMachine /p:Test=false /p:Sign=false' - language: csharp - additionalParameters: '-SourceToolsList @("semmle") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "Arcade" - -TsaCodebaseName "Arcade" - -TsaPublish $False' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f8cf65dbfe..ea309271cf 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,7 +29,6 @@ variables: value: false - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: Templating-SDLValidation-Params - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - group: DotNetBuilds storage account read tokens - name: _InternalRuntimeDownloadArgs @@ -38,177 +37,190 @@ variables: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: _InternalRuntimeDownloadArgs value: '' - - template: /eng/common/templates/variables/pool-providers.yml - -stages: -- stage: build - displayName: Build - jobs: - - ${{ if and( ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - # The localization setup for main branch. Note difference in package ID. Should not be used with release/ branches. - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: - - template: /eng/common/templates/job/onelocbuild.yml - parameters: - MirrorRepo: templating - LclSource: lclFilesfromPackage - LclPackageId: 'LCL-JUNO-PROD-TMPLTNGMAIN' - MirrorBranch: 'main' - JobNameSuffix: '_main' - condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') - - template: /eng/common/templates/jobs/jobs.yml - parameters: - enableMicrobuild: true - enablePublishBuildArtifacts: true - enablePublishTestResults: true - enablePublishBuildAssets: true - enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} - enableSourceBuild: true - enableTelemetry: true - helixRepo: dotnet/templating - jobs: - - job: Windows_NT - timeoutInMinutes: 90 - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - vmImage: 1es-windows-2019-open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - variables: - - _InternalBuildArgs: '' - - # Only enable publishing in non-public, non PR scenarios. - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat, symweb-symbol-server-pat - # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT - - group: Publish-Build-Assets - - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) - /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) - /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + - template: /eng/common/templates-official/variables/pool-providers.yml - strategy: - matrix: - Build_Release: - _BuildConfig: Release - # PRs or external builds are not signed. - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - _SignType: test - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - _SignType: real - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - Build_Debug: - _BuildConfig: Debug - _SignType: test +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release - steps: - - checkout: self - clean: true - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - # Use utility script to run script command dependent on agent OS. - - script: eng/common/cibuild.cmd - -configuration $(_BuildConfig) - -prepareMachine - -integrationTest - $(_InternalBuildArgs) - $(_InternalRuntimeDownloadArgs) - displayName: Windows Build / Publish +# This is necessary because 1ES doesn't work using strategy & matrix +parameters: +- name: buildConfigurations + type: object + default: + # Always build the Release configuration, but never sign for PRs. + - buildConfig: Release + - buildConfig: Debug - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - job: OSX - pool: - vmImage: 'macOS-latest' - strategy: - matrix: - debug_configuration: - _BuildConfig: Debug - _SignType: none - release_configuration: - _BuildConfig: Release - _SignType: none - steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - script: eng/common/cibuild.sh - --configuration $(_BuildConfig) - --prepareMachine - --integrationTest - $(_InternalRuntimeDownloadArgs) - name: Build - displayName: Build +extends: + ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + ${{ else }}: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1esPipelines + parameters: + sdl: + sourceAnalysisPool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: build + displayName: Build + jobs: + - ${{ if and( ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + # The localization setup for main branch. Note difference in package ID. Should not be used with release/ branches. + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/main') }}: + - template: /eng/common/templates-official/job/onelocbuild.yml@self + parameters: + MirrorRepo: templating + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-TMPLTNGMAIN' + MirrorBranch: 'main' + JobNameSuffix: '_main' + condition: eq(variables['Build.SourceBranch'], 'refs/heads/main') + - template: /eng/common/templates-official/jobs/jobs.yml@self + parameters: + enableMicrobuild: true + enablePublishBuildArtifacts: true + enablePublishTestResults: true + enablePublishBuildAssets: true + enablePublishUsingPipelines: ${{ variables._PublishUsingPipelines }} + enableSourceBuild: true + enableTelemetry: true + helixRepo: dotnet/templating + jobs: + - ${{ each config in parameters.buildConfigurations }}: + - job: Windows_NT_${{ config.buildConfig }} + displayName: Windows_NT ${{ config.buildConfig }} + timeoutInMinutes: 90 + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-windows-2022 + variables: + - _BuildConfig: ${{ config.buildConfig }} + - _SignType: test + - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: + - _SignType: real + - _InternalBuildArgs: '' + # Only enable publishing in non-public, non PR scenarios. + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat, symweb-symbol-server-pat + # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT + - group: Publish-Build-Assets + - _InternalBuildArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - - job: Linux - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - vmImage: ubuntu-latest - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 - strategy: - matrix: - debug_configuration: - _BuildConfig: Debug - _SignType: none - release_configuration: - _BuildConfig: Release - _SignType: none - steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - script: eng/common/cibuild.sh - --configuration $(_BuildConfig) - --prepareMachine - --integrationTest - $(_InternalRuntimeDownloadArgs) - name: Build - displayName: Build - condition: succeeded() + steps: + - checkout: self + clean: true + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: PowerShell@2 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + # Use utility script to run script command dependent on agent OS. + - script: eng/common/cibuild.cmd + -configuration $(_BuildConfig) + -prepareMachine + -integrationTest + $(_InternalBuildArgs) + $(_InternalRuntimeDownloadArgs) + displayName: Windows Build / Publish -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: eng\common\templates\post-build\post-build.yml - parameters: - publishingInfraVersion: 3 - # Symbol validation isn't being very reliable lately. This should be enabled back - # once this issue is resolved: https://github.com/dotnet/arcade/issues/2871 - enableSymbolValidation: false - # Sourcelink validation isn't passing for Arcade due to some regressions. This should be - # enabled back once this issue is resolved: https://github.com/dotnet/arcade/issues/2912 - enableSourceLinkValidation: false - publishDependsOn: - - Validate - # This is to enable SDL runs part of Post-Build Validation Stage - SDLValidationParameters: - enable: true - continueOnError: false - params: ' -SourceToolsList @("policheck","credscan") - -ArtifactToolsList @("binskim") - -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName $(_TsaRepsitoryName) - -TsaCodebaseName $(_TsaCodebaseName) - -TsaPublish $True' + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - ${{ each config in parameters.buildConfigurations }}: + - job: OSX_${{ config.buildConfig }} + displayName: OSX ${{ config.buildConfig }} + pool: + vmImage: 'macOS-latest' + variables: + - _BuildConfig: ${{ config.buildConfig }} + - _SignType: none + steps: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: eng/common/cibuild.sh + --configuration $(_BuildConfig) + --prepareMachine + --integrationTest + $(_InternalRuntimeDownloadArgs) + name: Build + displayName: Build + + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - ${{ each config in parameters.buildConfigurations }}: + - job: Linux_${{ config.buildConfig }} + displayName: Linux ${{ config.buildConfig }} + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: 1es-ubuntu-2204-open + os: linux + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: 1es-ubuntu-2204 + os: linux + variables: + - _BuildConfig: ${{ config.buildConfig }} + - _SignType: none + steps: + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + arguments: $(Build.SourcesDirectory)/NuGet.config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + - script: eng/common/cibuild.sh + --configuration $(_BuildConfig) + --prepareMachine + --integrationTest + $(_InternalRuntimeDownloadArgs) + name: Build + displayName: Build + condition: succeeded() + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: eng/common/templates-official/post-build/post-build.yml@self + parameters: + publishingInfraVersion: 3 + # Symbol validation isn't being very reliable lately. This should be enabled back + # once this issue is resolved: https://github.com/dotnet/arcade/issues/2871 + enableSymbolValidation: false + # Sourcelink validation isn't passing for Arcade due to some regressions. This should be + # enabled back once this issue is resolved: https://github.com/dotnet/arcade/issues/2912 + enableSourceLinkValidation: false + publishDependsOn: + - Validate + # This is to enable SDL runs part of Post-Build Validation Stage + SDLValidationParameters: + enable: true + continueOnError: false + params: ' -SourceToolsList @("policheck","credscan") + -ArtifactToolsList @("binskim") + -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName $(_TsaRepsitoryName) + -TsaCodebaseName $(_TsaCodebaseName) + -TsaPublish $True' From 50bc2fe18a92d29271a11c063738677253cbd839 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 3 Apr 2024 12:58:11 +0000 Subject: [PATCH 55/93] Update dependencies from https://github.com/dotnet/arcade build 20240329.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24177.1 -> To Version 8.0.0-beta.24179.4 From 11459c9acd8bd8c16f6b26096531ab1e05f73e11 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 4 Apr 2024 12:47:56 +0000 Subject: [PATCH 56/93] Update dependencies from https://github.com/dotnet/arcade build 20240329.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24177.1 -> To Version 8.0.0-beta.24179.4 From daf4042eabbe93467cee6867793ad3efc08daab8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 5 Apr 2024 12:37:48 +0000 Subject: [PATCH 57/93] Update dependencies from https://github.com/dotnet/arcade build 20240329.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24177.1 -> To Version 8.0.0-beta.24179.4 From 4b5e7f63a43565ca3a11dc5101c5b77a054c40f0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 6 Apr 2024 13:08:07 +0000 Subject: [PATCH 58/93] Update dependencies from https://github.com/dotnet/arcade build 20240404.3 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24179.4 -> To Version 8.0.0-beta.24204.3 --- eng/Version.Details.xml | 4 ++-- eng/common/templates-official/steps/component-governance.yml | 2 +- eng/common/templates/steps/component-governance.yml | 2 +- global.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a7cf334b98..620b538aaa 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - fc2b7849b25c4a21457feb6da5fc7c9806a80976 + 188340e12c0a372b1681ad6a5e72c608021efdba diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml index 0ecec47b0c..cbba059670 100644 --- a/eng/common/templates-official/steps/component-governance.yml +++ b/eng/common/templates-official/steps/component-governance.yml @@ -4,7 +4,7 @@ parameters: steps: - ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" displayName: Set skipComponentGovernanceDetection variable - ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - task: ComponentGovernanceComponentDetection@0 diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml index 0ecec47b0c..cbba059670 100644 --- a/eng/common/templates/steps/component-governance.yml +++ b/eng/common/templates/steps/component-governance.yml @@ -4,7 +4,7 @@ parameters: steps: - ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" displayName: Set skipComponentGovernanceDetection variable - ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - task: ComponentGovernanceComponentDetection@0 diff --git a/global.json b/global.json index 140b3870cd..67b7f26bc1 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24179.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24204.3" } } From 0a270fb541ace7b5c2f576a96fd9b79b0e9f7525 Mon Sep 17 00:00:00 2001 From: Noah Gilson Date: Thu, 18 Apr 2024 12:32:51 -0700 Subject: [PATCH 59/93] stabilize branding --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index fba21eb049..46a8520552 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -6,7 +6,7 @@ true 8.0.300 - false + true release true From b508e1d3774d4855d5ed64f5d581fda313c8403c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 26 Apr 2024 12:55:03 +0000 Subject: [PATCH 60/93] Update dependencies from https://github.com/dotnet/arcade build 20240425.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24204.3 -> To Version 8.0.0-beta.24225.1 --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 620b538aaa..eba66e4585 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + 67d23f4ba1813b315e7e33c71d18b63475f5c5f8 diff --git a/global.json b/global.json index 67b7f26bc1..3a40de507b 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24204.3" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24225.1" } } From ea33e8bf498d5923a94c004fa74c05ee9fd2f9fe Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 27 Apr 2024 12:48:42 +0000 Subject: [PATCH 61/93] Update dependencies from https://github.com/dotnet/arcade build 20240425.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24204.3 -> To Version 8.0.0-beta.24225.1 From 12ba77f507859a5c0d171062778f3a0c9ee7eadf Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sun, 28 Apr 2024 12:39:22 +0000 Subject: [PATCH 62/93] Update dependencies from https://github.com/dotnet/arcade build 20240425.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24204.3 -> To Version 8.0.0-beta.24225.1 From 3d7bc251b9a30939689fb6723fcae550e700c908 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 29 Apr 2024 12:38:28 +0000 Subject: [PATCH 63/93] Update dependencies from https://github.com/dotnet/arcade build 20240425.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24204.3 -> To Version 8.0.0-beta.24225.1 From 0b904842c2ccff7a23957f80553590410df721aa Mon Sep 17 00:00:00 2001 From: Michael Yanni Date: Mon, 29 Apr 2024 12:47:37 -0700 Subject: [PATCH 64/93] Remove debug build for internal pipeline. Remove custom build tags since they aren't applicable. --- azure-pipelines.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4f029ee5c6..dac2e9651d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -53,7 +53,6 @@ parameters: default: # Always build the Release configuration, but never sign for PRs. - buildConfig: Release - - buildConfig: Debug extends: ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: @@ -66,8 +65,6 @@ extends: name: $(DncEngInternalBuildPool) image: 1es-windows-2022 os: windows - customBuildTags: - - ES365AIMigrationTooling stages: - stage: build displayName: Build From eae6408ced7cfd85b55e809e878ccceb58cc88c9 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 30 Apr 2024 12:54:29 +0000 Subject: [PATCH 65/93] Update dependencies from https://github.com/dotnet/arcade build 20240425.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24204.3 -> To Version 8.0.0-beta.24225.1 From 1eedc1dbf6dcb6abe529c9bb43c5adddc09e563f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 1 May 2024 12:35:52 +0000 Subject: [PATCH 66/93] Update dependencies from https://github.com/dotnet/arcade build 20240425.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24204.3 -> To Version 8.0.0-beta.24225.1 From 2044ae211681b63923df85e11096ceb75dd9ba49 Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Wed, 1 May 2024 14:38:19 -0700 Subject: [PATCH 67/93] Update branding to 8.0.301 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 46a8520552..ab8efb49b9 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.300 + 8.0.301 true release From d35ed388222e728600dd01a5f5098b60e1e7e173 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 2 May 2024 12:39:19 +0000 Subject: [PATCH 68/93] Update dependencies from https://github.com/dotnet/arcade build 20240425.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24204.3 -> To Version 8.0.0-beta.24225.1 From 538f92d10ab1b80f9377ecdfa0bc369a4b167d9b Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Fri, 3 May 2024 12:58:48 +0000 Subject: [PATCH 69/93] Update dependencies from https://github.com/dotnet/arcade build 20240425.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24204.3 -> To Version 8.0.0-beta.24225.1 From 9a341aa16f756c96d0e15830d1835084ddd48e70 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sat, 4 May 2024 12:39:59 +0000 Subject: [PATCH 70/93] Update dependencies from https://github.com/dotnet/arcade build 20240425.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24204.3 -> To Version 8.0.0-beta.24225.1 From 9aff1d448a640fcfe64b0381465a80e0d015c2fd Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Sun, 5 May 2024 12:39:38 +0000 Subject: [PATCH 71/93] Update dependencies from https://github.com/dotnet/arcade build 20240425.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24204.3 -> To Version 8.0.0-beta.24225.1 From 687291df5285e8994004e805a7588ab8c7dc3afc Mon Sep 17 00:00:00 2001 From: Matt Thalman Date: Tue, 7 May 2024 08:03:48 -0500 Subject: [PATCH 72/93] Update SBRP dependency --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index eba66e4585..4a90f05ca4 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ 3dc05150cf234f76f6936dcb2853d31a0da1f60e - + https://github.com/dotnet/source-build-reference-packages - b4fa7f2e1e65ef49881be2ab2df27624280a8c55 + 6f814daa935e08b578b1c0c65a1f26ea3317f517 From 29d69966453cf3a5d89a9eb952a2a9657d0f38ea Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Thu, 16 May 2024 17:32:43 -0700 Subject: [PATCH 73/93] Update branding to 8.0.302 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index ab8efb49b9..d73f30af06 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.301 + 8.0.302 true release From ca6fa7de73b49931c581a42b0032b55d229bb552 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 14:32:53 -0700 Subject: [PATCH 74/93] [release/8.0.3xx] Update dependencies from dotnet/arcade (#7969) * Update dependencies from https://github.com/dotnet/arcade build 20240514.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24225.1 -> To Version 8.0.0-beta.24264.1 * Update dependencies from https://github.com/dotnet/arcade build 20240516.3 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24225.1 -> To Version 8.0.0-beta.24266.3 * Update dependencies from https://github.com/dotnet/arcade build 20240516.3 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24225.1 -> To Version 8.0.0-beta.24266.3 * Update dependencies from https://github.com/dotnet/arcade build 20240516.3 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24225.1 -> To Version 8.0.0-beta.24266.3 * Update dependencies from https://github.com/dotnet/arcade build 20240516.3 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24225.1 -> To Version 8.0.0-beta.24266.3 * Remove redundant equality * Remove unused parameter --------- Co-authored-by: dotnet-maestro[bot] Co-authored-by: Jason Zhai --- eng/Version.Details.xml | 4 +- .../job/source-index-stage1.yml | 49 +++++++++++++------ .../templates/job/source-index-stage1.yml | 44 ++++++++++++----- global.json | 2 +- .../ConfigModel/ParameterSymbol.cs | 2 +- .../Mocks/MockPackageManager.cs | 2 +- 6 files changed, 69 insertions(+), 34 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4a90f05ca4..f3fda017bb 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 67d23f4ba1813b315e7e33c71d18b63475f5c5f8 + e6f70c7dd528f05cd28cec2a179d58c22e91d9ac diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index f0513aee5b..43ee0c202f 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -14,15 +15,15 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - - template: /eng/common/templates-official/variables/pool-providers.yml + - template: /eng/common/templates/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} @@ -33,24 +34,23 @@ jobs: demands: ImageOverride -equals windows.vs2019.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - image: windows.vs2022.amd64 - os: windows + demands: ImageOverride -equals windows.vs2019.amd64 steps: - ${{ each preStep in parameters.preSteps }}: - ${{ preStep }} - task: UseDotNet@2 - displayName: Use .NET Core SDK 6 + displayName: Use .NET 8 SDK inputs: packageType: sdk - version: 6.0.x + version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -62,7 +62,24 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + + - script: | + echo "Client ID: $(ARM_CLIENT_ID)" + echo "ID Token: $(ARM_ID_TOKEN)" + echo "Tenant ID: $(ARM_TENANT_ID)" + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 + displayName: Upload stage1 artifacts to source index \ No newline at end of file diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b98202aa02..43ee0c202f 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -14,14 +15,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - template: /eng/common/templates/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: @@ -40,16 +41,16 @@ jobs: - ${{ preStep }} - task: UseDotNet@2 - displayName: Use .NET Core SDK 6 + displayName: Use .NET 8 SDK inputs: packageType: sdk - version: 6.0.x + version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -61,7 +62,24 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + + - script: | + echo "Client ID: $(ARM_CLIENT_ID)" + echo "ID Token: $(ARM_ID_TOKEN)" + echo "Tenant ID: $(ARM_TENANT_ID)" + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 + displayName: Upload stage1 artifacts to source index \ No newline at end of file diff --git a/global.json b/global.json index 3a40de507b..63925fe917 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24225.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24266.3" } } diff --git a/src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/ConfigModel/ParameterSymbol.cs b/src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/ConfigModel/ParameterSymbol.cs index ea5b4464d9..3fc52ac290 100644 --- a/src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/ConfigModel/ParameterSymbol.cs +++ b/src/Microsoft.TemplateEngine.Orchestrator.RunnableProjects/ConfigModel/ParameterSymbol.cs @@ -62,7 +62,7 @@ internal ParameterSymbol(string name, JObject jObject, string? defaultOverride) DefaultIfOptionWithoutValue = "true"; } - if (DefaultValue == null && IsRequired != true) + if (DefaultValue == null && !IsRequired) { DefaultValue = ParameterConverter.GetDefault(DataType); } diff --git a/test/Microsoft.TemplateEngine.Edge.UnitTests/Mocks/MockPackageManager.cs b/test/Microsoft.TemplateEngine.Edge.UnitTests/Mocks/MockPackageManager.cs index 1141d92b7c..9f83fe4d11 100644 --- a/test/Microsoft.TemplateEngine.Edge.UnitTests/Mocks/MockPackageManager.cs +++ b/test/Microsoft.TemplateEngine.Edge.UnitTests/Mocks/MockPackageManager.cs @@ -65,7 +65,7 @@ public Task DownloadPackageAsync(string downloadPath, string i Array.Empty())); } - public Task<(string LatestVersion, bool IsLatestVersion, IReadOnlyList Vulnerabilities)> GetLatestVersionAsync(string identifier, string? version = null, string? additionalNuGetSource = null, CancellationToken cancellationToken = default) + public Task<(string LatestVersion, bool IsLatestVersion, IReadOnlyList Vulnerabilities)> GetLatestVersionAsync(string identifier, string? version = null) { // names of exceptions throw them for test purposes return identifier switch From b743fcd503ffba3940fc2472e1d840f3cad1b469 Mon Sep 17 00:00:00 2001 From: Marc Paine Date: Thu, 30 May 2024 16:13:34 -0700 Subject: [PATCH 75/93] Increment branding to allow for razor hotfix --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index d73f30af06..bce83f0a07 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.302 + 8.0.303 true release From 1bddd10408f6e7cabcce51d89d2c40919914423d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 10:15:52 -0700 Subject: [PATCH 76/93] [release/8.0.3xx] Update dependencies from dotnet/arcade (#8017) * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 * Update dependencies from https://github.com/dotnet/arcade build 20240520.4 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24266.3 -> To Version 8.0.0-beta.24270.4 --------- Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f3fda017bb..75b4efdcd8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - e6f70c7dd528f05cd28cec2a179d58c22e91d9ac + f2b2071632d5d4c46d0f904f2b0d917b1752551b diff --git a/global.json b/global.json index 63925fe917..99ecda69a5 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24266.3" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24270.4" } } From 877facad1cb4c451ba23191ee422381fd4584e9d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 11 Jun 2024 12:56:27 +0000 Subject: [PATCH 77/93] Update dependencies from https://github.com/dotnet/arcade build 20240610.5 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24270.4 -> To Version 8.0.0-beta.24310.5 --- eng/Version.Details.xml | 4 ++-- .../job/source-index-stage1.yml | 16 +++++++--------- eng/common/templates/job/source-index-stage1.yml | 11 ++++------- global.json | 2 +- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 75b4efdcd8..2627eb0252 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - f2b2071632d5d4c46d0f904f2b0d917b1752551b + 9f6799fdc16ae19b3e9478c55b997a6aab839d09 diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index 43ee0c202f..60dfb6b2d1 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -23,7 +23,7 @@ jobs: value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - template: /eng/common/templates/variables/pool-providers.yml + - template: /eng/common/templates-official/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} @@ -34,7 +34,8 @@ jobs: demands: ImageOverride -equals windows.vs2019.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 + image: windows.vs2022.amd64 + os: windows steps: - ${{ each preStep in parameters.preSteps }}: @@ -70,16 +71,13 @@ jobs: scriptType: 'ps' scriptLocation: 'inlineScript' inlineScript: | - echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" - echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" - echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" - script: | - echo "Client ID: $(ARM_CLIENT_ID)" - echo "ID Token: $(ARM_ID_TOKEN)" - echo "Tenant ID: $(ARM_TENANT_ID)" az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) displayName: "Login to Azure" - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index \ No newline at end of file + displayName: Upload stage1 artifacts to source index diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 43ee0c202f..0b6bb89dc7 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -70,16 +70,13 @@ jobs: scriptType: 'ps' scriptLocation: 'inlineScript' inlineScript: | - echo "##vso[task.setvariable variable=ARM_CLIENT_ID]$env:servicePrincipalId" - echo "##vso[task.setvariable variable=ARM_ID_TOKEN]$env:idToken" - echo "##vso[task.setvariable variable=ARM_TENANT_ID]$env:tenantId" + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" - script: | - echo "Client ID: $(ARM_CLIENT_ID)" - echo "ID Token: $(ARM_ID_TOKEN)" - echo "Tenant ID: $(ARM_TENANT_ID)" az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) displayName: "Login to Azure" - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 - displayName: Upload stage1 artifacts to source index \ No newline at end of file + displayName: Upload stage1 artifacts to source index diff --git a/global.json b/global.json index 99ecda69a5..d01b556188 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24270.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24310.5" } } From 2c8137bda50484d21e263aaac51dad537ba57cc8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 14 Jun 2024 06:53:43 -0700 Subject: [PATCH 78/93] Update dependencies from https://github.com/dotnet/arcade build 20240611.3 (#8129) Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24310.5 -> To Version 8.0.0-beta.24311.3 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2627eb0252..4544006bae 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 9f6799fdc16ae19b3e9478c55b997a6aab839d09 + c214b6ad17aedca4fa48294d80f6c52ef2463081 diff --git a/global.json b/global.json index d01b556188..14cfa706da 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24310.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24311.3" } } From 04f9763b4c593b8207e790d52fec5ea51c02016c Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Tue, 2 Jul 2024 08:48:17 -0700 Subject: [PATCH 79/93] Update arcade --- eng/Version.Details.xml | 4 +- eng/common/post-build/publish-using-darc.ps1 | 15 +++---- .../job/publish-build-assets.yml | 12 +++--- .../templates-official/job/source-build.yml | 8 ++++ .../templates-official/jobs/source-build.yml | 8 ++++ .../post-build/post-build.yml | 8 ++-- .../steps/enable-internal-runtimes.yml | 28 ++++++++++++ .../steps/get-delegation-sas.yml | 43 +++++++++++++++++++ .../steps/get-federated-access-token.yml | 28 ++++++++++++ .../templates/job/publish-build-assets.yml | 12 +++--- eng/common/templates/job/source-build.yml | 8 ++++ eng/common/templates/jobs/source-build.yml | 8 ++++ .../templates/post-build/post-build.yml | 8 ++-- .../post-build/setup-maestro-vars.yml | 28 ++++++------ .../steps/enable-internal-runtimes.yml | 28 ++++++++++++ .../templates/steps/get-delegation-sas.yml | 43 +++++++++++++++++++ .../steps/get-federated-access-token.yml | 28 ++++++++++++ global.json | 2 +- 18 files changed, 278 insertions(+), 41 deletions(-) create mode 100644 eng/common/templates-official/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates-official/steps/get-delegation-sas.yml create mode 100644 eng/common/templates-official/steps/get-federated-access-token.yml create mode 100644 eng/common/templates/steps/enable-internal-runtimes.yml create mode 100644 eng/common/templates/steps/get-delegation-sas.yml create mode 100644 eng/common/templates/steps/get-federated-access-token.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4544006bae..a03418605a 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - c214b6ad17aedca4fa48294d80f6c52ef2463081 + 8b879da4e449c48d99f3f642fc429379a64e8fe8 diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 5a3a32ea8d..238945cb5a 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -2,7 +2,6 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $PublishingInfraVersion, [Parameter(Mandatory=$true)][string] $AzdoToken, - [Parameter(Mandatory=$true)][string] $MaestroToken, [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, @@ -31,13 +30,13 @@ try { } & $darc add-build-to-channel ` - --id $buildId ` - --publishing-infra-version $PublishingInfraVersion ` - --default-channels ` - --source-branch main ` - --azdev-pat $AzdoToken ` - --bar-uri $MaestroApiEndPoint ` - --password $MaestroToken ` + --id $buildId ` + --publishing-infra-version $PublishingInfraVersion ` + --default-channels ` + --source-branch main ` + --azdev-pat "$AzdoToken" ` + --bar-uri "$MaestroApiEndPoint" ` + --ci ` @optionalParams if ($LastExitCode -ne 0) { diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 589ac80a18..d01739c128 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -76,13 +76,16 @@ jobs: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Build Assets inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: > + -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) @@ -144,7 +147,6 @@ jobs: arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index f193dfbe23..f983033bb0 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -62,6 +68,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml - template: /eng/common/templates-official/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index 08e5db9bb1..5cf6a269c0 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates-official/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index da1f40958b..0dfa387e7b 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -272,14 +272,16 @@ stages: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml new file mode 100644 index 0000000000..93a8394a66 --- /dev/null +++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml new file mode 100644 index 0000000000..c0e8f91317 --- /dev/null +++ b/eng/common/templates-official/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml new file mode 100644 index 0000000000..e3786cef6d --- /dev/null +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 8ec0151def..9fd69fa7c9 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -74,13 +74,16 @@ jobs: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Build Assets inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: > + -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) @@ -140,7 +143,6 @@ jobs: arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 8a3deef2b7..c0ff472b69 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -61,6 +67,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - template: /eng/common/templates/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index a15b07eb51..5f46bfa895 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index aba44a25a3..2db4933468 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -268,14 +268,16 @@ stages: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 0c87f149a4..64b9abc685 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -11,13 +11,14 @@ steps: artifactName: ReleaseConfigs checkDownloadedFiles: true - - task: PowerShell@2 + - task: AzureCLI@2 name: setReleaseVars displayName: Set Release Configs Vars inputs: - targetType: inline - pwsh: true - script: | + azureSubscription: "Darc: Maestro Production" + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | try { if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt @@ -31,15 +32,16 @@ steps: $AzureDevOpsBuildId = $Env:Build_BuildId } else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + . $(Build.SourcesDirectory)\eng\common\tools.ps1 + $darc = Get-Darc + $buildInfo = & $darc get-build ` + --id ${{ parameters.BARBuildId }} ` + --extended ` + --output-format json ` + --ci ` + | convertFrom-Json - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId + $BarId = ${{ parameters.BARBuildId }} $Channels = $Env:PromoteToMaestroChannels -split "," $Channels = $Channels -join "][" $Channels = "[$Channels]" @@ -65,6 +67,4 @@ steps: exit 1 } env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml new file mode 100644 index 0000000000..54dc9416c5 --- /dev/null +++ b/eng/common/templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml new file mode 100644 index 0000000000..c0e8f91317 --- /dev/null +++ b/eng/common/templates/steps/get-delegation-sas.yml @@ -0,0 +1,43 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml new file mode 100644 index 0000000000..c8c49cc0e8 --- /dev/null +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -0,0 +1,28 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" \ No newline at end of file diff --git a/global.json b/global.json index 14cfa706da..6d85580a38 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24311.3" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24352.1" } } From 8eb35917a343bb9c2826dda6b414dc0893e1102b Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Tue, 2 Jul 2024 08:43:04 -0700 Subject: [PATCH 80/93] Remove internal runtime args --- azure-pipelines-pr.yml | 38 +------------------------------------- azure-pipelines.yml | 41 +++-------------------------------------- 2 files changed, 4 insertions(+), 75 deletions(-) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index f8cf65dbfe..f13acc248d 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -29,15 +29,6 @@ variables: value: false - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: Templating-SDLValidation-Params - - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - group: DotNetBuilds storage account read tokens - - name: _InternalRuntimeDownloadArgs - value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal - /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - name: _InternalRuntimeDownloadArgs - value: '' - template: /eng/common/templates/variables/pool-providers.yml stages: @@ -104,21 +95,12 @@ stages: steps: - checkout: self clean: true - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) # Use utility script to run script command dependent on agent OS. - script: eng/common/cibuild.cmd -configuration $(_BuildConfig) -prepareMachine -integrationTest $(_InternalBuildArgs) - $(_InternalRuntimeDownloadArgs) displayName: Windows Build / Publish - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: @@ -134,19 +116,10 @@ stages: _BuildConfig: Release _SignType: none steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - script: eng/common/cibuild.sh --configuration $(_BuildConfig) --prepareMachine --integrationTest - $(_InternalRuntimeDownloadArgs) name: Build displayName: Build @@ -154,7 +127,7 @@ stages: - job: Linux pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - vmImage: ubuntu-latest + vmImage: ubuntu-latest ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 @@ -167,19 +140,10 @@ stages: _BuildConfig: Release _SignType: none steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - script: eng/common/cibuild.sh --configuration $(_BuildConfig) --prepareMachine --integrationTest - $(_InternalRuntimeDownloadArgs) name: Build displayName: Build condition: succeeded() diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1353e298b8..9f93270776 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,14 +29,6 @@ variables: value: false - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: Templating-SDLValidation-Params - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - group: DotNetBuilds storage account read tokens - - name: _InternalRuntimeDownloadArgs - value: /p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal - /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - name: _InternalRuntimeDownloadArgs - value: '' - template: /eng/common/templates-official/variables/pool-providers.yml resources: @@ -104,7 +96,7 @@ extends: - _SignType: test - ${{ if notin(variables['Build.Reason'], 'PullRequest') }}: - _SignType: real - - _InternalBuildArgs: '' + - _InternalBuildArgs: '' # Only enable publishing in non-public, non PR scenarios. - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat, symweb-symbol-server-pat @@ -116,21 +108,12 @@ extends: steps: - checkout: self clean: true - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: PowerShell@2 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) # Use utility script to run script command dependent on agent OS. - script: eng/common/cibuild.cmd -configuration $(_BuildConfig) -prepareMachine -integrationTest $(_InternalBuildArgs) - $(_InternalRuntimeDownloadArgs) displayName: Windows Build / Publish - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: @@ -143,22 +126,13 @@ extends: - _BuildConfig: ${{ config.buildConfig }} - _SignType: none steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - script: eng/common/cibuild.sh --configuration $(_BuildConfig) --prepareMachine --integrationTest - $(_InternalRuntimeDownloadArgs) name: Build displayName: Build - + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - ${{ each config in parameters.buildConfigurations }}: - job: Linux_${{ config.buildConfig }} @@ -176,23 +150,14 @@ extends: - _BuildConfig: ${{ config.buildConfig }} - _SignType: none steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - script: eng/common/cibuild.sh --configuration $(_BuildConfig) --prepareMachine --integrationTest - $(_InternalRuntimeDownloadArgs) name: Build displayName: Build condition: succeeded() - + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - template: eng/common/templates-official/post-build/post-build.yml@self parameters: From b075025585371a933364d3c8189cd8e50ef4c87b Mon Sep 17 00:00:00 2001 From: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> Date: Tue, 2 Jul 2024 13:08:47 -0700 Subject: [PATCH 81/93] Update branding to 8.0.304 (#8177) --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index bce83f0a07..e3df6e7365 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.303 + 8.0.304 true release From 350f32942e8db0ab82b6ecc1c5710de386298e55 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 3 Jul 2024 02:40:17 +0000 Subject: [PATCH 82/93] [release/8.0.3xx] Update dependencies from dotnet/arcade (#8146) [release/8.0.3xx] Update dependencies from dotnet/arcade - Merge branch 'release/8.0.3xx' into darc-release/8.0.3xx-7a6529c7-dfb0-4829-ab5a-832c1749a35f From f9821237ce60404576252e302e59f5b2ff41cee8 Mon Sep 17 00:00:00 2001 From: Farhad Alizada <104755925+f-alizada@users.noreply.github.com> Date: Thu, 20 Jun 2024 17:57:01 +0200 Subject: [PATCH 83/93] Onboard new inter-branch merge flow [Workflow] --- .github/workflows/inter-branch-merge-flow.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/inter-branch-merge-flow.yml diff --git a/.github/workflows/inter-branch-merge-flow.yml b/.github/workflows/inter-branch-merge-flow.yml new file mode 100644 index 0000000000..7cd5b0c7a5 --- /dev/null +++ b/.github/workflows/inter-branch-merge-flow.yml @@ -0,0 +1,13 @@ +name: Inter-branch merge workflow +on: + push: + branches: + - release/** + +permissions: + contents: write + pull-requests: write + +jobs: + Merge: + uses: dotnet/arcade/.github/workflows/inter-branch-merge-base.yml@main From 67812a314587997f9efd0a436ad04a17f11f449d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Wed, 10 Jul 2024 12:55:26 +0000 Subject: [PATCH 84/93] Update dependencies from https://github.com/dotnet/arcade build 20240709.3 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24352.1 -> To Version 8.0.0-beta.24359.3 --- eng/Version.Details.xml | 4 ++-- .../templates-official/job/publish-build-assets.yml | 9 ++++++--- eng/common/templates/job/publish-build-assets.yml | 9 ++++++--- global.json | 2 +- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a03418605a..8eea8f5dca 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 8b879da4e449c48d99f3f642fc429379a64e8fe8 + db87887481d4110c09a1004191002482fdd7e4f2 diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index d01739c128..ba3e7df815 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -140,11 +140,14 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -WaitPublishingFinish true diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 9fd69fa7c9..57a41f0a3e 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -136,11 +136,14 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' -WaitPublishingFinish true diff --git a/global.json b/global.json index 6d85580a38..6879014b6f 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24352.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24359.3" } } From e2ba13670386f562c27ca8e00fa1413030ac218a Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 11 Jul 2024 18:28:48 +0000 Subject: [PATCH 85/93] [release/8.0.3xx] Update dependencies from dotnet/arcade (#8213) [release/8.0.3xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8eea8f5dca..1c1af53162 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - db87887481d4110c09a1004191002482fdd7e4f2 + c9efa535175049eb9cba06cae1f8c3d5dbe768a9 diff --git a/global.json b/global.json index 6879014b6f..6d86fe094b 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24359.3" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24360.5" } } From 3ca7353f6078a4532a3cc7e73d9b6bf79ed5d29f Mon Sep 17 00:00:00 2001 From: vseanreesermsft <78103370+vseanreesermsft@users.noreply.github.com> Date: Tue, 6 Aug 2024 13:30:16 -0700 Subject: [PATCH 86/93] Update branding to 8.0.305 (#8260) --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index e3df6e7365..98cee68935 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.304 + 8.0.305 true release From 19ed9a197d8d29712899dac54b9f4a32418e7133 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 6 Aug 2024 13:32:30 -0700 Subject: [PATCH 87/93] [release/8.0.3xx] Update dependencies from dotnet/arcade (#8233) * Update dependencies from https://github.com/dotnet/arcade build 20240716.7 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24366.7 * Update dependencies from https://github.com/dotnet/arcade build 20240717.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24367.1 * Update dependencies from https://github.com/dotnet/arcade build 20240717.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24367.1 * Update dependencies from https://github.com/dotnet/arcade build 20240717.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24367.1 * Update dependencies from https://github.com/dotnet/arcade build 20240717.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24367.1 * Update dependencies from https://github.com/dotnet/arcade build 20240717.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24367.1 * Update dependencies from https://github.com/dotnet/arcade build 20240722.3 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24372.3 * Update dependencies from https://github.com/dotnet/arcade build 20240722.3 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24372.3 * Update dependencies from https://github.com/dotnet/arcade build 20240726.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24376.1 * Update dependencies from https://github.com/dotnet/arcade build 20240726.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24376.1 * Update dependencies from https://github.com/dotnet/arcade build 20240726.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24376.1 * Update dependencies from https://github.com/dotnet/arcade build 20240726.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24376.1 * Update dependencies from https://github.com/dotnet/arcade build 20240726.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24376.1 * Update dependencies from https://github.com/dotnet/arcade build 20240726.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24376.1 * Update dependencies from https://github.com/dotnet/arcade build 20240726.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24376.1 * Update dependencies from https://github.com/dotnet/arcade build 20240726.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24376.1 * Update dependencies from https://github.com/dotnet/arcade build 20240726.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24376.1 * Update dependencies from https://github.com/dotnet/arcade build 20240726.1 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24360.5 -> To Version 8.0.0-beta.24376.1 --------- Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- eng/common/sdl/NuGet.config | 4 ++-- eng/common/sdl/execute-all-sdl-tools.ps1 | 4 +--- eng/common/sdl/init-sdl.ps1 | 8 -------- eng/common/sdl/sdl.ps1 | 4 +++- .../job/publish-build-assets.yml | 2 +- .../templates-official/post-build/post-build.yml | 2 +- .../templates-official/steps/execute-sdl.yml | 2 -- .../steps/get-federated-access-token.yml | 14 +++++++++++++- eng/common/templates/job/publish-build-assets.yml | 2 +- eng/common/templates/post-build/post-build.yml | 2 +- eng/common/templates/steps/execute-sdl.yml | 7 ++++--- .../templates/steps/get-federated-access-token.yml | 14 +++++++++++++- global.json | 4 ++-- 14 files changed, 44 insertions(+), 29 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1c1af53162..22f69184b8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - c9efa535175049eb9cba06cae1f8c3d5dbe768a9 + 1e2be7464703499cf98e20536fb4da4218c8fce1 diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config index 3849bdb3cf..5bfbb02ef0 100644 --- a/eng/common/sdl/NuGet.config +++ b/eng/common/sdl/NuGet.config @@ -5,11 +5,11 @@ - + - + diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 4715d75e97..81ded5b7f4 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -6,7 +6,6 @@ Param( [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located - [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list # format. @@ -75,7 +74,7 @@ try { } Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel + & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -GuardianLoggerLevel $GuardianLoggerLevel } $gdnFolder = Join-Path $workingDirectory '.gdn' @@ -104,7 +103,6 @@ try { -TargetDirectory $targetDirectory ` -GdnFolder $gdnFolder ` -ToolsList $tools ` - -AzureDevOpsAccessToken $AzureDevOpsAccessToken ` -GuardianLoggerLevel $GuardianLoggerLevel ` -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams ` diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 3ac1d92b37..588ff8e22f 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -3,7 +3,6 @@ Param( [string] $Repository, [string] $BranchName='master', [string] $WorkingDirectory, - [string] $AzureDevOpsAccessToken, [string] $GuardianLoggerLevel='Standard' ) @@ -21,14 +20,7 @@ $ci = $true # Don't display the console progress UI - it's a huge perf hit $ProgressPreference = 'SilentlyContinue' -# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file -$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) -$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" -$zipFile = "$WorkingDirectory/gdn.zip" - Add-Type -AssemblyName System.IO.Compression.FileSystem -$gdnFolder = (Join-Path $WorkingDirectory '.gdn') try { # if the folder does not exist, we'll do a guardian init and push it to the remote repository diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1 index 648c5068d7..7fe603fe99 100644 --- a/eng/common/sdl/sdl.ps1 +++ b/eng/common/sdl/sdl.ps1 @@ -4,6 +4,8 @@ function Install-Gdn { [Parameter(Mandatory=$true)] [string]$Path, + [string]$Source = "https://pkgs.dev.azure.com/dnceng/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json", + # If omitted, install the latest version of Guardian, otherwise install that specific version. [string]$Version ) @@ -19,7 +21,7 @@ function Install-Gdn { $ci = $true . $PSScriptRoot\..\tools.ps1 - $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") + $argumentList = @("install", "Microsoft.Guardian.Cli.win-x64", "-Source $Source", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") if ($Version) { $argumentList += "-Version $Version" diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index ba3e7df815..0117328800 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -149,7 +149,7 @@ jobs: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index 0dfa387e7b..b81b8770b3 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -281,7 +281,7 @@ stages: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml index 07426fde05..301d5c591e 100644 --- a/eng/common/templates-official/steps/execute-sdl.yml +++ b/eng/common/templates-official/steps/execute-sdl.yml @@ -9,8 +9,6 @@ parameters: steps: - task: NuGetAuthenticate@1 - inputs: - nuGetServiceConnections: GuardianConnect - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml index e3786cef6d..55e33bd38f 100644 --- a/eng/common/templates-official/steps/get-federated-access-token.yml +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -3,6 +3,12 @@ parameters: type: string - name: outputVariableName type: string +- name: stepName + type: string + default: 'getFederatedAccessToken' +- name: condition + type: string + default: '' # Resource to get a token for. Common values include: # - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps # - 'https://storage.azure.com/' for storage @@ -10,10 +16,16 @@ parameters: - name: resource type: string default: '499b84ac-1321-427f-aa17-267ca6975798' +- name: isStepOutputVariable + type: boolean + default: false steps: - task: AzureCLI@2 displayName: 'Getting federated access token for feeds' + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} inputs: azureSubscription: ${{ parameters.federatedServiceConnection }} scriptType: 'pscore' @@ -25,4 +37,4 @@ steps: exit 1 } Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 57a41f0a3e..cc2b346ba8 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -145,7 +145,7 @@ jobs: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 2db4933468..c3b6a3012f 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -277,7 +277,7 @@ stages: scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml index 07426fde05..fe0ebf8c90 100644 --- a/eng/common/templates/steps/execute-sdl.yml +++ b/eng/common/templates/steps/execute-sdl.yml @@ -9,8 +9,6 @@ parameters: steps: - task: NuGetAuthenticate@1 - inputs: - nuGetServiceConnections: GuardianConnect - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' @@ -36,16 +34,19 @@ steps: displayName: Execute SDL (Overridden) continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} + env: + GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken) - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: ${{ parameters.executeAllSdlToolsScript }} -GuardianCliLocation $(GuardianCliLocation) -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} displayName: Execute SDL continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} + env: + GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken) - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: # We want to publish the Guardian results and configuration for easy diagnosis. However, the diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml index c8c49cc0e8..55e33bd38f 100644 --- a/eng/common/templates/steps/get-federated-access-token.yml +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -3,6 +3,12 @@ parameters: type: string - name: outputVariableName type: string +- name: stepName + type: string + default: 'getFederatedAccessToken' +- name: condition + type: string + default: '' # Resource to get a token for. Common values include: # - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps # - 'https://storage.azure.com/' for storage @@ -10,10 +16,16 @@ parameters: - name: resource type: string default: '499b84ac-1321-427f-aa17-267ca6975798' +- name: isStepOutputVariable + type: boolean + default: false steps: - task: AzureCLI@2 displayName: 'Getting federated access token for feeds' + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} inputs: azureSubscription: ${{ parameters.federatedServiceConnection }} scriptType: 'pscore' @@ -25,4 +37,4 @@ steps: exit 1 } Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken" \ No newline at end of file + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/global.json b/global.json index 6d86fe094b..b80ec64013 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "8.0.101" + "dotnet": "8.0.107" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24360.5" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24376.1" } } From 401e1de595f3fae1e785004dc1f2dbac2b382a0d Mon Sep 17 00:00:00 2001 From: Jason Zhai Date: Tue, 6 Aug 2024 20:08:40 -0700 Subject: [PATCH 88/93] Revert eng/Version.Details.xml --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b395ee9c3f..22f69184b8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -6,9 +6,9 @@ 3dc05150cf234f76f6936dcb2853d31a0da1f60e - + https://github.com/dotnet/source-build-reference-packages - 30ed464acd37779c64e9dc652d4460543ebf9966 + 6f814daa935e08b578b1c0c65a1f26ea3317f517 From fa45b12b7ef0216d9bb35b99267e18c4ed71f974 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 14 Aug 2024 16:47:55 +0000 Subject: [PATCH 89/93] Update dependencies from https://github.com/dotnet/arcade build 20240812.1 (#8279) [release/8.0.3xx] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- .../templates-official/steps/get-federated-access-token.yml | 2 +- eng/common/templates/steps/get-federated-access-token.yml | 2 +- global.json | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 22f69184b8..110dd738b7 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 1e2be7464703499cf98e20536fb4da4218c8fce1 + 770e16f44e6727d0efe1168e62279a399cc92edd diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml index d556f9c9fb..55e33bd38f 100644 --- a/eng/common/templates-official/steps/get-federated-access-token.yml +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -37,4 +37,4 @@ steps: exit 1 } Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml index d556f9c9fb..55e33bd38f 100644 --- a/eng/common/templates/steps/get-federated-access-token.yml +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -37,4 +37,4 @@ steps: exit 1 } Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" - Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/global.json b/global.json index b80ec64013..616015e9e3 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.107" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24376.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24412.1" } } From 7778fb62bcdc16e0bab701e2ec69f88c370d8b73 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Thu, 15 Aug 2024 12:57:38 +0000 Subject: [PATCH 90/93] Update dependencies from https://github.com/dotnet/arcade build 20240813.2 Microsoft.DotNet.Arcade.Sdk From Version 8.0.0-beta.24412.1 -> To Version 8.0.0-beta.24413.2 --- eng/Version.Details.xml | 4 ++-- global.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 110dd738b7..651c43c64e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 770e16f44e6727d0efe1168e62279a399cc92edd + 51321b7e150a2f426cb9e1334687bdfab68ec323 diff --git a/global.json b/global.json index 616015e9e3..a2cb2eadfd 100644 --- a/global.json +++ b/global.json @@ -1,8 +1,8 @@ { "tools": { - "dotnet": "8.0.107" + "dotnet": "8.0.108" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24412.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24413.2" } } From 1b3e17798a3bf544217069d22c20731af1b20873 Mon Sep 17 00:00:00 2001 From: Jason Zhai Date: Thu, 15 Aug 2024 23:49:42 -0700 Subject: [PATCH 91/93] Revert NuGet.config --- NuGet.config | 5 ----- 1 file changed, 5 deletions(-) diff --git a/NuGet.config b/NuGet.config index fa7398bab2..d0ff87cda9 100644 --- a/NuGet.config +++ b/NuGet.config @@ -4,11 +4,6 @@ - - - - - From 4b408bd6b79a14c764de0139ba1c89b40f9c99ee Mon Sep 17 00:00:00 2001 From: Sean Reeser Date: Thu, 29 Aug 2024 11:06:14 -0700 Subject: [PATCH 92/93] Update branding to 8.0.306 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 5fd7db6500..ea2396a9ff 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -4,7 +4,7 @@ true true - 8.0.305 + 8.0.306 true release From 0d38d5d096c10eafbee6e5b9d3ceb600594cac10 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 29 Aug 2024 23:24:49 +0000 Subject: [PATCH 93/93] [release/8.0.3xx] Update dependencies from dotnet/arcade (#8313) [release/8.0.3xx] Update dependencies from dotnet/arcade --- NuGet.config | 17 ++++++++++++++++- eng/Version.Details.xml | 4 ++-- eng/common/templates/steps/telemetry-start.yml | 2 +- global.json | 2 +- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/NuGet.config b/NuGet.config index d0ff87cda9..bc64adc532 100644 --- a/NuGet.config +++ b/NuGet.config @@ -4,6 +4,11 @@ + + + + + @@ -16,5 +21,15 @@ - + + + + + + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 467f2c55f9..16fb64a20f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -17,9 +17,9 @@ - + https://github.com/dotnet/arcade - 51321b7e150a2f426cb9e1334687bdfab68ec323 + 80264e60280e2815e7d65871081ccac04a32445c diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml index 32c01ef0b5..6abbcb33a6 100644 --- a/eng/common/templates/steps/telemetry-start.yml +++ b/eng/common/templates/steps/telemetry-start.yml @@ -8,7 +8,7 @@ parameters: steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}: - - task: AzureKeyVault@1 + - task: AzureKeyVault@2 inputs: azureSubscription: 'HelixProd_KeyVault' KeyVaultName: HelixProdKV diff --git a/global.json b/global.json index a2cb2eadfd..da51920af9 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "8.0.108" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24413.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24426.2" } }