diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 35884346f5a..57a9f2f56c0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -7,205 +7,205 @@ Note: if the Uri is a new place, you will need to add a subscription from that p --> - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/runtime - 56610095196ac12a397b1acd00835db4d86849b9 + 5111fdc0dc464f01647d6b6078342f451bf3a499 - + https://github.com/dotnet/arcade - 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc + e543752a4d92d8df16a9f95b89fcfdc50aeabf74 - + https://github.com/dotnet/arcade - 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc + e543752a4d92d8df16a9f95b89fcfdc50aeabf74 - + https://github.com/dotnet/arcade - 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc + e543752a4d92d8df16a9f95b89fcfdc50aeabf74 - + https://github.com/dotnet/arcade - 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc + e543752a4d92d8df16a9f95b89fcfdc50aeabf74 - + https://github.com/dotnet/arcade - 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc + e543752a4d92d8df16a9f95b89fcfdc50aeabf74 - + https://github.com/dotnet/arcade - 6b1c5542109fd5d89a9bde9eb6aacb0ad04e18bc + e543752a4d92d8df16a9f95b89fcfdc50aeabf74 diff --git a/eng/Versions.props b/eng/Versions.props index 5e35e885f09..12741577032 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -13,35 +13,35 @@ - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 5.0.0-preview.7.20320.5 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 6.0.0 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 - 9.0.0-preview.4.24219.3 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 + 9.0.0-preview.4.24222.10 @@ -54,9 +54,9 @@ - 9.0.0-beta.24219.5 - 9.0.0-beta.24219.5 - 9.0.0-beta.24219.5 + 9.0.0-beta.24223.8 + 9.0.0-beta.24223.8 + 9.0.0-beta.24223.8 17.4.0-preview-20220707-01 diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml new file mode 100644 index 00000000000..dc3bd560a50 --- /dev/null +++ b/eng/common/core-templates/job/job.yml @@ -0,0 +1,266 @@ +parameters: +# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + cancelTimeoutInMinutes: '' + condition: '' + container: '' + continueOnError: false + dependsOn: '' + displayName: '' + pool: '' + steps: [] + strategy: '' + timeoutInMinutes: '' + variables: [] + workspace: '' + templateContext: {} + +# Job base template specific parameters + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + # publishing defaults + artifacts: '' + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishBuildAssets: false + enablePublishTestResults: false + enablePublishUsingPipelines: false + enableBuildRetry: false + disableComponentGovernance: '' + componentGovernanceIgnoreDirectories: '' + mergeTestResults: false + testRunTitle: '' + testResultsFormat: '' + name: '' + preSteps: [] + artifactPublishSteps: [] + runAsPublic: false + +# Sbom related params + enableSbom: true + PackageVersion: 9.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + +# 1es specific parameters + is1ESPipeline: '' + +jobs: +- job: ${{ parameters.name }} + + ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: + cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} + + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + + ${{ if ne(parameters.container, '') }}: + container: ${{ parameters.container }} + + ${{ if ne(parameters.continueOnError, '') }}: + continueOnError: ${{ parameters.continueOnError }} + + ${{ if ne(parameters.dependsOn, '') }}: + dependsOn: ${{ parameters.dependsOn }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: ${{ parameters.displayName }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + + ${{ if ne(parameters.strategy, '') }}: + strategy: ${{ parameters.strategy }} + + ${{ if ne(parameters.timeoutInMinutes, '') }}: + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + ${{ if ne(parameters.templateContext, '') }}: + templateContext: ${{ parameters.templateContext }} + + variables: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: + - name: DOTNET_CLI_TELEMETRY_PROFILE + value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' + # Retry signature validation up to three times, waiting 2 seconds between attempts. + # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures + - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY + value: 3,2000 + - ${{ each variable in parameters.variables }}: + # handle name-value variable syntax + # example: + # - name: [key] + # value: [value] + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + + # handle variable groups + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + # handle template variable syntax + # example: + # - template: path/to/template.yml + # parameters: + # [key]: [value] + - ${{ if ne(variable.template, '') }}: + - template: ${{ variable.template }} + ${{ if ne(variable.parameters, '') }}: + parameters: ${{ variable.parameters }} + + # handle key-value variable syntax. + # example: + # - [key]: [value] + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: + - ${{ each pair in variable }}: + - name: ${{ pair.key }} + value: ${{ pair.value }} + + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: DotNet-HelixApi-Access + + ${{ if ne(parameters.workspace, '') }}: + workspace: ${{ parameters.workspace }} + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild plugin + inputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + env: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: + - task: NuGetAuthenticate@1 + + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + + - ${{ each step in parameters.steps }}: + - ${{ step }} + + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} + continueOnError: true + + - template: /eng/common/core-templates/steps/component-governance.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ if eq(parameters.disableComponentGovernance, '') }}: + ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: + disableComponentGovernance: false + ${{ else }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: MicroBuildCleanup@1 + displayName: Execute Microbuild cleanup tasks + condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + env: + TeamName: $(_TeamName) + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - template: /eng/common/core-templates/steps/generate-sbom.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + PackageVersion: ${{ parameters.packageVersion}} + BuildDropPath: ${{ parameters.buildDropPath }} + IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} + publishArtifacts: false + + # Publish test results + - ${{ if and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')) }}: + - ${{ if eq(parameters.testResultsFormat, 'xunit') }}: + - task: PublishTestResults@2 + displayName: Publish XUnit Test Results + inputs: + testResultsFormat: 'xUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + - ${{ if eq(parameters.testResultsFormat, 'vstest') }}: + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() + + # gather artifacts + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - task: CopyFiles@2 + displayName: Gather logs for publish to artifacts + inputs: + SourceFolder: 'artifacts/log' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/log' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - task: CopyFiles@2 + displayName: Gather logs for publish to artifacts + inputs: + SourceFolder: 'artifacts/log/$(_BuildConfig)' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - task: CopyFiles@2 + displayName: Gather buildconfiguration for build retry + inputs: + SourceFolder: '$(Build.SourcesDirectory)/eng/common/BuildConfiguration' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/eng/common/BuildConfiguration' + + - ${{ each step in parameters.artifactPublishSteps }}: + - ${{ step }} diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml new file mode 100644 index 00000000000..00feec8ebbc --- /dev/null +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -0,0 +1,121 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: '' + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + ReusePr: true + UseLfLineEndings: true + UseCheckedInLocProjectJson: false + SkipLocProjectJsonGeneration: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + GitHubOrg: dotnet + MirrorRepo: '' + MirrorBranch: main + condition: '' + JobNameSuffix: '' + is1ESPipeline: '' +jobs: +- job: OneLocBuild${{ parameters.JobNameSuffix }} + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild${{ parameters.JobNameSuffix }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: eng/Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + isShouldReusePrSelected: ${{ parameters.ReusePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + ${{ if ne(parameters.MirrorRepo, '') }}: + isMirrorRepoSelected: true + gitHubOrganization: ${{ parameters.GitHubOrg }} + mirrorRepo: ${{ parameters.MirrorRepo }} + mirrorBranch: ${{ parameters.MirrorBranch }} + condition: ${{ parameters.condition }} + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish Localization Files + pathToPublish: '$(Build.ArtifactStagingDirectory)/loc' + publishLocation: Container + artifactName: Loc + condition: ${{ parameters.condition }} + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish LocProject.json + pathToPublish: '$(Build.SourcesDirectory)/eng/Localize/' + publishLocation: Container + artifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml new file mode 100644 index 00000000000..8fe9299542c --- /dev/null +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -0,0 +1,172 @@ +parameters: + configuration: 'Debug' + + # Optional: condition for the job to run + condition: '' + + # Optional: 'true' if future jobs should run even if this job fails + continueOnError: false + + # Optional: dependencies of the job + dependsOn: '' + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: {} + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishUsingPipelines: false + + # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing + publishAssetsImmediately: false + + artifactsPublishingAdditionalParameters: '' + + signingValidationAdditionalParameters: '' + + is1ESPipeline: '' + +jobs: +- job: Asset_Registry_Publish + + dependsOn: ${{ parameters.dependsOn }} + timeoutInMinutes: 150 + + ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + displayName: Publish Assets + ${{ else }}: + displayName: Publish to Build Asset Registry + + variables: + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - name: runCodesignValidationInjection + value: false + # unconditional - needed for logs publishing (redactor tool version) + - template: /eng/common/core-templates/post-build/common-variables.yml + + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 + os: windows + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - checkout: self + fetchDepth: 3 + clean: true + + - task: DownloadBuildArtifacts@0 + displayName: Download artifact + inputs: + artifactName: AssetManifests + downloadPath: '$(Build.StagingDirectory)/Download' + checkDownloadedFiles: true + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Build Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' + /p:BuildAssetRegistryToken=$(MaestroAccessToken) + /p:MaestroApiEndpoint=https://maestro.dot.net + /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} + /p:OfficialBuildId=$(Build.BuildNumber) + condition: ${{ parameters.condition }} + continueOnError: ${{ parameters.continueOnError }} + + - task: powershell@2 + displayName: Create ReleaseConfigs Artifact + inputs: + targetType: inline + script: | + New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force + $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" + Add-Content -Path $filePath -Value $(BARBuildId) + Add-Content -Path $filePath -Value "$(DefaultChannels)" + Add-Content -Path $filePath -Value $(IsStableBuild) + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish ReleaseConfigs Artifact + pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs' + publishLocation: Container + artifactName: ReleaseConfigs + + - task: powershell@2 + displayName: Check if SymbolPublishingExclusionsFile.txt exists + inputs: + targetType: inline + script: | + $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + if(Test-Path -Path $symbolExclusionfile) + { + Write-Host "SymbolExclusionFile exists" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" + } + else{ + Write-Host "Symbols Exclusion file does not exist" + Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" + } + + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish SymbolPublishingExclusionsFile Artifact + condition: eq(variables['SymbolExclusionFile'], 'true') + pathToPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + publishLocation: Container + artifactName: ReleaseConfigs + + - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion 3 + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml new file mode 100644 index 00000000000..c0ce4b3c861 --- /dev/null +++ b/eng/common/core-templates/job/source-build.yml @@ -0,0 +1,80 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + + is1ESPipeline: '' + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + + ${{ if eq(parameters.platform.pool, '') }}: + # The default VM host AzDO pool. This should be capable of running Docker containers: almost all + # source-build builds run in Docker, including the default managed platform. + # /eng/common/core-templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic + ${{ if eq(parameters.is1ESPipeline, 'true') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals build.ubuntu.2004.amd64 + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + image: 1es-mariner-2 + os: linux + ${{ else }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - template: /eng/common/core-templates/steps/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + platform: ${{ parameters.platform }} diff --git a/eng/common/core-templates/job/source-index-stage1.yml b/eng/common/core-templates/job/source-index-stage1.yml new file mode 100644 index 00000000000..9c6e5ae3c3e --- /dev/null +++ b/eng/common/core-templates/job/source-index-stage1.yml @@ -0,0 +1,73 @@ +parameters: + runAsPublic: false + sourceIndexPackageVersion: 1.0.1-20240129.2 + sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json + sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" + preSteps: [] + binlogPath: artifacts/log/Debug/Build.binlog + condition: '' + dependsOn: '' + pool: '' + is1ESPipeline: '' + +jobs: +- job: SourceIndexStage1 + dependsOn: ${{ parameters.dependsOn }} + condition: ${{ parameters.condition }} + variables: + - name: SourceIndexPackageVersion + value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexPackageSource + value: ${{ parameters.sourceIndexPackageSource }} + - name: BinlogPath + value: ${{ parameters.binlogPath }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - group: source-dot-net stage1 variables + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + ${{ if ne(parameters.pool, '') }}: + pool: ${{ parameters.pool }} + ${{ if eq(parameters.pool, '') }}: + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: windows.vs2022.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: windows.vs2022.amd64 + + steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} + + - task: UseDotNet@2 + displayName: Use .NET 8 SDK + inputs: + packageType: sdk + version: 8.0.x + installationPath: $(Agent.TempDirectory)/dotnet + workingDirectory: $(Agent.TempDirectory) + + - script: | + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + displayName: Download Tools + # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. + workingDirectory: $(Agent.TempDirectory) + + - script: ${{ parameters.sourceIndexBuildCommand }} + displayName: Build Repository + + - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output + displayName: Process Binlog into indexable sln + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + displayName: Upload stage1 artifacts to source index + env: + BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/core-templates/jobs/codeql-build.yml b/eng/common/core-templates/jobs/codeql-build.yml new file mode 100644 index 00000000000..f2144252cc6 --- /dev/null +++ b/eng/common/core-templates/jobs/codeql-build.yml @@ -0,0 +1,33 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + # Optional: if specified, restore and use this version of Guardian instead of the default. + overrideGuardianVersion: '' + is1ESPipeline: '' + +jobs: +- template: /eng/common/core-templates/jobs/jobs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: false + enableTelemetry: true + + variables: + - group: Publish-Build-Assets + # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in + # sync with the packages.config file. + - name: DefaultGuardianVersion + value: 0.109.0 + - name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + - name: GuardianVersion + value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + + jobs: ${{ parameters.jobs }} + diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml new file mode 100644 index 00000000000..ea69be4341c --- /dev/null +++ b/eng/common/core-templates/jobs/jobs.yml @@ -0,0 +1,119 @@ +parameters: + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md + continueOnError: false + + # Optional: Include PublishBuildArtifacts task + enablePublishBuildArtifacts: false + + # Optional: Enable publishing using release pipelines + enablePublishUsingPipelines: false + + # Optional: Enable running the source-build jobs to build repo from source + enableSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/core-templates/jobs/source-build.yml for options + sourceBuildParameters: [] + + graphFileGeneration: + # Optional: Enable generating the graph files at the end of the build + enabled: false + # Optional: Include toolset dependencies in the generated graph files + includeToolset: false + + # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job + jobs: [] + + # Optional: Override automatically derived dependsOn value for "publish build assets" job + publishBuildAssetsDependsOn: '' + + # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. + publishAssetsImmediately: false + + # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) + artifactsPublishingAdditionalParameters: '' + signingValidationAdditionalParameters: '' + + # Optional: should run as a public build even in the internal project + # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + runAsPublic: false + + enableSourceIndex: false + sourceIndexParams: {} + + artifacts: {} + is1ESPipeline: '' + +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + +jobs: +- ${{ each job in parameters.jobs }}: + - ${{ if eq(parameters.is1ESPipeline, 'true') }}: + - template: /eng/common/templates-official/job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + + - ${{ else }}: + - template: /eng/common/templates/job/job.yml + parameters: + # pass along parameters + ${{ each parameter in parameters }}: + ${{ if ne(parameter.key, 'jobs') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # pass along job properties + ${{ each property in job }}: + ${{ if ne(property.key, 'job') }}: + ${{ property.key }}: ${{ property.value }} + + name: ${{ job.job }} + +- ${{ if eq(parameters.enableSourceBuild, true) }}: + - template: /eng/common/core-templates/jobs/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.enableSourceIndex, 'true') }}: + - template: ../job/source-index-stage1.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + runAsPublic: ${{ parameters.runAsPublic }} + ${{ each parameter in parameters.sourceIndexParams }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + - ${{ if eq(parameters.enableSourceBuild, true) }}: + - Source_Build_Complete + + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/core-templates/jobs/source-build.yml b/eng/common/core-templates/jobs/source-build.yml new file mode 100644 index 00000000000..d8e5d008522 --- /dev/null +++ b/eng/common/core-templates/jobs/source-build.yml @@ -0,0 +1,50 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/core-templates/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # This is the default platform provided by Arcade, intended for use by a managed-only repo. + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. If no platforms are specified, + # one job runs on 'defaultManagedPlatform'. + platforms: [] + + is1ESPipeline: '' + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(length(parameters.platforms), 0) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(length(parameters.platforms), 0) }}: + - template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/core-templates/post-build/common-variables.yml b/eng/common/core-templates/post-build/common-variables.yml new file mode 100644 index 00000000000..b9ede10bf09 --- /dev/null +++ b/eng/common/core-templates/post-build/common-variables.yml @@ -0,0 +1,24 @@ +variables: + - group: Publish-Build-Assets + + # Whether the build is internal or not + - name: IsInternalBuild + value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} + + # Default Maestro++ API Endpoint and API Version + - name: MaestroApiEndPoint + value: "https://maestro.dot.net" + - name: MaestroApiAccessToken + value: $(MaestroAccessToken) + - name: MaestroApiVersion + value: "2020-02-20" + + - name: SourceLinkCLIVersion + value: 3.0.0 + - name: SymbolToolVersion + value: 1.0.1 + - name: BinlogToolVersion + value: 1.0.11 + + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml new file mode 100644 index 00000000000..ed1e6692f73 --- /dev/null +++ b/eng/common/core-templates/post-build/post-build.yml @@ -0,0 +1,298 @@ +parameters: + # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. + # Publishing V1 is no longer supported + # Publishing V2 is no longer supported + # Publishing V3 is the default + - name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + + - name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + + - name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + + - name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + + - name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + + - name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + + - name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + + - name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + + - name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + + # These parameters let the user customize the call to sdk-task.ps1 for publishing + # symbols & general artifacts as well as for signing validation + - name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + + - name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + + - name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + + # Which stages should finish execution before post-build stages start + - name: validateDependsOn + type: object + default: + - build + + - name: publishDependsOn + type: object + default: + - Validate + + # Optional: Call asset publishing rather than running in a separate stage + - name: publishAssetsImmediately + type: boolean + default: false + + - name: is1ESPipeline + type: boolean + default: false + +stages: +- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets + variables: + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobs: + - job: + displayName: NuGet Validation + condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + itemPattern: | + ** + !**/Microsoft.SourceBuild.Intermediate.*.nupkg + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) + + - job: + displayName: SourceLink Validation + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022 + os: windows + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + +- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: + - stage: publish_using_darc + ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: + dependsOn: ${{ parameters.publishDependsOn }} + ${{ else }}: + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc + variables: + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + jobs: + - job: + displayName: Publish Using Darc + timeoutInMinutes: 120 + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: AzurePipelines-EO + image: 1ESPT-Windows2022 + demands: Cmd + os: windows + # If it's not devdiv, it's dnceng + ${{ else }}: + name: NetCore1ESPool-Publishing-Internal + image: windows.vs2019.amd64 + os: windows + steps: + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: NuGetAuthenticate@1 + + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish true + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/core-templates/post-build/setup-maestro-vars.yml b/eng/common/core-templates/post-build/setup-maestro-vars.yml new file mode 100644 index 00000000000..8d56b572679 --- /dev/null +++ b/eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -0,0 +1,74 @@ +parameters: + BARBuildId: '' + PromoteToChannelIds: '' + is1ESPipeline: '' + +steps: + - ${{ if eq(parameters.is1ESPipeline, '') }}: + - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + + - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Release Configs + inputs: + buildType: current + artifactName: ReleaseConfigs + checkDownloadedFiles: true + + - task: PowerShell@2 + name: setReleaseVars + displayName: Set Release Configs Vars + inputs: + targetType: inline + pwsh: true + script: | + try { + if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + $Channels = $Content | Select -Index 1 + $IsStableBuild = $Content | Select -Index 2 + + $AzureDevOpsProject = $Env:System_TeamProject + $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId + $AzureDevOpsBuildId = $Env:Build_BuildId + } + else { + $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + + $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' + $apiHeaders.Add('Accept', 'application/json') + $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") + + $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } + + $BarId = $Env:BARBuildId + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels]" + + $IsStableBuild = $buildInfo.stable + $AzureDevOpsProject = $buildInfo.azureDevOpsProject + $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId + $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId + } + + Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" + Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" + + Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" + Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" + Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } + env: + MAESTRO_API_TOKEN: $(MaestroApiAccessToken) + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/core-templates/post-build/trigger-subscription.yml b/eng/common/core-templates/post-build/trigger-subscription.yml new file mode 100644 index 00000000000..da669030daf --- /dev/null +++ b/eng/common/core-templates/post-build/trigger-subscription.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Triggering subscriptions + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 + arguments: -SourceRepo $(Build.Repository.Uri) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/core-templates/steps/add-build-to-channel.yml b/eng/common/core-templates/steps/add-build-to-channel.yml new file mode 100644 index 00000000000..f67a210d62f --- /dev/null +++ b/eng/common/core-templates/steps/add-build-to-channel.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/core-templates/steps/component-governance.yml b/eng/common/core-templates/steps/component-governance.yml new file mode 100644 index 00000000000..df449a34c11 --- /dev/null +++ b/eng/common/core-templates/steps/component-governance.yml @@ -0,0 +1,14 @@ +parameters: + disableComponentGovernance: false + componentGovernanceIgnoreDirectories: '' + is1ESPipeline: false + +steps: +- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: + - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" + displayName: Set skipComponentGovernanceDetection variable +- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: + - task: ComponentGovernanceComponentDetection@0 + continueOnError: true + inputs: + ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml new file mode 100644 index 00000000000..d938b60e1bb --- /dev/null +++ b/eng/common/core-templates/steps/generate-sbom.yml @@ -0,0 +1,54 @@ +# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. +# PackageName - The name of the package this SBOM represents. +# PackageVersion - The version of the package this SBOM represents. +# ManifestDirPath - The path of the directory where the generated manifest files will be placed +# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. + +parameters: + PackageVersion: 9.0.0 + BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageName: '.NET' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom + IgnoreDirectories: '' + sbomContinueOnError: true + is1ESPipeline: false + # disable publishArtifacts if some other step is publishing the artifacts (like job.yml). + publishArtifacts: true + +steps: +- task: PowerShell@2 + displayName: Prep for SBOM generation in (Non-linux) + condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) + inputs: + filePath: ./eng/common/generate-sbom-prep.ps1 + arguments: ${{parameters.manifestDirPath}} + +# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 +- script: | + chmod +x ./eng/common/generate-sbom-prep.sh + ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} + displayName: Prep for SBOM generation in (Linux) + condition: eq(variables['Agent.Os'], 'Linux') + continueOnError: ${{ parameters.sbomContinueOnError }} + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generate SBOM manifest' + continueOnError: ${{ parameters.sbomContinueOnError }} + inputs: + PackageName: ${{ parameters.packageName }} + BuildDropPath: ${{ parameters.buildDropPath }} + PackageVersion: ${{ parameters.packageVersion }} + ManifestDirPath: ${{ parameters.manifestDirPath }} + ${{ if ne(parameters.IgnoreDirectories, '') }}: + AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' + +- ${{ if eq(parameters.publishArtifacts, 'true')}}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish SBOM manifest + continueOnError: ${{parameters.sbomContinueOnError}} + targetPath: '${{ parameters.manifestDirPath }}' + artifactName: $(ARTIFACT_NAME) + diff --git a/eng/common/core-templates/steps/publish-build-artifacts.yml b/eng/common/core-templates/steps/publish-build-artifacts.yml new file mode 100644 index 00000000000..f24ce346684 --- /dev/null +++ b/eng/common/core-templates/steps/publish-build-artifacts.yml @@ -0,0 +1,20 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false +- name: args + type: object + default: {} +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - template: /eng/common/templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ each parameter in parameters.args }}: + ${{ parameter.key }}: ${{ parameter.value }} +- ${{ else }}: + - template: /eng/common/templates-official/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + ${{ each parameter in parameters.args }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml new file mode 100644 index 00000000000..8c5ea77b586 --- /dev/null +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -0,0 +1,59 @@ +parameters: + StageLabel: '' + JobLabel: '' + CustomSensitiveDataList: '' + # A default - in case value from eng/common/core-templates/post-build/common-variables.yml is not passed + BinlogToolVersion: '1.0.11' + is1ESPipeline: false + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: PowerShell@2 + displayName: Redact Logs + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 + # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml + # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + # If the file exists - sensitive data for redaction will be sourced from it + # (single entry per line, lines starting with '# ' are considered comments and skipped) + arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' + -BinlogToolVersion ${{parameters.BinlogToolVersion}} + -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + '$(publishing-dnceng-devdiv-code-r-build-re)' + '$(MaestroAccessToken)' + '$(dn-bot-all-orgs-artifact-feeds-rw)' + '$(akams-client-id)' + '$(akams-client-secret)' + '$(microsoft-symbol-server-pat)' + '$(symweb-symbol-server-pat)' + '$(dn-bot-all-orgs-build-rw-code-rw)' + ${{parameters.CustomSensitiveDataList}} + continueOnError: true + condition: always() + +- task: CopyFiles@2 + displayName: Gather post build logs + inputs: + SourceFolder: '$(Build.SourcesDirectory)/PostBuildLogs' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + +- template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish Logs + pathToPublish: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' + publishLocation: Container + artifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/core-templates/steps/publish-pipeline-artifacts.yml b/eng/common/core-templates/steps/publish-pipeline-artifacts.yml new file mode 100644 index 00000000000..2efec04dc2c --- /dev/null +++ b/eng/common/core-templates/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,20 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false + +- name: args + type: object + default: {} + +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - template: /eng/common/templates/steps/publish-pipeline-artifacts.yml + parameters: + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} +- ${{ else }}: + - template: /eng/common/templates-official/steps/publish-pipeline-artifacts.yml + parameters: + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/core-templates/steps/retain-build.yml b/eng/common/core-templates/steps/retain-build.yml new file mode 100644 index 00000000000..83d97a26a01 --- /dev/null +++ b/eng/common/core-templates/steps/retain-build.yml @@ -0,0 +1,28 @@ +parameters: + # Optional azure devops PAT with build execute permissions for the build's organization, + # only needed if the build that should be retained ran on a different organization than + # the pipeline where this template is executing from + Token: '' + # Optional BuildId to retain, defaults to the current running build + BuildId: '' + # Azure devops Organization URI for the build in the https://dev.azure.com/ format. + # Defaults to the organization the current pipeline is running on + AzdoOrgUri: '$(System.CollectionUri)' + # Azure devops project for the build. Defaults to the project the current pipeline is running on + AzdoProject: '$(System.TeamProject)' + +steps: + - task: powershell@2 + inputs: + targetType: 'filePath' + filePath: eng/common/retain-build.ps1 + pwsh: true + arguments: > + -AzdoOrgUri: ${{parameters.AzdoOrgUri}} + -AzdoProject ${{parameters.AzdoProject}} + -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} + -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} + displayName: Enable permanent build retention + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/core-templates/steps/send-to-helix.yml b/eng/common/core-templates/steps/send-to-helix.yml new file mode 100644 index 00000000000..68fa739c4ab --- /dev/null +++ b/eng/common/core-templates/steps/send-to-helix.yml @@ -0,0 +1,93 @@ +# Please remember to update the documentation if you make changes to these parameters! +parameters: + HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' + HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number + HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues + HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY + HelixProjectArguments: '' # optional -- arguments passed to the build command + HelixConfiguration: '' # optional -- additional property attached to a job + HelixPreCommands: '' # optional -- commands to run before Helix work item execution + HelixPostCommands: '' # optional -- commands to run after Helix work item execution + WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects + WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload + XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true + XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects + XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects + XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner + XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json + WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." + IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) + Creator: '' # optional -- if the build is external, use this to specify who is sending the job + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() + continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + displayName: ${{ parameters.DisplayNamePrefix }} (Windows) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} (Unix) + env: + BuildConfig: $(_BuildConfig) + HelixSource: ${{ parameters.HelixSource }} + HelixType: ${{ parameters.HelixType }} + HelixBuild: ${{ parameters.HelixBuild }} + HelixConfiguration: ${{ parameters.HelixConfiguration }} + HelixTargetQueues: ${{ parameters.HelixTargetQueues }} + HelixAccessToken: ${{ parameters.HelixAccessToken }} + HelixPreCommands: ${{ parameters.HelixPreCommands }} + HelixPostCommands: ${{ parameters.HelixPostCommands }} + WorkItemDirectory: ${{ parameters.WorkItemDirectory }} + WorkItemCommand: ${{ parameters.WorkItemCommand }} + WorkItemTimeout: ${{ parameters.WorkItemTimeout }} + CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} + XUnitProjects: ${{ parameters.XUnitProjects }} + XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} + XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} + XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} + XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} + DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} + DotNetCliVersion: ${{ parameters.DotNetCliVersion }} + WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} + Creator: ${{ parameters.Creator }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/core-templates/steps/source-build.yml b/eng/common/core-templates/steps/source-build.yml new file mode 100644 index 00000000000..bdd725b496f --- /dev/null +++ b/eng/common/core-templates/steps/source-build.yml @@ -0,0 +1,134 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/core-templates/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + is1ESPipeline: false + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + # If building on the internal project, the artifact feeds variable may be available (usually only if needed) + # In that case, call the feed setup script to add internal feeds corresponding to public ones. + # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. + # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those + # changes. + internalRestoreArgs= + if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then + # Temporarily work around https://github.com/dotnet/arcade/issues/7709 + chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh + $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) + internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' + + # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. + # This only works if there is a username/email configured, which won't be the case in most CI runs. + git config --get user.email + if [ $? -ne 0 ]; then + git config user.email dn-bot@microsoft.com + git config user.name dn-bot + fi + fi + + # If building on the internal project, the internal storage variable may be available (usually only if needed) + # In that case, add variables to allow the download of internal runtimes if the specified versions are not found + # in the default public locations. + internalRuntimeDownloadArgs= + if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then + internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' + fi + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + runtimeOsArgs= + if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then + runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' + fi + + baseOsArgs= + if [ '${{ parameters.platform.baseOS }}' != '' ]; then + baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' + fi + + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + + assetManifestFileName=SourceBuild_RidSpecific.xml + if [ '${{ parameters.platform.name }}' != '' ]; then + assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack $publishArgs -bl \ + $officialBuildArgs \ + $internalRuntimeDownloadArgs \ + $internalRestoreArgs \ + $targetRidArgs \ + $runtimeOsArgs \ + $baseOsArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true \ + /p:DotNetBuildSourceOnly=true \ + /p:DotNetBuildRepo=true \ + /p:AssetManifestFileName=$assetManifestFileName + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/sb/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish BuildLogs + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() + +# Manually inject component detection so that we can ignore the source build upstream cache, which contains +# a nupkg cache of input packages (a local feed). +# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' +# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets +- task: ComponentGovernanceComponentDetection@0 + displayName: Component Detection (Exclude upstream cache) + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/core-templates/variables/pool-providers.yml b/eng/common/core-templates/variables/pool-providers.yml new file mode 100644 index 00000000000..41053d382a2 --- /dev/null +++ b/eng/common/core-templates/variables/pool-providers.yml @@ -0,0 +1,8 @@ +parameters: + is1ESPipeline: false + +variables: + - ${{ if eq(parameters.is1ESPipeline, 'true') }}: + - template: /eng/common/templates-official/variables/pool-providers.yml + - ${{ else }}: + - template: /eng/common/templates/variables/pool-providers.yml \ No newline at end of file diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 091023970f1..aab40de3fd9 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.5" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.10.0-pre.4.0" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md new file mode 100644 index 00000000000..595917fac1a --- /dev/null +++ b/eng/common/template-guidance.md @@ -0,0 +1,137 @@ +# Overview + +Arcade provides templates for public (`/templates`) and 1ES pipeline templates (`/templates-official`) scenarios. Pipelines which are required to be managed by 1ES pipeline templates should reference `/templates-offical`, all other pipelines may reference `/templates`. + +## How to use + +Basic guidance is: + +- 1ES Pipeline Template or 1ES Microbuild template runs should reference `eng/common/templates-official`. Any internal production-graded pipeline should use these templates. + +- All other runs should reference `eng/common/templates`. + +See [azure-pipelines.yml](../../azure-pipelines.yml) (templates-official example) or [azure-pipelines-pr.yml](../../azure-pipelines-pr.yml) (templates example) for examples. + +#### The `templateIs1ESManaged` parameter + +The `templateIs1ESManaged` is available on most templates and affects which of the variants is used for nested templates. See [Development Notes](#development-notes) below for more information on the `templateIs1ESManaged1 parameter. + +- For templates under `job/`, `jobs/`, `steps`, or `post-build/`, this parameter must be explicitly set. + +## Multiple outputs + +1ES pipeline templates impose a policy where every publish artifact execution results in additional security scans being injected into your pipeline. When using `templates-official/jobs/jobs.yml`, Arcade reduces the number of additional security injections by gathering all publishing outputs into the [Build.ArtifactStagingDirectory](https://learn.microsoft.com/en-us/azure/devops/pipelines/build/variables?view=azure-devops&tabs=yaml#build-variables-devops-services), and utilizing the [outputParentDirectory](https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/1es-pipeline-templates/features/outputs#multiple-outputs) feature of 1ES pipeline templates. When implementing your pipeline, if you ensure publish artifacts are located in the `$(Build.ArtifactStagingDirectory)`, and utilize the 1ES provided template context, then you can reduce the number of security scans for your pipeline. + +Example: +``` yaml +# azure-pipelines.yml +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + stages: + - stage: build + jobs: + - template: /eng/common/templates-official/jobs/jobs.yml@self + parameters: + # 1ES makes use of outputs to reduce security task injection overhead + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish logs from source' + continueOnError: true + condition: always() + targetPath: $(Build.ArtifactStagingDirectory)/artifacts/log + artifactName: Logs + jobs: + - job: Windows + steps: + - script: echo "friendly neighborhood" > artifacts/marvel/spiderman.txt + # copy build outputs to artifact staging directory for publishing + - task: CopyFiles@2 + displayName: Gather build output + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/marvel' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/marvel' +``` + +Note: Multiple outputs are ONLY applicable to 1ES PT publishing (only usable when referencing `templates-official`). + +# Development notes + +**Folder / file structure** + +``` text +eng\common\ + [templates || templates-official]\ + job\ + job.yml (shim + artifact publishing logic) + onelocbuild.yml (shim) + publish-build-assets.yml (shim) + source-build.yml (shim) + source-index-stage1.yml (shim) + jobs\ + codeql-build.yml (shim) + jobs.yml (shim) + source-build.yml (shim) + post-build\ + post-build.yml (shim) + trigger-subscription.yml (shim) + common-variabls.yml (shim) + setup-maestro-vars.yml (shim) + steps\ + publish-build-artifacts.yml (logic) + publish-pipeline-artifacts.yml (logic) + add-build-channel.yml (shim) + component-governance.yml (shim) + generate-sbom.yml (shim) + publish-logs.yml (shim) + retain-build.yml (shim) + send-to-helix.yml (shim) + source-build.yml (shim) + variables\ + pool-providers.yml (logic + redirect) # templates/variables/pool-providers.yml will redirect to templates-official/variables/pool-providers.yml if you are running in the internal project + sdl-variables.yml (logic) + core-templates\ + job\ + job.yml (logic) + onelocbuild.yml (logic) + publish-build-assets.yml (logic) + source-build.yml (logic) + source-index-stage1.yml (logic) + jobs\ + codeql-build.yml (logic) + jobs.yml (logic) + source-build.yml (logic) + post-build\ + common-variabls.yml (logic) + post-build.yml (logic) + setup-maestro-vars.yml (logic) + trigger-subscription.yml (logic) + steps\ + add-build-to-channel.yml (logic) + component-governance.yml (logic) + generate-sbom.yml (logic) + publish-build-artifacts.yml (redirect) + publish-logs.yml (logic) + publish-pipeline-artifacts.yml (redirect) + retain-build.yml (logic) + send-to-helix.yml (logic) + source-build.yml (logic) + variables\ + pool-providers.yml (redirect) +``` + +In the table above, a file is designated as "shim", "logic", or "redirect". + +- shim - represents a yaml file which is an intermediate step between pipeline logic and .Net Core Engineering's templates (`core-templates`) and defines the `is1ESPipeline` parameter value. + +- logic - represents actual base template logic. + +- redirect- represents a file in `core-templates` which redirects to the "logic" file in either `templates` or `templates-official`. + +Logic for Arcade's templates live **primarily** in the `core-templates` folder. The exceptions to the location of the logic files are around artifact publishing, which is handled differently between 1es pipeline templates and standard templates. `templates` and `templates-official` provide shim entry points which redirect to `core-templates` while also defining the `is1ESPipeline` parameter. If a shim is referenced in `templates`, then `is1ESPipeline` is set to `false`. If a shim is referenced in `templates-official`, then `is1ESPipeline` is set to `true`. + +Within `templates` and `templates-official`, the templates at the "stages", and "jobs" / "job" level have been replaced with shims. Templates at the "steps" and "variables" level are typically too granular to be replaced with shims and instead persist logic which is directly applicable to either scenario. + +Within `core-templates`, there are a handful of places where logic is dependent on which shim entry point was used. In those places, we redirect back to the respective logic file in `templates` or `templates-official`. diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 761acc5eb62..4724e9aaa80 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -1,264 +1,62 @@ -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -parameters: -# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - cancelTimeoutInMinutes: '' - condition: '' - container: '' - continueOnError: false - dependsOn: '' - displayName: '' - pool: '' - steps: [] - strategy: '' - timeoutInMinutes: '' - variables: [] - workspace: '' - templateContext: '' - -# Job base template specific parameters - # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md - artifacts: '' - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishBuildAssets: false - enablePublishTestResults: false - enablePublishUsingPipelines: false - enableBuildRetry: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' - mergeTestResults: false - testRunTitle: '' - testResultsFormat: '' - name: '' - preSteps: [] - runAsPublic: false -# Sbom related params - enableSbom: true - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - jobs: -- job: ${{ parameters.name }} - - ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.continueOnError, '') }}: - continueOnError: ${{ parameters.continueOnError }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - - ${{ if ne(parameters.timeoutInMinutes, '') }}: - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - ${{ if ne(parameters.templateContext, '') }}: - templateContext: ${{ parameters.templateContext }} - - variables: - - ${{ if ne(parameters.enableTelemetry, 'false') }}: - - name: DOTNET_CLI_TELEMETRY_PROFILE - value: '$(Build.Repository.Uri)' - - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - - name: EnableRichCodeNavigation - value: 'true' - # Retry signature validation up to three times, waiting 2 seconds between attempts. - # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY - value: 3,2000 - - ${{ each variable in parameters.variables }}: - # handle name-value variable syntax - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - # handle template variable syntax - # example: - # - template: path/to/template.yml - # parameters: - # [key]: [value] - - ${{ if ne(variable.template, '') }}: - - template: ${{ variable.template }} - ${{ if ne(variable.parameters, '') }}: - parameters: ${{ variable.parameters }} - - # handle key-value variable syntax. - # example: - # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - ${{ if ne(parameters.workspace, '') }}: - workspace: ${{ parameters.workspace }} - - steps: - - ${{ if ne(parameters.preSteps, '') }}: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - env: - TeamName: $(_TeamName) - MicroBuildOutputFolderOverride: '$(Agent.TempDirectory)' - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@1 - - - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - - task: DownloadPipelineArtifact@2 - inputs: - buildType: current - artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} - targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} - itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} - - - ${{ each step in parameters.steps }}: - - ${{ step }} - - - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - - task: RichCodeNavIndexer@0 - displayName: RichCodeNav Upload - inputs: - languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} - environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }} - richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} - continueOnError: true - - - template: /eng/common/templates-official/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: - TeamName: $(_TeamName) - - - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: 'artifacts/bin' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' - - task: CopyFiles@2 - displayName: Gather packages for publish to artifacts - inputs: - SourceFolder: 'artifacts/packages' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish pipeline artifacts - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - continueOnError: true - condition: always() - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: 'artifacts/log' - artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)_Attempt$(System.JobAttempt)') }} - displayName: 'Publish logs' - continueOnError: true - condition: always() - - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} - continueOnError: true - condition: always() - - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - - task: PublishTestResults@2 - displayName: Publish XUnit Test Results - inputs: - testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: - - task: PublishTestResults@2 - displayName: Publish TRX Test Results - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates-official/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion}} - BuildDropPath: ${{ parameters.buildDropPath }} - IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' - artifactName: 'BuildConfiguration' - displayName: 'Publish build retry configuration' - continueOnError: true \ No newline at end of file +- template: /eng/common/core-templates/job/job.yml + parameters: + is1ESPipeline: true + + # publish artifacts + # for 1ES managed templates, use the templateContext.output to handle multiple outputs. + templateContext: + outputParentDirectory: $(Build.ArtifactStagingDirectory) + outputs: + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - output: buildArtifacts + displayName: Publish pipeline artifacts + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + condition: always() + continueOnError: true + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)_Attempt$(System.JobAttempt)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + + - ${{ if eq(parameters.enablePublishBuildArtifacts, true) }}: + - output: buildArtifacts + displayName: Publish Logs + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + publishLocation: Container + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - output: pipelineArtifact + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/eng/common/BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: + - output: pipelineArtifact + displayName: Publish SBOM manifest + continueOnError: true + targetPath: $(Build.ArtifactStagingDirectory)/sbom + artifactName: $(ARTIFACT_NAME) + + # add any outputs provided via root yaml + - ${{ if ne(parameters.templateContext.outputs, '') }}: + - ${{ each output in parameters.templateContext.outputs }}: + - ${{ output }} + + # add any remaining templateContext properties + ${{ each context in parameters.templateContext }}: + ${{ if and(ne(context.key, 'outputParentDirectory'), ne(context.key, 'outputs')) }}: + ${{ context.key }}: ${{ context.value }} + + ${{ each parameter in parameters }}: + ${{ if and(ne(parameter.key, 'templateContext'), ne(parameter.key, 'is1ESPipeline')) }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml index 52b4d05d3f8..0f0c514b912 100644 --- a/eng/common/templates-official/job/onelocbuild.yml +++ b/eng/common/templates-official/job/onelocbuild.yml @@ -1,112 +1,7 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: '' - - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - SourcesDirectory: $(Build.SourcesDirectory) - CreatePr: true - AutoCompletePr: false - ReusePr: true - UseLfLineEndings: true - UseCheckedInLocProjectJson: false - SkipLocProjectJsonGeneration: false - LanguageSet: VS_Main_Languages - LclSource: lclFilesInRepo - LclPackageId: '' - RepoType: gitHub - GitHubOrg: dotnet - MirrorRepo: '' - MirrorBranch: main - condition: '' - JobNameSuffix: '' - jobs: -- job: OneLocBuild${{ parameters.JobNameSuffix }} - - dependsOn: ${{ parameters.dependsOn }} - - displayName: OneLocBuild${{ parameters.JobNameSuffix }} - - variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - - name: _GenerateLocProjectArguments - value: -SourcesDirectory ${{ parameters.SourcesDirectory }} - -LanguageSet "${{ parameters.LanguageSet }}" - -CreateNeutralXlfs - - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: - - name: _GenerateLocProjectArguments - value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - - steps: - - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 - arguments: $(_GenerateLocProjectArguments) - displayName: Generate LocProject.json - condition: ${{ parameters.condition }} - - - task: OneLocBuild@2 - displayName: OneLocBuild - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - locProj: eng/Localize/LocProject.json - outDir: $(Build.ArtifactStagingDirectory) - lclSource: ${{ parameters.LclSource }} - lclPackageId: ${{ parameters.LclPackageId }} - isCreatePrSelected: ${{ parameters.CreatePr }} - isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} - ${{ if eq(parameters.CreatePr, true) }}: - isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} - packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" - ${{ if ne(parameters.MirrorRepo, '') }}: - isMirrorRepoSelected: true - gitHubOrganization: ${{ parameters.GitHubOrg }} - mirrorRepo: ${{ parameters.MirrorRepo }} - mirrorBranch: ${{ parameters.MirrorBranch }} - condition: ${{ parameters.condition }} - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Localization Files - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} +- template: /eng/common/core-templates/job/onelocbuild.yml + parameters: + is1ESPipeline: true - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish LocProject.json - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 38340d3e386..d667a70e8de 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -1,159 +1,7 @@ -parameters: - configuration: 'Debug' - - # Optional: condition for the job to run - condition: '' - - # Optional: 'true' if future jobs should run even if this job fails - continueOnError: false - - # Optional: dependencies of the job - dependsOn: '' - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishUsingPipelines: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishAssetsImmediately: false - - artifactsPublishingAdditionalParameters: '' - - signingValidationAdditionalParameters: '' - jobs: -- job: Asset_Registry_Publish - - dependsOn: ${{ parameters.dependsOn }} - timeoutInMinutes: 150 - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - displayName: Publish Assets - ${{ else }}: - displayName: Publish to Build Asset Registry - - variables: - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - - name: runCodesignValidationInjection - value: false - # unconditional - needed for logs publishing (redactor tool version) - - template: /eng/common/templates-official/post-build/common-variables.yml - - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 - os: windows - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - checkout: self - fetchDepth: 3 - clean: true - - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: NuGetAuthenticate@1 - - - task: PowerShell@2 - displayName: Publish Build Assets - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro.dot.net - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:OfficialBuildId=$(Build.BuildNumber) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: powershell@2 - displayName: Create ReleaseConfigs Artifact - inputs: - targetType: inline - script: | - New-Item -Path "$(Build.StagingDirectory)/ReleaseConfigs" -ItemType Directory -Force - $filePath = "$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt" - Add-Content -Path $filePath -Value $(BARBuildId) - Add-Content -Path $filePath -Value "$(DefaultChannels)" - Add-Content -Path $filePath -Value $(IsStableBuild) - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish ReleaseConfigs Artifact - inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - task: powershell@2 - displayName: Check if SymbolPublishingExclusionsFile.txt exists - inputs: - targetType: inline - script: | - $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" - if(Test-Path -Path $symbolExclusionfile) - { - Write-Host "SymbolExclusionFile exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" - } - else{ - Write-Host "Symbols Exclusion file does not exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" - } - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' +- template: /eng/common/core-templates/job/publish-build-assets.yml + parameters: + is1ESPipeline: true - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - template: /eng/common/templates-official/steps/publish-logs.yml - parameters: - JobLabel: 'Publish_Artifacts_Logs' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index 2180e97a284..1a480034b67 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -1,67 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. The template produces a server job with a - # default ID 'Source_Build_Complete' to put in a dependency list if necessary. - - # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. - jobNamePrefix: 'Source_Build' - - # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for - # managed-only repositories. This is an object with these properties: - # - # name: '' - # The name of the job. This is included in the job ID. - # targetRID: '' - # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false - # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than - # linux-x64), and compiling against distro-provided packages rather than portable ones. - # skipPublishValidation: false - # Disables publishing validation. By default, a check is performed to ensure no packages are - # published by source-build. - # container: '' - # A container to use. Runs in docker. - # pool: {} - # A pool to use. Runs directly on an agent. - # buildScript: '' - # Specifies the build script to invoke to perform the build in the repo. The default - # './build.sh' should work for typical Arcade repositories, but this is customizable for - # difficult situations. - # jobProperties: {} - # A list of job properties to inject at the top level, for potential extensibility beyond - # container and pool. - platform: {} - jobs: -- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} - displayName: Source-Build (${{ parameters.platform.name }}) - - ${{ each property in parameters.platform.jobProperties }}: - ${{ property.key }}: ${{ property.value }} - - ${{ if ne(parameters.platform.container, '') }}: - container: ${{ parameters.platform.container }} - - ${{ if eq(parameters.platform.pool, '') }}: - # The default VM host AzDO pool. This should be capable of running Docker containers: almost all - # source-build builds run in Docker, including the default managed platform. - # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals build.ubuntu.2004.amd64 - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-mariner-2 - os: linux - - ${{ if ne(parameters.platform.pool, '') }}: - pool: ${{ parameters.platform.pool }} - - workspace: - clean: all +- template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: true - steps: - - template: /eng/common/templates-official/steps/source-build.yml - parameters: - platform: ${{ parameters.platform }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index 53a9ef51fd8..6d5ead316f9 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,67 +1,7 @@ -parameters: - runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240129.2 - sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json - sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" - preSteps: [] - binlogPath: artifacts/log/Debug/Build.binlog - condition: '' - dependsOn: '' - pool: '' - jobs: -- job: SourceIndexStage1 - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} - - name: SourceIndexPackageSource - value: ${{ parameters.sourceIndexPackageSource }} - - name: BinlogPath - value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - image: windows.vs2022.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - image: windows.vs2022.amd64 - - steps: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - task: UseDotNet@2 - displayName: Use .NET 8 SDK - inputs: - packageType: sdk - version: 8.0.x - installationPath: $(Agent.TempDirectory)/dotnet - workingDirectory: $(Agent.TempDirectory) - - - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: Download Tools - # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. - workingDirectory: $(Agent.TempDirectory) - - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository - - - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: Process Binlog into indexable sln +- template: /eng/common/core-templates/job/source-index-stage1.yml + parameters: + is1ESPipeline: true - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml index b68d3c2f319..a726322ecfe 100644 --- a/eng/common/templates-official/jobs/codeql-build.yml +++ b/eng/common/templates-official/jobs/codeql-build.yml @@ -1,31 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - jobs: -- template: /eng/common/templates-official/jobs/jobs.yml +- template: /eng/common/core-templates/jobs/codeql-build.yml parameters: - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishTestResults: false - enablePublishBuildAssets: false - enablePublishUsingPipelines: false - enableTelemetry: true + is1ESPipeline: true - variables: - - group: Publish-Build-Assets - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.109.0 - - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - jobs: ${{ parameters.jobs }} - + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml index 857a0f8ba43..007deddaea0 100644 --- a/eng/common/templates-official/jobs/jobs.yml +++ b/eng/common/templates-official/jobs/jobs.yml @@ -1,97 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: Enable publishing using release pipelines - enablePublishUsingPipelines: false - - # Optional: Enable running the source-build jobs to build repo from source - enableSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates-official/jobs/source-build.yml for options - sourceBuildParameters: [] - - graphFileGeneration: - # Optional: Enable generating the graph files at the end of the build - enabled: false - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - - # Optional: Override automatically derived dependsOn value for "publish build assets" job - publishBuildAssetsDependsOn: '' - - # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. - publishAssetsImmediately: false - - # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) - artifactsPublishingAdditionalParameters: '' - signingValidationAdditionalParameters: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - enableSourceIndex: false - sourceIndexParams: {} - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - jobs: -- ${{ each job in parameters.jobs }}: - - template: ../job/job.yml - parameters: - # pass along parameters - ${{ each parameter in parameters }}: - ${{ if ne(parameter.key, 'jobs') }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # pass along job properties - ${{ each property in job }}: - ${{ if ne(property.key, 'job') }}: - ${{ property.key }}: ${{ property.value }} - - name: ${{ job.job }} - -- ${{ if eq(parameters.enableSourceBuild, true) }}: - - template: /eng/common/templates-official/jobs/source-build.yml - parameters: - allCompletedJobId: Source_Build_Complete - ${{ each parameter in parameters.sourceBuildParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if eq(parameters.enableSourceIndex, 'true') }}: - - template: ../job/source-index-stage1.yml - parameters: - runAsPublic: ${{ parameters.runAsPublic }} - ${{ each parameter in parameters.sourceIndexParams }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - Source_Build_Complete +- template: /eng/common/core-templates/jobs/jobs.yml + parameters: + is1ESPipeline: true - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index 2076f4e25b4..483e7b611f3 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -1,46 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. A job is created for each platform, as - # well as an optional server job that completes when all platform jobs complete. - - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - - # See /eng/common/templates-official/job/source-build.yml - jobNamePrefix: 'Source_Build' - - # This is the default platform provided by Arcade, intended for use by a managed-only repo. - defaultManagedPlatform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' - - # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. If no platforms are specified, - # one job runs on 'defaultManagedPlatform'. - platforms: [] - jobs: +- template: /eng/common/core-templates/jobs/source-build.yml + parameters: + is1ESPipeline: true -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - -- ${{ each platform in parameters.platforms }}: - - template: /eng/common/templates-official/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ platform }} - -- ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/templates-official/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml index b9ede10bf09..c32fc49233f 100644 --- a/eng/common/templates-official/post-build/common-variables.yml +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -1,24 +1,8 @@ variables: - - group: Publish-Build-Assets +- template: /eng/common/core-templates/post-build/common-variables.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true - # Whether the build is internal or not - - name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro.dot.net" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - name: SymbolToolVersion - value: 1.0.1 - - name: BinlogToolVersion - value: 1.0.11 - - - name: runCodesignValidationInjection - value: false + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index da1f40958b4..2364c0fd4a5 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -1,285 +1,8 @@ -parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - - stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - - job: - displayName: NuGet Validation - condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) - - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - -- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: ${{ parameters.publishDependsOn }} - ${{ else }}: - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Publish using Darc - variables: - - template: common-variables.yml - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - - job: - displayName: Publish Using Darc - timeoutInMinutes: 120 - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: NetCore1ESPool-Publishing-Internal - image: windows.vs2019.amd64 - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: NuGetAuthenticate@1 +- template: /eng/common/core-templates/post-build/post-build.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml index 0c87f149a4a..024397d8786 100644 --- a/eng/common/templates-official/post-build/setup-maestro-vars.yml +++ b/eng/common/templates-official/post-build/setup-maestro-vars.yml @@ -1,70 +1,8 @@ -parameters: - BARBuildId: '' - PromoteToChannelIds: '' - steps: - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - targetType: inline - pwsh: true - script: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" +- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: true - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml index f67a210d62f..543dea8c696 100644 --- a/eng/common/templates-official/steps/add-build-to-channel.yml +++ b/eng/common/templates-official/steps/add-build-to-channel.yml @@ -1,13 +1,7 @@ -parameters: - ChannelId: 0 - steps: -- task: PowerShell@2 - displayName: Add Build to Channel - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 - arguments: -BuildId $(BARBuildId) - -ChannelId ${{ parameters.ChannelId }} - -MaestroApiAccessToken $(MaestroApiAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) +- template: /eng/common/core-templates/steps/add-build-to-channel.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml index 0ecec47b0c9..30bb3985ca2 100644 --- a/eng/common/templates-official/steps/component-governance.yml +++ b/eng/common/templates-official/steps/component-governance.yml @@ -1,13 +1,7 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file +- template: /eng/common/core-templates/steps/component-governance.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml index 488b560e8ba..9a89a4706d9 100644 --- a/eng/common/templates-official/steps/generate-sbom.yml +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -1,48 +1,7 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - -parameters: - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 -- script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }} - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- task: 1ES.PublishPipelineArtifact@1 - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - inputs: - targetPath: '${{parameters.manifestDirPath}}' - artifactName: $(ARTIFACT_NAME) +- template: /eng/common/core-templates/steps/generate-sbom.yml + parameters: + is1ESPipeline: true + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/publish-build-artifacts.yml b/eng/common/templates-official/steps/publish-build-artifacts.yml new file mode 100644 index 00000000000..100a3fc9849 --- /dev/null +++ b/eng/common/templates-official/steps/publish-build-artifacts.yml @@ -0,0 +1,41 @@ +parameters: +- name: displayName + type: string + default: 'Publish to Build Artifact' + +- name: condition + type: string + default: succeeded() + +- name: artifactName + type: string + +- name: pathToPublish + type: string + +- name: continueOnError + type: boolean + default: false + +- name: publishLocation + type: string + default: 'Container' + +- name: is1ESPipeline + type: boolean + default: true + +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates-official cannot be referenced from a non-1ES managed template': error +- task: 1ES.PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + condition: ${{ parameters.condition }} + ${{ if parameters.continueOnError }}: + continueOnError: ${{ parameters.continueOnError }} + inputs: + PublishLocation: ${{ parameters.publishLocation }} + PathtoPublish: ${{ parameters.pathToPublish }} + ${{ if parameters.artifactName }}: + ArtifactName: ${{ parameters.artifactName }} + diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml index 84b2f559c56..579fd531e94 100644 --- a/eng/common/templates-official/steps/publish-logs.yml +++ b/eng/common/templates-official/steps/publish-logs.yml @@ -1,49 +1,7 @@ -parameters: - StageLabel: '' - JobLabel: '' - CustomSensitiveDataList: '' - # A default - in case value from eng/common/templates-official/post-build/common-variables.yml is not passed - BinlogToolVersion: '1.0.11' - steps: -- task: Powershell@2 - displayName: Prepare Binlogs to Upload - inputs: - targetType: inline - script: | - New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - continueOnError: true - condition: always() - -- task: PowerShell@2 - displayName: Redact Logs - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 - # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml - # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - # If the file exists - sensitive data for redaction will be sourced from it - # (single entry per line, lines starting with '# ' are considered comments and skipped) - arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' - -BinlogToolVersion ${{parameters.BinlogToolVersion}} - -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - '$(publishing-dnceng-devdiv-code-r-build-re)' - '$(MaestroAccessToken)' - '$(dn-bot-all-orgs-artifact-feeds-rw)' - '$(akams-client-id)' - '$(akams-client-secret)' - '$(microsoft-symbol-server-pat)' - '$(symweb-symbol-server-pat)' - '$(dn-bot-all-orgs-build-rw-code-rw)' - ${{parameters.CustomSensitiveDataList}} - continueOnError: true - condition: always() - -- task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' - PublishLocation: Container - ArtifactName: PostBuildLogs - continueOnError: true - condition: always() +- template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/publish-pipeline-artifacts.yml b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml new file mode 100644 index 00000000000..15d74a9d916 --- /dev/null +++ b/eng/common/templates-official/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,26 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: true + +- name: args + type: object + default: {} + +steps: +- ${{ if ne(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates-official cannot be referenced from a non-1ES managed template': error +- task: 1ES.PublishPipelineArtifact@1 + displayName: ${{ coalesce(parameters.args.displayName, 'Publish to Build Artifact') }} + ${{ if parameters.args.condition }}: + condition: ${{ parameters.args.condition }} + ${{ else }}: + condition: succeeded() + ${{ if parameters.args.continueOnError }}: + continueOnError: ${{ parameters.args.continueOnError }} + inputs: + targetPath: ${{ parameters.args.targetPath }} + ${{ if parameters.args.artifactName }}: + artifactName: ${{ parameters.args.artifactName }} + ${{ if parameters.args.properties }}: + properties: ${{ properties.args.properties }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml index 83d97a26a01..5594551508a 100644 --- a/eng/common/templates-official/steps/retain-build.yml +++ b/eng/common/templates-official/steps/retain-build.yml @@ -1,28 +1,7 @@ -parameters: - # Optional azure devops PAT with build execute permissions for the build's organization, - # only needed if the build that should be retained ran on a different organization than - # the pipeline where this template is executing from - Token: '' - # Optional BuildId to retain, defaults to the current running build - BuildId: '' - # Azure devops Organization URI for the build in the https://dev.azure.com/ format. - # Defaults to the organization the current pipeline is running on - AzdoOrgUri: '$(System.CollectionUri)' - # Azure devops project for the build. Defaults to the project the current pipeline is running on - AzdoProject: '$(System.TeamProject)' - steps: - - task: powershell@2 - inputs: - targetType: 'filePath' - filePath: eng/common/retain-build.ps1 - pwsh: true - arguments: > - -AzdoOrgUri: ${{parameters.AzdoOrgUri}} - -AzdoProject ${{parameters.AzdoProject}} - -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} - -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} - displayName: Enable permanent build retention - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - BUILD_ID: $(Build.BuildId) \ No newline at end of file +- template: /eng/common/core-templates/steps/retain-build.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml index 68fa739c4ab..6500f21bf84 100644 --- a/eng/common/templates-official/steps/send-to-helix.yml +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -1,93 +1,7 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY - HelixProjectArguments: '' # optional -- arguments passed to the build command - HelixConfiguration: '' # optional -- additional property attached to a job - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} +- template: /eng/common/core-templates/steps/send-to-helix.yml + parameters: + is1ESPipeline: true + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml index 53ed57b6d48..8f92c49e7b0 100644 --- a/eng/common/templates-official/steps/source-build.yml +++ b/eng/common/templates-official/steps/source-build.yml @@ -1,131 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. - - # This is a 'steps' template, and is intended for advanced scenarios where the existing build - # infra has a careful build methodology that must be followed. For example, a repo - # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline - # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to - # GitHub. Using this steps template leaves room for that infra to be included. - - # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' - # for details. The entire object is described in the 'job' template for simplicity, even though - # the usage of the properties on this object is split between the 'job' and 'steps' templates. - platform: {} - steps: -# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) -- script: | - set -x - df -h - - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. - # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. - # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those - # changes. - internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then - # Temporarily work around https://github.com/dotnet/arcade/issues/7709 - chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) - internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. - # This only works if there is a username/email configured, which won't be the case in most CI runs. - git config --get user.email - if [ $? -ne 0 ]; then - git config user.email dn-bot@microsoft.com - git config user.name dn-bot - fi - fi - - # If building on the internal project, the internal storage variable may be available (usually only if needed) - # In that case, add variables to allow the download of internal runtimes if the specified versions are not found - # in the default public locations. - internalRuntimeDownloadArgs= - if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' - fi - - buildConfig=Release - # Check if AzDO substitutes in a build config from a variable, and use it if so. - if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then - buildConfig='$(_BuildConfig)' - fi - - officialBuildArgs= - if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then - officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' - fi - - targetRidArgs= - if [ '${{ parameters.platform.targetRID }}' != '' ]; then - targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' - fi - - runtimeOsArgs= - if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then - runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' - fi - - baseOsArgs= - if [ '${{ parameters.platform.baseOS }}' != '' ]; then - baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' - fi - - publishArgs= - if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then - publishArgs='--publish' - fi - - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml - fi - - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ - --configuration $buildConfig \ - --restore --build --pack $publishArgs -bl \ - $officialBuildArgs \ - $internalRuntimeDownloadArgs \ - $internalRestoreArgs \ - $targetRidArgs \ - $runtimeOsArgs \ - $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:DotNetBuildSourceOnly=true \ - /p:DotNetBuildRepo=true \ - /p:AssetManifestFileName=$assetManifestFileName - displayName: Build - -# Upload build logs for diagnosis. -- task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - artifacts/sb/prebuilt-report/** - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: succeededOrFailed() - -- task: 1ES.PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) - continueOnError: true - condition: succeededOrFailed() +- template: /eng/common/core-templates/steps/source-build.yml + parameters: + is1ESPipeline: true -# Manually inject component detection so that we can ignore the source build upstream cache, which contains -# a nupkg cache of input packages (a local feed). -# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' -# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets -- task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection (Exclude upstream cache) - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml deleted file mode 100644 index 7870f93bc17..00000000000 --- a/eng/common/templates/job/execute-sdl.yml +++ /dev/null @@ -1,139 +0,0 @@ -parameters: - enable: 'false' # Whether the SDL validation job should execute or not - overrideParameters: '' # Optional: to override values for parameters. - additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth - # diagnosis of problems with specific tool configurations. - publishGuardianDirectoryToPipeline: false - # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL - # parameters rather than relying on YAML. It may be better to use a local script, because you can - # reproduce results locally without piecing together a command based on the YAML. - executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' - # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named - # 'continueOnError', the parameter value is not correctly picked up. - # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter - sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; - # optional: determines if build artifacts should be downloaded. - downloadArtifacts: true - # optional: determines if this job should search the directory of downloaded artifacts for - # 'tar.gz' and 'zip' archive files and extract them before running SDL validation tasks. - extractArchiveArtifacts: false - dependsOn: '' # Optional: dependencies of the job - artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts - # Usage: - # artifactNames: - # - 'BlobArtifacts' - # - 'Artifacts_Windows_NT_Release' - # Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts, - # not pipeline artifacts, so doesn't affect the use of this parameter. - pipelineArtifactNames: [] - -jobs: -- job: Run_SDL - dependsOn: ${{ parameters.dependsOn }} - displayName: Run SDL tool - condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true')) - variables: - - group: DotNet-VSTS-Bot - - name: AzDOProjectName - value: ${{ parameters.AzDOProjectName }} - - name: AzDOPipelineId - value: ${{ parameters.AzDOPipelineId }} - - name: AzDOBuildId - value: ${{ parameters.AzDOBuildId }} - - template: /eng/common/templates/variables/sdl-variables.yml - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - template: /eng/common/templates/variables/pool-providers.yml - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - checkout: self - clean: true - - # If the template caller didn't provide an AzDO parameter, set them all up as Maestro vars. - - ${{ if not(and(parameters.AzDOProjectName, parameters.AzDOPipelineId, parameters.AzDOBuildId)) }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml - - - ${{ if ne(parameters.downloadArtifacts, 'false')}}: - - ${{ if ne(parameters.artifactNames, '') }}: - - ${{ each artifactName in parameters.artifactNames }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: ${{ artifactName }} - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - checkDownloadedFiles: true - - ${{ if eq(parameters.artifactNames, '') }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: specific files - itemPattern: "**" - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - checkDownloadedFiles: true - - - ${{ each artifactName in parameters.pipelineArtifactNames }}: - - task: DownloadPipelineArtifact@2 - displayName: Download Pipeline Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: ${{ artifactName }} - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - checkDownloadedFiles: true - - - powershell: eng/common/sdl/trim-assets-version.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts - displayName: Trim the version from the NuGet packages - continueOnError: ${{ parameters.sdlContinueOnError }} - - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts - displayName: Extract Blob Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts - displayName: Extract Package Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - - ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}: - - powershell: eng/common/sdl/extract-artifact-archives.ps1 - -InputPath $(Build.ArtifactStagingDirectory)\artifacts - -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts - displayName: Extract Archive Artifacts - continueOnError: ${{ parameters.sdlContinueOnError }} - - - template: /eng/common/templates/steps/execute-sdl.yml - parameters: - overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} - executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} - overrideParameters: ${{ parameters.overrideParameters }} - additionalParameters: ${{ parameters.additionalParameters }} - publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} - sdlContinueOnError: ${{ parameters.sdlContinueOnError }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index a3277bf15c5..bd0059384b0 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,259 +1,58 @@ -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -parameters: -# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - cancelTimeoutInMinutes: '' - condition: '' - container: '' - continueOnError: false - dependsOn: '' - displayName: '' - pool: '' - steps: [] - strategy: '' - timeoutInMinutes: '' - variables: [] - workspace: '' - templateContext: '' - -# Job base template specific parameters - # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md - artifacts: '' - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishBuildAssets: false - enablePublishTestResults: false - enablePublishUsingPipelines: false - enableBuildRetry: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' - mergeTestResults: false - testRunTitle: '' - testResultsFormat: '' - name: '' - preSteps: [] - runAsPublic: false -# Sbom related params - enableSbom: true - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - jobs: -- job: ${{ parameters.name }} - - ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.continueOnError, '') }}: - continueOnError: ${{ parameters.continueOnError }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - - ${{ if ne(parameters.timeoutInMinutes, '') }}: - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - ${{ if ne(parameters.templateContext, '') }}: - templateContext: ${{ parameters.templateContext }} - - variables: - - ${{ if ne(parameters.enableTelemetry, 'false') }}: - - name: DOTNET_CLI_TELEMETRY_PROFILE - value: '$(Build.Repository.Uri)' - - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - - name: EnableRichCodeNavigation - value: 'true' - # Retry signature validation up to three times, waiting 2 seconds between attempts. - # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY - value: 3,2000 - - ${{ each variable in parameters.variables }}: - # handle name-value variable syntax - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - # handle template variable syntax - # example: - # - template: path/to/template.yml - # parameters: - # [key]: [value] - - ${{ if ne(variable.template, '') }}: - - template: ${{ variable.template }} - ${{ if ne(variable.parameters, '') }}: - parameters: ${{ variable.parameters }} - - # handle key-value variable syntax. - # example: - # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - ${{ if ne(parameters.workspace, '') }}: - workspace: ${{ parameters.workspace }} - - steps: - - ${{ if ne(parameters.preSteps, '') }}: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@3 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - env: - TeamName: $(_TeamName) - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@1 - - - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - - task: DownloadPipelineArtifact@2 - inputs: - buildType: current - artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} - targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} - itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} - - - ${{ each step in parameters.steps }}: - - ${{ step }} - - - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - - task: RichCodeNavIndexer@0 - displayName: RichCodeNav Upload - inputs: - languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} - environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }} - richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} - continueOnError: true - - - template: /eng/common/templates/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: - TeamName: $(_TeamName) - - - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: 'artifacts/bin' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' - - task: CopyFiles@2 - displayName: Gather packages for publish to artifacts - inputs: - SourceFolder: 'artifacts/packages' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - task: PublishBuildArtifacts@1 - displayName: Publish pipeline artifacts - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - continueOnError: true - condition: always() - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - publish: artifacts/log - artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - displayName: Publish logs - continueOnError: true - condition: always() - - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} - continueOnError: true - condition: always() - - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - - task: PublishTestResults@2 - displayName: Publish XUnit Test Results - inputs: - testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: - - task: PublishTestResults@2 - displayName: Publish TRX Test Results - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion}} - BuildDropPath: ${{ parameters.buildDropPath }} - IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration - artifact: BuildConfiguration - displayName: Publish build retry configuration - continueOnError: true +- template: /eng/common/core-templates/job/job.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ if and(ne(parameter.key, 'steps'), ne(parameter.key, 'is1ESPipeline')) }}: + ${{ parameter.key }}: ${{ parameter.value }} + + steps: + - ${{ each step in parameters.steps }}: + - ${{ step }} + + artifactPublishSteps: + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish pipeline artifacts + pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + publishLocation: Container + artifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + targetPath: '$(Build.ArtifactStagingDirectory)/artifacts/log' + artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: 'Publish logs' + continueOnError: true + condition: always() + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: + - template: /eng/common/core-templates/steps/publish-build-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + displayName: Publish Logs + pathToPublish: '$(Build.ArtifactStagingDirectory)/artifacts/log/$(_BuildConfig)' + publishLocation: Container + artifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} + continueOnError: true + condition: always() + + - ${{ if eq(parameters.enableBuildRetry, 'true') }}: + - template: /eng/common/core-templates/steps/publish-pipeline-artifacts.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + args: + targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' + artifactName: 'BuildConfiguration' + displayName: 'Publish build retry configuration' + continueOnError: true diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 60ab00c4de3..ff829dc4c70 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -1,109 +1,7 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: '' - - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - SourcesDirectory: $(Build.SourcesDirectory) - CreatePr: true - AutoCompletePr: false - ReusePr: true - UseLfLineEndings: true - UseCheckedInLocProjectJson: false - SkipLocProjectJsonGeneration: false - LanguageSet: VS_Main_Languages - LclSource: lclFilesInRepo - LclPackageId: '' - RepoType: gitHub - GitHubOrg: dotnet - MirrorRepo: '' - MirrorBranch: main - condition: '' - JobNameSuffix: '' - jobs: -- job: OneLocBuild${{ parameters.JobNameSuffix }} - - dependsOn: ${{ parameters.dependsOn }} - - displayName: OneLocBuild${{ parameters.JobNameSuffix }} - - variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - - name: _GenerateLocProjectArguments - value: -SourcesDirectory ${{ parameters.SourcesDirectory }} - -LanguageSet "${{ parameters.LanguageSet }}" - -CreateNeutralXlfs - - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: - - name: _GenerateLocProjectArguments - value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 - arguments: $(_GenerateLocProjectArguments) - displayName: Generate LocProject.json - condition: ${{ parameters.condition }} - - - task: OneLocBuild@2 - displayName: OneLocBuild - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - locProj: eng/Localize/LocProject.json - outDir: $(Build.ArtifactStagingDirectory) - lclSource: ${{ parameters.LclSource }} - lclPackageId: ${{ parameters.LclPackageId }} - isCreatePrSelected: ${{ parameters.CreatePr }} - isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} - ${{ if eq(parameters.CreatePr, true) }}: - isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} - packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" - ${{ if ne(parameters.MirrorRepo, '') }}: - isMirrorRepoSelected: true - gitHubOrganization: ${{ parameters.GitHubOrg }} - mirrorRepo: ${{ parameters.MirrorRepo }} - mirrorBranch: ${{ parameters.MirrorBranch }} - condition: ${{ parameters.condition }} - - - task: PublishBuildArtifacts@1 - displayName: Publish Localization Files - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} +- template: /eng/common/core-templates/job/onelocbuild.yml + parameters: + is1ESPipeline: false - - task: PublishBuildArtifacts@1 - displayName: Publish LocProject.json - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index bb42240f865..ab2edec2adb 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -1,155 +1,7 @@ -parameters: - configuration: 'Debug' - - # Optional: condition for the job to run - condition: '' - - # Optional: 'true' if future jobs should run even if this job fails - continueOnError: false - - # Optional: dependencies of the job - dependsOn: '' - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishUsingPipelines: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishAssetsImmediately: false - - artifactsPublishingAdditionalParameters: '' - - signingValidationAdditionalParameters: '' - jobs: -- job: Asset_Registry_Publish - - dependsOn: ${{ parameters.dependsOn }} - timeoutInMinutes: 150 - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - displayName: Publish Assets - ${{ else }}: - displayName: Publish to Build Asset Registry - - variables: - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - - name: runCodesignValidationInjection - value: false - # unconditional - needed for logs publishing (redactor tool version) - - template: /eng/common/templates/post-build/common-variables.yml - - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - checkout: self - fetchDepth: 3 - clean: true - - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: NuGetAuthenticate@1 - - - task: PowerShell@2 - displayName: Publish Build Assets - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro.dot.net - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:OfficialBuildId=$(Build.BuildNumber) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: powershell@2 - displayName: Create ReleaseConfigs Artifact - inputs: - targetType: inline - script: | - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - - - task: PublishBuildArtifacts@1 - displayName: Publish ReleaseConfigs Artifact - inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - task: powershell@2 - displayName: Check if SymbolPublishingExclusionsFile.txt exists - inputs: - targetType: inline - script: | - $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" - if(Test-Path -Path $symbolExclusionfile) - { - Write-Host "SymbolExclusionFile exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" - } - else{ - Write-Host "Symbols Exclusion file does not exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" - } - - - task: PublishBuildArtifacts@1 - displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' +- template: /eng/common/core-templates/job/publish-build-assets.yml + parameters: + is1ESPipeline: false - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - template: /eng/common/templates/steps/publish-logs.yml - parameters: - JobLabel: 'Publish_Artifacts_Logs' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index d7ed209494c..e44d47b1d76 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -1,66 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. The template produces a server job with a - # default ID 'Source_Build_Complete' to put in a dependency list if necessary. - - # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. - jobNamePrefix: 'Source_Build' - - # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for - # managed-only repositories. This is an object with these properties: - # - # name: '' - # The name of the job. This is included in the job ID. - # targetRID: '' - # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false - # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than - # linux-x64), and compiling against distro-provided packages rather than portable ones. - # skipPublishValidation: false - # Disables publishing validation. By default, a check is performed to ensure no packages are - # published by source-build. - # container: '' - # A container to use. Runs in docker. - # pool: {} - # A pool to use. Runs directly on an agent. - # buildScript: '' - # Specifies the build script to invoke to perform the build in the repo. The default - # './build.sh' should work for typical Arcade repositories, but this is customizable for - # difficult situations. - # jobProperties: {} - # A list of job properties to inject at the top level, for potential extensibility beyond - # container and pool. - platform: {} - jobs: -- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} - displayName: Source-Build (${{ parameters.platform.name }}) - - ${{ each property in parameters.platform.jobProperties }}: - ${{ property.key }}: ${{ property.value }} - - ${{ if ne(parameters.platform.container, '') }}: - container: ${{ parameters.platform.container }} - - ${{ if eq(parameters.platform.pool, '') }}: - # The default VM host AzDO pool. This should be capable of running Docker containers: almost all - # source-build builds run in Docker, including the default managed platform. - # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 - - ${{ if ne(parameters.platform.pool, '') }}: - pool: ${{ parameters.platform.pool }} - - workspace: - clean: all +- template: /eng/common/core-templates/job/source-build.yml + parameters: + is1ESPipeline: false - steps: - - template: /eng/common/templates/steps/source-build.yml - parameters: - platform: ${{ parameters.platform }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b5a3e5c4a6c..89f3291593c 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,67 +1,7 @@ -parameters: - runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240129.2 - sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json - sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" - preSteps: [] - binlogPath: artifacts/log/Debug/Build.binlog - condition: '' - dependsOn: '' - pool: '' - jobs: -- job: SourceIndexStage1 - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} - - name: SourceIndexPackageSource - value: ${{ parameters.sourceIndexPackageSource }} - - name: BinlogPath - value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - - template: /eng/common/templates/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 - - steps: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - task: UseDotNet@2 - displayName: Use .NET 8 SDK - inputs: - packageType: sdk - version: 8.0.x - installationPath: $(Agent.TempDirectory)/dotnet - workingDirectory: $(Agent.TempDirectory) - - - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: Download Tools - # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. - workingDirectory: $(Agent.TempDirectory) - - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository - - - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: Process Binlog into indexable sln +- template: /eng/common/core-templates/job/source-index-stage1.yml + parameters: + is1ESPipeline: false - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/jobs/codeql-build.yml b/eng/common/templates/jobs/codeql-build.yml index f7dc5ea4aaa..517f24d6a52 100644 --- a/eng/common/templates/jobs/codeql-build.yml +++ b/eng/common/templates/jobs/codeql-build.yml @@ -1,31 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - jobs: -- template: /eng/common/templates/jobs/jobs.yml +- template: /eng/common/core-templates/jobs/codeql-build.yml parameters: - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishTestResults: false - enablePublishBuildAssets: false - enablePublishUsingPipelines: false - enableTelemetry: true + is1ESPipeline: false - variables: - - group: Publish-Build-Assets - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.109.0 - - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - jobs: ${{ parameters.jobs }} - + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 289bb2396ce..388e9037b3e 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -1,97 +1,7 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: Enable publishing using release pipelines - enablePublishUsingPipelines: false - - # Optional: Enable running the source-build jobs to build repo from source - enableSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates/jobs/source-build.yml for options - sourceBuildParameters: [] - - graphFileGeneration: - # Optional: Enable generating the graph files at the end of the build - enabled: false - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - - # Optional: Override automatically derived dependsOn value for "publish build assets" job - publishBuildAssetsDependsOn: '' - - # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. - publishAssetsImmediately: false - - # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) - artifactsPublishingAdditionalParameters: '' - signingValidationAdditionalParameters: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - enableSourceIndex: false - sourceIndexParams: {} - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - jobs: -- ${{ each job in parameters.jobs }}: - - template: ../job/job.yml - parameters: - # pass along parameters - ${{ each parameter in parameters }}: - ${{ if ne(parameter.key, 'jobs') }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # pass along job properties - ${{ each property in job }}: - ${{ if ne(property.key, 'job') }}: - ${{ property.key }}: ${{ property.value }} - - name: ${{ job.job }} - -- ${{ if eq(parameters.enableSourceBuild, true) }}: - - template: /eng/common/templates/jobs/source-build.yml - parameters: - allCompletedJobId: Source_Build_Complete - ${{ each parameter in parameters.sourceBuildParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if eq(parameters.enableSourceIndex, 'true') }}: - - template: ../job/source-index-stage1.yml - parameters: - runAsPublic: ${{ parameters.runAsPublic }} - ${{ each parameter in parameters.sourceIndexParams }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - Source_Build_Complete +- template: /eng/common/core-templates/jobs/jobs.yml + parameters: + is1ESPipeline: false - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index da91481ff1d..818d4c326db 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -1,46 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. A job is created for each platform, as - # well as an optional server job that completes when all platform jobs complete. - - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - - # See /eng/common/templates/job/source-build.yml - jobNamePrefix: 'Source_Build' - - # This is the default platform provided by Arcade, intended for use by a managed-only repo. - defaultManagedPlatform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9' - - # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. If no platforms are specified, - # one job runs on 'defaultManagedPlatform'. - platforms: [] - jobs: +- template: /eng/common/core-templates/jobs/source-build.yml + parameters: + is1ESPipeline: false -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - -- ${{ each platform in parameters.platforms }}: - - template: /eng/common/templates/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ platform }} - -- ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/templates/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index b9ede10bf09..7fa10587559 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,24 +1,8 @@ variables: - - group: Publish-Build-Assets +- template: /eng/common/core-templates/post-build/common-variables.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false - # Whether the build is internal or not - - name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro.dot.net" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - name: SymbolToolVersion - value: 1.0.1 - - name: BinlogToolVersion - value: 1.0.11 - - - name: runCodesignValidationInjection - value: false + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index ee70e2b399c..53ede714bdd 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,282 +1,8 @@ -parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - - stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - - template: /eng/common/templates/variables/pool-providers.yml - jobs: - - job: - displayName: NuGet Validation - condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) - - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - enable: ${{ parameters.SDLValidationParameters.enable }} - publishGuardianDirectoryToPipeline: ${{ parameters.SDLValidationParameters.publishGdn }} - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - -- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: ${{ parameters.publishDependsOn }} - ${{ else }}: - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Publish using Darc - variables: - - template: common-variables.yml - - template: /eng/common/templates/variables/pool-providers.yml - jobs: - - job: - displayName: Publish Using Darc - timeoutInMinutes: 120 - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ else }}: - name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2019.amd64 - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: NuGetAuthenticate@1 +- template: /eng/common/core-templates/post-build/post-build.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 0c87f149a4a..a79fab5b441 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -1,70 +1,8 @@ -parameters: - BARBuildId: '' - PromoteToChannelIds: '' - steps: - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - targetType: inline - pwsh: true - script: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" +- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + # Specifies whether to use 1ES + is1ESPipeline: false - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/common/templates/steps/add-build-to-channel.yml b/eng/common/templates/steps/add-build-to-channel.yml index f67a210d62f..42bbba161b9 100644 --- a/eng/common/templates/steps/add-build-to-channel.yml +++ b/eng/common/templates/steps/add-build-to-channel.yml @@ -1,13 +1,7 @@ -parameters: - ChannelId: 0 - steps: -- task: PowerShell@2 - displayName: Add Build to Channel - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 - arguments: -BuildId $(BARBuildId) - -ChannelId ${{ parameters.ChannelId }} - -MaestroApiAccessToken $(MaestroApiAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) +- template: /eng/common/core-templates/steps/add-build-to-channel.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/build-reason.yml b/eng/common/templates/steps/build-reason.yml deleted file mode 100644 index eba58109b52..00000000000 --- a/eng/common/templates/steps/build-reason.yml +++ /dev/null @@ -1,12 +0,0 @@ -# build-reason.yml -# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons -# to include steps (',' separated). -parameters: - conditions: '' - steps: [] - -steps: - - ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}: - - ${{ parameters.steps }} - - ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}: - - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/component-governance.yml b/eng/common/templates/steps/component-governance.yml index 0ecec47b0c9..c12a5f8d21d 100644 --- a/eng/common/templates/steps/component-governance.yml +++ b/eng/common/templates/steps/component-governance.yml @@ -1,13 +1,7 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file +- template: /eng/common/core-templates/steps/component-governance.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/execute-codeql.yml b/eng/common/templates/steps/execute-codeql.yml deleted file mode 100644 index 3930b163021..00000000000 --- a/eng/common/templates/steps/execute-codeql.yml +++ /dev/null @@ -1,32 +0,0 @@ -parameters: - # Language that should be analyzed. Defaults to csharp - language: csharp - # Build Commands - buildCommands: '' - overrideParameters: '' # Optional: to override values for parameters. - additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - # Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth - # diagnosis of problems with specific tool configurations. - publishGuardianDirectoryToPipeline: false - # The script to run to execute all SDL tools. Use this if you want to use a script to define SDL - # parameters rather than relying on YAML. It may be better to use a local script, because you can - # reproduce results locally without piecing together a command based on the YAML. - executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1' - # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named - # 'continueOnError', the parameter value is not correctly picked up. - # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter - # optional: determines whether to continue the build if the step errors; - sdlContinueOnError: false - -steps: -- template: /eng/common/templates/steps/execute-sdl.yml - parameters: - overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} - executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} - overrideParameters: ${{ parameters.overrideParameters }} - additionalParameters: '${{ parameters.additionalParameters }} - -CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")' - publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} - sdlContinueOnError: ${{ parameters.sdlContinueOnError }} \ No newline at end of file diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml deleted file mode 100644 index 07426fde05d..00000000000 --- a/eng/common/templates/steps/execute-sdl.yml +++ /dev/null @@ -1,88 +0,0 @@ -parameters: - overrideGuardianVersion: '' - executeAllSdlToolsScript: '' - overrideParameters: '' - additionalParameters: '' - publishGuardianDirectoryToPipeline: false - sdlContinueOnError: false - condition: '' - -steps: -- task: NuGetAuthenticate@1 - inputs: - nuGetServiceConnections: GuardianConnect - -- task: NuGetToolInstaller@1 - displayName: 'Install NuGet.exe' - -- ${{ if ne(parameters.overrideGuardianVersion, '') }}: - - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 - $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }} - Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" - displayName: Install Guardian (Overridden) - -- ${{ if eq(parameters.overrideGuardianVersion, '') }}: - - pwsh: | - Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl - . .\sdl.ps1 - $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts - Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation" - displayName: Install Guardian - -- ${{ if ne(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} - displayName: Execute SDL (Overridden) - continueOnError: ${{ parameters.sdlContinueOnError }} - condition: ${{ parameters.condition }} - -- ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} - -GuardianCliLocation $(GuardianCliLocation) - -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) - ${{ parameters.additionalParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.sdlContinueOnError }} - condition: ${{ parameters.condition }} - -- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: - # We want to publish the Guardian results and configuration for easy diagnosis. However, the - # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default - # tooling files. Some of these files are large and aren't useful during an investigation, so - # exclude them by simply deleting them before publishing. (As of writing, there is no documented - # way to selectively exclude a dir from the pipeline artifact publish task.) - - task: DeleteFiles@1 - displayName: Delete Guardian dependencies to avoid uploading - inputs: - SourceFolder: $(Agent.BuildDirectory)/.gdn - Contents: | - c - i - condition: succeededOrFailed() - - - publish: $(Agent.BuildDirectory)/.gdn - artifact: GuardianConfiguration - displayName: Publish GuardianConfiguration - condition: succeededOrFailed() - - # Publish the SARIF files in a container named CodeAnalysisLogs to enable integration - # with the "SARIF SAST Scans Tab" Azure DevOps extension - - task: CopyFiles@2 - displayName: Copy SARIF files - inputs: - flattenFolders: true - sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/ - contents: '**/*.sarif' - targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs - condition: succeededOrFailed() - - # Use PublishBuildArtifacts because the SARIF extension only checks this case - # see microsoft/sarif-azuredevops-extension#4 - - task: PublishBuildArtifacts@1 - displayName: Publish SARIF files to CodeAnalysisLogs container - inputs: - pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs - artifactName: CodeAnalysisLogs - condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates/steps/generate-sbom.yml b/eng/common/templates/steps/generate-sbom.yml index a06373f38fa..26dc00a2e0f 100644 --- a/eng/common/templates/steps/generate-sbom.yml +++ b/eng/common/templates/steps/generate-sbom.yml @@ -1,48 +1,7 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - -parameters: - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 -- script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }} - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- task: PublishPipelineArtifact@1 - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - inputs: - targetPath: '${{parameters.manifestDirPath}}' - artifactName: $(ARTIFACT_NAME) +- template: /eng/common/core-templates/steps/generate-sbom.yml + parameters: + is1ESPipeline: false + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/publish-build-artifacts.yml b/eng/common/templates/steps/publish-build-artifacts.yml new file mode 100644 index 00000000000..6428a98dfef --- /dev/null +++ b/eng/common/templates/steps/publish-build-artifacts.yml @@ -0,0 +1,40 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false + +- name: displayName + type: string + default: 'Publish to Build Artifact' + +- name: condition + type: string + default: succeeded() + +- name: artifactName + type: string + +- name: pathToPublish + type: string + +- name: continueOnError + type: boolean + default: false + +- name: publishLocation + type: string + default: 'Container' + +steps: +- ${{ if eq(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates cannot be referenced from a 1ES managed template': error +- task: PublishBuildArtifacts@1 + displayName: ${{ parameters.displayName }} + condition: ${{ parameters.condition }} + ${{ if parameters.continueOnError }}: + continueOnError: ${{ parameters.continueOnError }} + inputs: + PublishLocation: ${{ parameters.publishLocation }} + PathtoPublish: ${{ parameters.pathToPublish }} + ${{ if parameters.artifactName }}: + ArtifactName: ${{ parameters.artifactName }} \ No newline at end of file diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index 80861297ddc..4ea86bd8823 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -1,49 +1,7 @@ -parameters: - StageLabel: '' - JobLabel: '' - CustomSensitiveDataList: '' - # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed - BinlogToolVersion: '1.0.11' - steps: -- task: Powershell@2 - displayName: Prepare Binlogs to Upload - inputs: - targetType: inline - script: | - New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - continueOnError: true - condition: always() - -- task: PowerShell@2 - displayName: Redact Logs - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 - # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml - # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - # If the file exists - sensitive data for redaction will be sourced from it - # (single entry per line, lines starting with '# ' are considered comments and skipped) - arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' - -BinlogToolVersion ${{parameters.BinlogToolVersion}} - -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - '$(publishing-dnceng-devdiv-code-r-build-re)' - '$(MaestroAccessToken)' - '$(dn-bot-all-orgs-artifact-feeds-rw)' - '$(akams-client-id)' - '$(akams-client-secret)' - '$(microsoft-symbol-server-pat)' - '$(symweb-symbol-server-pat)' - '$(dn-bot-all-orgs-build-rw-code-rw)' - ${{parameters.CustomSensitiveDataList}} - continueOnError: true - condition: always() - -- task: PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' - PublishLocation: Container - ArtifactName: PostBuildLogs - continueOnError: true - condition: always() +- template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/publish-pipeline-artifacts.yml b/eng/common/templates/steps/publish-pipeline-artifacts.yml new file mode 100644 index 00000000000..88c56354128 --- /dev/null +++ b/eng/common/templates/steps/publish-pipeline-artifacts.yml @@ -0,0 +1,34 @@ +parameters: +- name: is1ESPipeline + type: boolean + default: false + +- name: args + type: object + default: {} + +steps: +- ${{ if eq(parameters.is1ESPipeline, true) }}: + - 'eng/common/templates cannot be referenced from a 1ES managed template': error +- task: PublishPipelineArtifact@1 + displayName: ${{ coalesce(parameters.args.displayName, 'Publish to Build Artifact') }} + ${{ if parameters.args.condition }}: + condition: ${{ parameters.args.condition }} + ${{ else }}: + condition: succeeded() + ${{ if parameters.args.continueOnError }}: + continueOnError: ${{ parameters.args.continueOnError }} + inputs: + TargetPath: ${{ parameters.args.TargetPath }} + ${{ if parameters.args.ArtifactName }}: + ArtifactName: ${{ parameters.args.ArtifactName }} + ${{ if parameters.args.PublishLocation }}: + PublishLocation: ${{ parameters.args.PublishLocation }} + ${{ if parameters.args.FileSharePath }}: + FileSharePath: ${{ parameters.args.FileSharePath }} + ${{ if parameters.args.Parallel }}: + Parallel: ${{ parameters.args.Parallel }} + ${{ if parameters.args.ParallelCount }}: + ParallelCount: ${{ parameters.args.ParallelCount }} + ${{ if parameters.args.Properties }}: + Properties: ${{ properties.args.Properties }} \ No newline at end of file diff --git a/eng/common/templates/steps/retain-build.yml b/eng/common/templates/steps/retain-build.yml index 83d97a26a01..8e841ace3d2 100644 --- a/eng/common/templates/steps/retain-build.yml +++ b/eng/common/templates/steps/retain-build.yml @@ -1,28 +1,7 @@ -parameters: - # Optional azure devops PAT with build execute permissions for the build's organization, - # only needed if the build that should be retained ran on a different organization than - # the pipeline where this template is executing from - Token: '' - # Optional BuildId to retain, defaults to the current running build - BuildId: '' - # Azure devops Organization URI for the build in the https://dev.azure.com/ format. - # Defaults to the organization the current pipeline is running on - AzdoOrgUri: '$(System.CollectionUri)' - # Azure devops project for the build. Defaults to the project the current pipeline is running on - AzdoProject: '$(System.TeamProject)' - steps: - - task: powershell@2 - inputs: - targetType: 'filePath' - filePath: eng/common/retain-build.ps1 - pwsh: true - arguments: > - -AzdoOrgUri: ${{parameters.AzdoOrgUri}} - -AzdoProject ${{parameters.AzdoProject}} - -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} - -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} - displayName: Enable permanent build retention - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - BUILD_ID: $(Build.BuildId) \ No newline at end of file +- template: /eng/common/core-templates/steps/retain-build.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/run-on-unix.yml b/eng/common/templates/steps/run-on-unix.yml deleted file mode 100644 index e1733814f65..00000000000 --- a/eng/common/templates/steps/run-on-unix.yml +++ /dev/null @@ -1,7 +0,0 @@ -parameters: - agentOs: '' - steps: [] - -steps: -- ${{ if ne(parameters.agentOs, 'Windows_NT') }}: - - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/run-on-windows.yml b/eng/common/templates/steps/run-on-windows.yml deleted file mode 100644 index 73e7e9c275a..00000000000 --- a/eng/common/templates/steps/run-on-windows.yml +++ /dev/null @@ -1,7 +0,0 @@ -parameters: - agentOs: '' - steps: [] - -steps: -- ${{ if eq(parameters.agentOs, 'Windows_NT') }}: - - ${{ parameters.steps }} diff --git a/eng/common/templates/steps/run-script-ifequalelse.yml b/eng/common/templates/steps/run-script-ifequalelse.yml deleted file mode 100644 index 3d1242f5587..00000000000 --- a/eng/common/templates/steps/run-script-ifequalelse.yml +++ /dev/null @@ -1,33 +0,0 @@ -parameters: - # if parameter1 equals parameter 2, run 'ifScript' command, else run 'elsescript' command - parameter1: '' - parameter2: '' - ifScript: '' - elseScript: '' - - # name of script step - name: Script - - # display name of script step - displayName: If-Equal-Else Script - - # environment - env: {} - - # conditional expression for step execution - condition: '' - -steps: -- ${{ if and(ne(parameters.ifScript, ''), eq(parameters.parameter1, parameters.parameter2)) }}: - - script: ${{ parameters.ifScript }} - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - env: ${{ parameters.env }} - condition: ${{ parameters.condition }} - -- ${{ if and(ne(parameters.elseScript, ''), ne(parameters.parameter1, parameters.parameter2)) }}: - - script: ${{ parameters.elseScript }} - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - env: ${{ parameters.env }} - condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 68fa739c4ab..39f99fc2762 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -1,93 +1,7 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY - HelixProjectArguments: '' # optional -- arguments passed to the build command - HelixConfiguration: '' # optional -- additional property attached to a job - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} +- template: /eng/common/core-templates/steps/send-to-helix.yml + parameters: + is1ESPipeline: false + + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 32738aa9380..23c1d6f4e9f 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -1,131 +1,7 @@ -parameters: - # This template adds arcade-powered source-build to CI. - - # This is a 'steps' template, and is intended for advanced scenarios where the existing build - # infra has a careful build methodology that must be followed. For example, a repo - # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline - # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to - # GitHub. Using this steps template leaves room for that infra to be included. - - # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml' - # for details. The entire object is described in the 'job' template for simplicity, even though - # the usage of the properties on this object is split between the 'job' and 'steps' templates. - platform: {} - steps: -# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) -- script: | - set -x - df -h - - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. - # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. - # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those - # changes. - internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then - # Temporarily work around https://github.com/dotnet/arcade/issues/7709 - chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) - internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. - # This only works if there is a username/email configured, which won't be the case in most CI runs. - git config --get user.email - if [ $? -ne 0 ]; then - git config user.email dn-bot@microsoft.com - git config user.name dn-bot - fi - fi - - # If building on the internal project, the internal storage variable may be available (usually only if needed) - # In that case, add variables to allow the download of internal runtimes if the specified versions are not found - # in the default public locations. - internalRuntimeDownloadArgs= - if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' - fi - - buildConfig=Release - # Check if AzDO substitutes in a build config from a variable, and use it if so. - if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then - buildConfig='$(_BuildConfig)' - fi - - officialBuildArgs= - if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then - officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' - fi - - targetRidArgs= - if [ '${{ parameters.platform.targetRID }}' != '' ]; then - targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' - fi - - runtimeOsArgs= - if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then - runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' - fi - - baseOsArgs= - if [ '${{ parameters.platform.baseOS }}' != '' ]; then - baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' - fi - - publishArgs= - if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then - publishArgs='--publish' - fi - - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml - fi - - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ - --configuration $buildConfig \ - --restore --build --pack $publishArgs -bl \ - $officialBuildArgs \ - $internalRuntimeDownloadArgs \ - $internalRestoreArgs \ - $targetRidArgs \ - $runtimeOsArgs \ - $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:DotNetBuildSourceOnly=true \ - /p:DotNetBuildRepo=true \ - /p:AssetManifestFileName=$assetManifestFileName - displayName: Build - -# Upload build logs for diagnosis. -- task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - artifacts/sb/prebuilt-report/** - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: succeededOrFailed() - -- task: PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) - continueOnError: true - condition: succeededOrFailed() +- template: /eng/common/core-templates/steps/source-build.yml + parameters: + is1ESPipeline: false -# Manually inject component detection so that we can ignore the source build upstream cache, which contains -# a nupkg cache of input packages (a local feed). -# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' -# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets -- task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection (Exclude upstream cache) - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/steps/telemetry-end.yml b/eng/common/templates/steps/telemetry-end.yml deleted file mode 100644 index fadc04ca1b9..00000000000 --- a/eng/common/templates/steps/telemetry-end.yml +++ /dev/null @@ -1,102 +0,0 @@ -parameters: - maxRetries: 5 - retryDelay: 10 # in seconds - -steps: -- bash: | - if [ "$AGENT_JOBSTATUS" = "Succeeded" ] || [ "$AGENT_JOBSTATUS" = "PartiallySucceeded" ]; then - errorCount=0 - else - errorCount=1 - fi - warningCount=0 - - curlStatus=1 - retryCount=0 - # retry loop to harden against spotty telemetry connections - # we don't retry successes and 4xx client errors - until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]] - do - if [ $retryCount -gt 0 ]; then - echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..." - sleep $RetryDelay - fi - - # create a temporary file for curl output - res=`mktemp` - - curlResult=` - curl --verbose --output $res --write-out "%{http_code}"\ - -H 'Content-Type: application/json' \ - -H "X-Helix-Job-Token: $Helix_JobToken" \ - -H 'Content-Length: 0' \ - -X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \ - --data-urlencode "errorCount=$errorCount" \ - --data-urlencode "warningCount=$warningCount"` - curlStatus=$? - - if [ $curlStatus -eq 0 ]; then - if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then - curlStatus=$curlResult - fi - fi - - let retryCount++ - done - - if [ $curlStatus -ne 0 ]; then - echo "Failed to Send Build Finish information after $retryCount retries" - vstsLogOutput="vso[task.logissue type=error;sourcepath=templates/steps/telemetry-end.yml;code=1;]Failed to Send Build Finish information: $curlStatus" - echo "##$vstsLogOutput" - exit 1 - fi - displayName: Send Unix Build End Telemetry - env: - # defined via VSTS variables in start-job.sh - Helix_JobToken: $(Helix_JobToken) - Helix_WorkItemId: $(Helix_WorkItemId) - MaxRetries: ${{ parameters.maxRetries }} - RetryDelay: ${{ parameters.retryDelay }} - condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT')) -- powershell: | - if (($env:Agent_JobStatus -eq 'Succeeded') -or ($env:Agent_JobStatus -eq 'PartiallySucceeded')) { - $ErrorCount = 0 - } else { - $ErrorCount = 1 - } - $WarningCount = 0 - - # Basic retry loop to harden against server flakiness - $retryCount = 0 - while ($retryCount -lt $env:MaxRetries) { - try { - Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" ` - -Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken } - break - } - catch { - $statusCode = $_.Exception.Response.StatusCode.value__ - if ($statusCode -ge 400 -and $statusCode -le 499) { - Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)" - Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message - exit 1 - } - Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..." - $retryCount++ - sleep $env:RetryDelay - continue - } - } - - if ($retryCount -ge $env:MaxRetries) { - Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries." - exit 1 - } - displayName: Send Windows Build End Telemetry - env: - # defined via VSTS variables in start-job.ps1 - Helix_JobToken: $(Helix_JobToken) - Helix_WorkItemId: $(Helix_WorkItemId) - MaxRetries: ${{ parameters.maxRetries }} - RetryDelay: ${{ parameters.retryDelay }} - condition: and(always(),eq(variables['Agent.Os'], 'Windows_NT')) diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml deleted file mode 100644 index 32c01ef0b55..00000000000 --- a/eng/common/templates/steps/telemetry-start.yml +++ /dev/null @@ -1,241 +0,0 @@ -parameters: - helixSource: 'undefined_defaulted_in_telemetry.yml' - helixType: 'undefined_defaulted_in_telemetry.yml' - buildConfig: '' - runAsPublic: false - maxRetries: 5 - retryDelay: 10 # in seconds - -steps: -- ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}: - - task: AzureKeyVault@1 - inputs: - azureSubscription: 'HelixProd_KeyVault' - KeyVaultName: HelixProdKV - SecretsFilter: 'HelixApiAccessToken' - condition: always() -- bash: | - # create a temporary file - jobInfo=`mktemp` - - # write job info content to temporary file - cat > $jobInfo < + /// Reads an array of primitives. + /// + public static unsafe T[] ReadPrimitiveArray(this BinaryReader reader, int count) + where T : unmanaged + { + ArgumentOutOfRangeException.ThrowIfNegative(count); + if (typeof(T) != typeof(bool) + && typeof(T) != typeof(byte) + && typeof(T) != typeof(sbyte) + && typeof(T) != typeof(char) + && typeof(T) != typeof(short) + && typeof(T) != typeof(ushort) + && typeof(T) != typeof(int) + && typeof(T) != typeof(uint) + && typeof(T) != typeof(long) + && typeof(T) != typeof(ulong) + && typeof(T) != typeof(float) + && typeof(T) != typeof(double)) + { + throw new ArgumentException($"Cannot read primitives of {typeof(T).Name}.", nameof(T)); + } + + if (count > 0 && reader.Remaining() < count * (typeof(T) == typeof(char) ? 1 : sizeof(T))) + { + throw new SerializationException("Not enough data to fill array."); + } + + if (count == 0) + { + return []; + } + + if (typeof(T) == typeof(char)) + { + // Need to handle different encodings + return (T[])(object)reader.ReadChars(count); + } + + T[] array = new T[count]; + + fixed (T* a = array) + { + Span arrayData = new(a, array.Length * sizeof(T)); + + if (reader.Read(arrayData) != arrayData.Length) + { + throw new SerializationException("Not enough data to fill array."); + } + + if (sizeof(T) != 1 && !BitConverter.IsLittleEndian) + { + if (sizeof(T) == 2) + { + Span ushorts = MemoryMarshal.Cast(arrayData); + BinaryPrimitives.ReverseEndianness(ushorts, ushorts); + } + else if (sizeof(T) == 4) + { + Span ints = MemoryMarshal.Cast(arrayData); + BinaryPrimitives.ReverseEndianness(ints, ints); + } + else if (sizeof(T) == 8) + { + Span longs = MemoryMarshal.Cast(arrayData); + BinaryPrimitives.ReverseEndianness(longs, longs); + } + else + { + throw new InvalidOperationException($"Cannot read primitives of {typeof(T).Name}."); + } + } + } + + return array; + } } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArrayRecordExtensions.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArrayRecordExtensions.cs new file mode 100644 index 00000000000..67132012e83 --- /dev/null +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArrayRecordExtensions.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.Serialization; + +namespace System.Windows.Forms.BinaryFormat; + +internal static class ArrayRecordExtensions +{ + internal static Array GetPrimitiveArray(this IPrimitiveTypeRecord primitiveArray) => primitiveArray.PrimitiveType switch + { + PrimitiveType.Boolean => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.Byte => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.Char => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.Decimal => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.Double => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.Int16 => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.Int32 => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.Int64 => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.SByte => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.Single => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.TimeSpan => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.DateTime => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.UInt16 => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.UInt32 => ((ArrayRecord)primitiveArray).AsArray(), + PrimitiveType.UInt64 => ((ArrayRecord)primitiveArray).AsArray(), + _ => throw new SerializationException($"Unexpected primitive array type: '{primitiveArray.PrimitiveType}'") + }; + + /// + /// Convert the source to an array, if it is not already an array. + /// + private static Array AsArray(this ArrayRecord record) where T : unmanaged + { + if (record.ArrayObjects is not T[] rawArray) + { + Debug.Fail("Should not have any primitive arrays that are not already arrays."); + throw new InvalidOperationException(); + } + + if (record is not IBinaryArray binaryArray || binaryArray.ArrayType is BinaryArrayType.Single or BinaryArrayType.Jagged) + { + return rawArray; + } + + if (binaryArray.ArrayType is not BinaryArrayType.Rectangular) + { + // This should not be possible. + throw new NotSupportedException(); + } + + Array array = Array.CreateInstance(typeof(T), binaryArray.Lengths.ToArray()); + Span flatSpan = array.GetArrayData(); + rawArray.AsSpan().CopyTo(flatSpan); + return array; + } +} diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySingleObject.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySingleObject.cs index 00580054237..025ce68aff0 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySingleObject.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySingleObject.cs @@ -13,23 +13,25 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class ArraySingleObject : ArrayRecord, IRecord +internal sealed class ArraySingleObject : + ArrayRecord, + IRecord, + IBinaryFormatParseable { public static RecordType RecordType => RecordType.ArraySingleObject; - public ArraySingleObject(Id objectId, IReadOnlyList arrayObjects) + public ArraySingleObject(Id objectId, IReadOnlyList arrayObjects) : base(new ArrayInfo(objectId, arrayObjects.Count), arrayObjects) { } static ArraySingleObject IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) + BinaryFormattedObject.ParseState state) { ArraySingleObject record = new( - ArrayInfo.Parse(reader, out Count length), - ReadRecords(reader, recordMap, length)); + ArrayInfo.Parse(state.Reader, out Count length), + ReadRecords(state, length)); - recordMap[record.ObjectId] = record; + state.RecordMap[record.ObjectId] = record; return record; } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySinglePrimitive.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySinglePrimitive.cs index 22cac84af12..84bdd35c0b6 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySinglePrimitive.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySinglePrimitive.cs @@ -13,7 +13,11 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class ArraySinglePrimitive : ArrayRecord, IRecord>, IPrimitiveTypeRecord +internal sealed class ArraySinglePrimitive : + ArrayRecord, + IBinaryFormatParseable, + IRecord>, + IPrimitiveTypeRecord where T : unmanaged { public PrimitiveType PrimitiveType { get; } @@ -26,16 +30,15 @@ public ArraySinglePrimitive(Id objectId, IReadOnlyList arrayObjects) PrimitiveType = TypeInfo.GetPrimitiveType(typeof(T)); } - static ArraySinglePrimitive IBinaryFormatParseable>.Parse( - BinaryReader reader, - RecordMap recordMap) + static ArrayRecord IBinaryFormatParseable.Parse( + BinaryFormattedObject.ParseState state) { - Id id = ArrayInfo.Parse(reader, out Count length); - PrimitiveType primitiveType = (PrimitiveType)reader.ReadByte(); + Id id = ArrayInfo.Parse(state.Reader, out Count length); + PrimitiveType primitiveType = (PrimitiveType)state.Reader.ReadByte(); Debug.Assert(typeof(T) == primitiveType.GetPrimitiveTypeType()); - ArraySinglePrimitive record = new(id, ReadPrimitiveTypes(reader, length)); - recordMap[record.ObjectId] = record; + ArraySinglePrimitive record = new(id, ReadPrimitiveTypes(state.Reader, length)); + state.RecordMap[record.ObjectId] = record; return record; } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySingleString.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySingleString.cs index 106b11ddeec..271caa5db7b 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySingleString.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ArraySingleString.cs @@ -13,23 +13,22 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class ArraySingleString : ArrayRecord, IRecord +internal sealed class ArraySingleString : ArrayRecord, IRecord, IBinaryFormatParseable { public static RecordType RecordType => RecordType.ArraySingleString; - public ArraySingleString(Id objectId, IReadOnlyList arrayObjects) + public ArraySingleString(Id objectId, IReadOnlyList arrayObjects) : base(new ArrayInfo(objectId, arrayObjects.Count), arrayObjects) { } static ArraySingleString IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) + BinaryFormattedObject.ParseState state) { ArraySingleString record = new( - ArrayInfo.Parse(reader, out Count length), - ReadRecords(reader, recordMap, length)); + ArrayInfo.Parse(state.Reader, out Count length), + ReadRecords(state, length)); - recordMap[record.ObjectId] = record; + state.RecordMap[record.ObjectId] = record; return record; } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryArray.PrimitiveBinaryArray.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryArray.PrimitiveBinaryArray.cs new file mode 100644 index 00000000000..dc32a374700 --- /dev/null +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryArray.PrimitiveBinaryArray.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Windows.Forms.BinaryFormat; + +internal static partial class BinaryArray +{ + /// + /// of primitive values. + /// + /// + private sealed class PrimitiveBinaryArray : ArrayRecord, IRecord>, IBinaryArray, IPrimitiveTypeRecord where T : unmanaged + { + public Count Rank { get; } + public BinaryArrayType ArrayType { get; } + public MemberTypeInfo TypeInfo { get; } + public IReadOnlyList Lengths { get; } + public PrimitiveType PrimitiveType { get; } + + internal PrimitiveBinaryArray( + Count rank, + BinaryArrayType arrayType, + IReadOnlyList lengths, + ArrayInfo arrayInfo, + MemberTypeInfo typeInfo, + BinaryReader reader) + : base(arrayInfo, ReadPrimitiveTypes(reader, arrayInfo.Length)) + { + Rank = rank; + ArrayType = arrayType; + TypeInfo = typeInfo; + Lengths = lengths; + PrimitiveType = (PrimitiveType)typeInfo[0].Info!; + } + + public override void Write(BinaryWriter writer) + { + throw new NotSupportedException(); + } + } +} diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryArray.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryArray.cs index b546fe3979b..2e7d30e66e5 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryArray.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryArray.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Runtime.Serialization; + namespace System.Windows.Forms.BinaryFormat; /// @@ -13,54 +15,141 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class BinaryArray : ArrayRecord, IRecord +internal static partial class BinaryArray { - public Count Rank { get; } - public BinaryArrayType Type { get; } - public MemberTypeInfo TypeInfo { get; } - - private BinaryArray( - Count rank, - BinaryArrayType type, - ArrayInfo arrayInfo, - MemberTypeInfo typeInfo, - IReadOnlyList arrayObjects) - : base(arrayInfo, arrayObjects) - { - Rank = rank; - Type = type; - TypeInfo = typeInfo; - } + private const int MaxRanks = 32; public static RecordType RecordType => RecordType.BinaryArray; - static BinaryArray IBinaryFormatParseable.Parse(BinaryReader reader, RecordMap recordMap) + internal static IBinaryArray Parse(BinaryFormattedObject.ParseState state) { - Id objectId = reader.ReadInt32(); - BinaryArrayType arrayType = (BinaryArrayType)reader.ReadByte(); - Count rank = reader.ReadInt32(); - Count length = reader.ReadInt32(); + Id objectId = state.Reader.ReadInt32(); + BinaryArrayType arrayType = (BinaryArrayType)state.Reader.ReadByte(); + + if (arrayType is not (BinaryArrayType.Single or BinaryArrayType.Rectangular or BinaryArrayType.Jagged)) + { + if (arrayType is BinaryArrayType.SingleOffset or BinaryArrayType.RectangularOffset or BinaryArrayType.JaggedOffset) + { + throw new NotSupportedException("Offset arrays are not supported."); + } + + throw new SerializationException("Invalid array type."); + } + + int rank = state.Reader.ReadInt32(); - if (arrayType != BinaryArrayType.Single || rank != 1) + if (arrayType is BinaryArrayType.Single or BinaryArrayType.Jagged) + { + // Jagged array is an array of arrays, there should always be one rank + // for the "outer" array. + if (rank != 1) + { + throw new SerializationException("Invalid array rank."); + } + } + else if (arrayType is BinaryArrayType.Rectangular) { - throw new NotSupportedException("Only single dimensional arrays are currently supported."); + // Multidimensional array + if (rank is < 2 or > MaxRanks) + { + throw new SerializationException("Invalid array rank."); + } } - MemberTypeInfo memberTypeInfo = MemberTypeInfo.Parse(reader, 1); - List arrayObjects = new(Math.Min(BinaryFormattedObject.MaxNewCollectionSize, length)); - (BinaryType type, object? typeInfo) = memberTypeInfo[0]; - for (int i = 0; i < length; i++) + int[] lengths = new int[rank]; + int length; + + if (arrayType is not BinaryArrayType.Rectangular) + { + length = state.Reader.ReadInt32(); + if (length < 0) + { + throw new SerializationException("Invalid array length."); + } + + lengths[0] = length; + } + else { - arrayObjects.Add(ReadValue(reader, recordMap, type, typeInfo)); + length = 1; + + for (int i = 0; i < rank; i++) + { + int rankLength = state.Reader.ReadInt32(); + if (rankLength < 0) + { + throw new SerializationException("Invalid array length."); + } + + // Rectangular (multidimensional) length is product of lengths. + // + // It is technically possible to have multidimensional arrays with + // a total length over int.MaxValue. Even with just bytes this would + // be a very large array (2GB). To constrain this reader we'll reject + // anything that goes over this. + length = checked(length * rankLength); + lengths[i] = rankLength; + } } - BinaryArray record = new(rank, arrayType, new ArrayInfo(objectId, length), memberTypeInfo, arrayObjects); - recordMap[objectId] = record; - return record; + MemberTypeInfo typeInfo = MemberTypeInfo.Parse(state.Reader, 1); + (BinaryType type, object? info) = typeInfo[0]; + + IBinaryArray array = type is not BinaryType.Primitive + ? new ObjectBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state) + : (PrimitiveType)info! switch + { + PrimitiveType.Boolean => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.Byte => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.SByte => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.Char => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.Int16 => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.UInt16 => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.Int32 => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.UInt32 => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.Int64 => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.UInt64 => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.Single => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.Double => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.Decimal => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.DateTime => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + PrimitiveType.TimeSpan => new PrimitiveBinaryArray(rank, arrayType, lengths, new(objectId, length), typeInfo, state.Reader), + _ => throw new SerializationException($"Invalid primitive type '{(PrimitiveType)info}'"), + }; + + state.RecordMap[objectId] = array; + return array; } - public override void Write(BinaryWriter writer) + /// + /// of objects. + /// + /// + private sealed class ObjectBinaryArray : ArrayRecord, IRecord, IBinaryArray { - throw new NotSupportedException(); + public Count Rank { get; } + public BinaryArrayType ArrayType { get; } + public MemberTypeInfo TypeInfo { get; } + public IReadOnlyList Lengths { get; } + + internal ObjectBinaryArray( + Count rank, + BinaryArrayType type, + IReadOnlyList lengths, + ArrayInfo arrayInfo, + MemberTypeInfo typeInfo, + BinaryFormattedObject.ParseState state) + : base(arrayInfo, ReadValues(state, typeInfo[0].Type, typeInfo[0].Info, arrayInfo.Length)) + { + Rank = rank; + ArrayType = type; + TypeInfo = typeInfo; + Lengths = lengths; + } + + public override void Write(BinaryWriter writer) + { + throw new NotSupportedException(); + } } } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.Options.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.Options.cs new file mode 100644 index 00000000000..944be7ba8d7 --- /dev/null +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.Options.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters; + +namespace System.Windows.Forms.BinaryFormat; + +#pragma warning disable SYSLIB0050 // Type or member is obsolete + +internal sealed partial class BinaryFormattedObject +{ + internal sealed class Options + { + /// + /// How exactly assembly names need to match for deserialization. + /// + + public FormatterAssemblyStyle AssemblyMatching { get; set; } = FormatterAssemblyStyle.Simple; + + /// + /// Type name binder. + /// + public SerializationBinder? Binder { get; set; } + + /// + /// Optional type provider. + /// + public ISurrogateSelector? SurrogateSelector { get; set; } + + /// + /// Streaming context. + /// + public StreamingContext StreamingContext { get; set; } = new(StreamingContextStates.All); + } +} + +#pragma warning restore SYSLIB0050 // Type or member is obsolete diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.ParseState.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.ParseState.cs new file mode 100644 index 00000000000..8c80c1d5cd6 --- /dev/null +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.ParseState.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Windows.Forms.BinaryFormat; + +internal sealed partial class BinaryFormattedObject +{ + /// + /// Parsing state for . + /// + internal sealed class ParseState + { + private readonly BinaryFormattedObject _format; + + public ParseState(BinaryReader reader, BinaryFormattedObject format) + { + Reader = reader; + _format = format; + } + + public BinaryReader Reader { get; } + public RecordMap RecordMap => _format._recordMap; + + [UnconditionalSuppressMessage( + "Trimming", + "IL2026:Members annotated with 'RequiresUnreferencedCodeAttribute' require dynamic access otherwise can break functionality when trimming application code", + Justification = """ + Incoming type names are coming off of the formatted stream. There is no way for user code to pass compile + time context for preserialized data. If a type can't be found on deserialization it won't matter any more + than any other case where the type can't be found (e.g. a missing assembly). The deserializer will fail + with information on the missing type that can be used to attribute to keep said type. + """)] + [return: DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)] + public Type GetType(string typeName, Id libraryId) => _format.GetType(typeName, libraryId); + } +} diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.TypeResolver.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.TypeResolver.cs new file mode 100644 index 00000000000..5ea89b4f0a2 --- /dev/null +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.TypeResolver.cs @@ -0,0 +1,160 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Reflection; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters; + +#pragma warning disable SYSLIB0050 // Type or member is obsolete + +namespace System.Windows.Forms.BinaryFormat; + +internal sealed partial class BinaryFormattedObject +{ + internal sealed class TypeResolver + { + private readonly FormatterAssemblyStyle _assemblyMatching; + private readonly SerializationBinder? _binder; + private readonly IReadOnlyRecordMap _recordMap; + + // This has to be Id as we use Id.Null for mscorlib types. + private readonly Dictionary _assemblies = []; + private readonly Dictionary<(string TypeName, Id LibraryId), Type> _types = []; + + private string? _lastTypeName; + private Id _lastLibraryId; + + [AllowNull] + private Type _lastType; + + public TypeResolver(Options options, IReadOnlyRecordMap recordMap) + { + _assemblyMatching = options.AssemblyMatching; + _binder = options.Binder; + _assemblies[Id.Null] = TypeInfo.MscorlibAssembly; + _recordMap = recordMap; + } + + /// + /// Resolves the given type name against the specified library. + /// + /// The library id, or for the "system" assembly. + [RequiresUnreferencedCode("Calls System.Reflection.Assembly.GetType(String)")] + [return: DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)] + internal Type GetType(string typeName, Id libraryId) + { + if (libraryId == _lastLibraryId && string.Equals(typeName, _lastTypeName)) + { + Debug.Assert(_lastType is not null); + return _lastType; + } + + _lastLibraryId = libraryId; + _lastTypeName = typeName; + + if (_types.TryGetValue((typeName, libraryId), out Type? cachedType)) + { + _lastType = cachedType; + return cachedType; + } + + string libraryName = libraryId.IsNull + ? TypeInfo.MscorlibAssemblyName + : ((BinaryLibrary)_recordMap[libraryId]).LibraryName; + + if (_binder?.BindToType(libraryName, typeName) is Type binderType) + { + // BinaryFormatter is inconsistent about what caching behavior you get with binders. + // It would always cache the last item from the binder, but wouldn't put the result + // in the type cache. This could lead to inconsistent results if the binder didn't + // always return the same result for a given set of strings. Choosing to always cache + // for performance. + + _types[(typeName, libraryId)] = binderType; + _lastType = binderType; + return binderType; + } + + if (!_assemblies.TryGetValue(libraryId, out Assembly? assembly)) + { + Debug.Assert(!libraryId.IsNull); + + AssemblyName assemblyName = new(libraryName); + try + { + assembly = Assembly.Load(assemblyName); + } + catch + { + if (_assemblyMatching != FormatterAssemblyStyle.Simple) + { + throw; + } + + assembly = Assembly.Load(assemblyName.Name!); + } + + _assemblies.Add(libraryId, assembly); + } + + Type? type = _assemblyMatching != FormatterAssemblyStyle.Simple + ? assembly.GetType(typeName) + : GetSimplyNamedTypeFromAssembly(assembly, typeName); + + _lastType = type ?? throw new SerializationException($"Could not find type '{typeName}'."); + return _lastType; + } + + [RequiresUnreferencedCode("Calls System.Reflection.Assembly.GetType(String, Boolean, Boolean)")] + private static Type? GetSimplyNamedTypeFromAssembly(Assembly assembly, string typeName) + { + // Catching any exceptions that could be thrown from a failure on assembly load + // This is necessary, for example, if there are generic parameters that are qualified + // with a version of the assembly that predates the one available. + + try + { + return assembly.GetType(typeName, throwOnError: false, ignoreCase: false); + } + catch (TypeLoadException) { } + catch (FileNotFoundException) { } + catch (FileLoadException) { } + catch (BadImageFormatException) { } + + return Type.GetType(typeName, ResolveSimpleAssemblyName, new TopLevelAssemblyTypeResolver(assembly).ResolveType, throwOnError: false); + + static Assembly? ResolveSimpleAssemblyName(AssemblyName assemblyName) + { + try + { + return Assembly.Load(assemblyName); + } + catch { } + + try + { + return Assembly.Load(assemblyName.Name!); + } + catch { } + + return null; + } + } + + private sealed class TopLevelAssemblyTypeResolver + { + private readonly Assembly _topLevelAssembly; + + public TopLevelAssemblyTypeResolver(Assembly topLevelAssembly) => _topLevelAssembly = topLevelAssembly; + + [RequiresUnreferencedCode("Calls System.Reflection.Assembly.GetType(String, Boolean, Boolean)")] + public Type? ResolveType(Assembly? assembly, string simpleTypeName, bool ignoreCase) + { + assembly ??= _topLevelAssembly; + return assembly.GetType(simpleTypeName, throwOnError: false, ignoreCase); + } + } + } +} + +#pragma warning restore SYSLIB0050 // Type or member is obsolete diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.cs index 858d804b050..7b87e58963f 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObject.cs @@ -1,7 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Runtime.Serialization; +using System.Reflection; +using System.Runtime.ExceptionServices; using System.Text; namespace System.Windows.Forms.BinaryFormat; @@ -19,42 +20,47 @@ namespace System.Windows.Forms.BinaryFormat; /// NOTE: Multidimensional and jagged arrays are not yet implemented. /// /// -internal sealed class BinaryFormattedObject +internal sealed partial class BinaryFormattedObject { // Don't reserve space in collections based on read lengths for more than this size to defend against corrupted lengths. -#if DEBUG internal const int MaxNewCollectionSize = 1024 * 10; -#else - internal const int MaxNewCollectionSize = 10; -#endif + + private static readonly Options s_defaultOptions = new(); private readonly List _records = []; private readonly RecordMap _recordMap = new(); + private readonly Options _options; + private TypeResolver? _typeResolver; + /// /// Creates by parsing . /// - public BinaryFormattedObject(Stream stream, bool leaveOpen = false) + public BinaryFormattedObject(Stream stream, Options? options = null) { ArgumentNullException.ThrowIfNull(stream); - using BinaryReader reader = new(stream, Encoding.UTF8, leaveOpen: leaveOpen); + using BinaryReader reader = new(stream, Encoding.UTF8, leaveOpen: true); + + _options = options ?? s_defaultOptions; + + ParseState state = new(reader, this); IRecord? currentRecord; do { try { - currentRecord = Record.ReadBinaryFormatRecord(reader, _recordMap); - } - catch (SerializationException) - { - throw; + currentRecord = Record.ReadBinaryFormatRecord(state); } catch (Exception ex) when (ex is ArgumentException or InvalidCastException or ArithmeticException or IOException) { // Make the exception easier to catch, but retain the original stack trace. throw ex.ConvertToSerializationException(); } + catch (TargetInvocationException ex) + { + throw ExceptionDispatchInfo.Capture(ex.InnerException!).SourceException.ConvertToSerializationException(); + } _records.Add(currentRecord); } @@ -76,4 +82,16 @@ public BinaryFormattedObject(Stream stream, bool leaveOpen = false) /// can be referenced by other records. /// public IRecord this[Id id] => _recordMap[id]; + + /// + /// Resolves the given type name against the specified library. + /// + /// The library id, or for the "system" assembly. + [RequiresUnreferencedCode("Calls System.Windows.Forms.BinaryFormat.BinaryFormattedObject.TypeResolver.GetType(String, Id)")] + [return: DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)] + internal Type GetType(string typeName, Id libraryId) + { + _typeResolver ??= new(_options, _recordMap); + return _typeResolver.GetType(typeName, libraryId); + } } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObjectExtensions.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObjectExtensions.cs index 561d9d45a27..296ef9e5204 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObjectExtensions.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryFormattedObjectExtensions.cs @@ -68,7 +68,7 @@ static bool Get(BinaryFormattedObject format, [NotNullWhen(true)] out object? va return false; } - value = new PointF((float)classInfo["x"], (float)classInfo["y"]); + value = new PointF((float)classInfo["x"]!, (float)classInfo["y"]!); return true; } @@ -96,10 +96,10 @@ static bool Get(BinaryFormattedObject format, [NotNullWhen(true)] out object? va } value = new RectangleF( - (float)classInfo["x"], - (float)classInfo["y"], - (float)classInfo["width"], - (float)classInfo["height"]); + (float)classInfo["x"]!, + (float)classInfo["y"]!, + (float)classInfo["width"]!, + (float)classInfo["height"]!); return true; } @@ -134,42 +134,42 @@ static bool Get(BinaryFormattedObject format, [NotNullWhen(true)] out object? va if (IsPrimitiveTypeClassName(systemClass.Name) && systemClass.MemberTypeInfo[0].Type == BinaryType.Primitive) { - value = systemClass.MemberValues[0]; + value = systemClass.MemberValues[0]!; return true; } if (systemClass.Name == typeof(TimeSpan).FullName) { - value = new TimeSpan((long)systemClass.MemberValues[0]); + value = new TimeSpan((long)systemClass.MemberValues[0]!); return true; } switch (systemClass.Name) { case TypeInfo.TimeSpanType: - value = new TimeSpan((long)systemClass.MemberValues[0]); + value = new TimeSpan((long)systemClass.MemberValues[0]!); return true; case TypeInfo.DateTimeType: - ulong ulongValue = (ulong)systemClass["dateData"]; + ulong ulongValue = (ulong)systemClass["dateData"]!; value = Unsafe.As(ref ulongValue); return true; case TypeInfo.DecimalType: ReadOnlySpan bits = [ - (int)systemClass["lo"], - (int)systemClass["mid"], - (int)systemClass["hi"], - (int)systemClass["flags"] + (int)systemClass["lo"]!, + (int)systemClass["mid"]!, + (int)systemClass["hi"]!, + (int)systemClass["flags"]! ]; value = new decimal(bits); return true; case TypeInfo.IntPtrType: // Rehydrating still throws even though casting doesn't any more - value = checked((nint)(long)systemClass.MemberValues[0]); + value = checked((nint)(long)systemClass.MemberValues[0]!); return true; case TypeInfo.UIntPtrType: - value = checked((nuint)(ulong)systemClass.MemberValues[0]); + value = checked((nuint)(ulong)systemClass.MemberValues[0]!); return true; default: return false; @@ -211,7 +211,7 @@ static bool Get(BinaryFormattedObject format, [NotNullWhen(true)] out object? li try { // Lists serialize the entire backing array. - if ((size = (int)classInfo["_size"]) > array.Length) + if ((size = (int)classInfo["_size"]!) > array.Length) { return false; } @@ -290,7 +290,7 @@ static bool Get(BinaryFormattedObject format, [NotNullWhen(true)] out object? va try { // Lists serialize the entire backing array. - if ((size = (int)classInfo["_size"]) > array.Length) + if ((size = (int)classInfo["_size"]!) > array.Length) { return false; } @@ -303,7 +303,7 @@ static bool Get(BinaryFormattedObject format, [NotNullWhen(true)] out object? va ArrayList arrayList = new(size); for (int i = 0; i < size; i++) { - if (!format.TryGetPrimitiveRecordValueOrNull((IRecord)array[i], out object? item)) + if (!format.TryGetPrimitiveRecordValueOrNull((IRecord)array[i]!, out object? item)) { return false; } @@ -402,8 +402,8 @@ static bool Get(BinaryFormattedObject format, [NotNullWhen(true)] out object? ha Hashtable temp = new(keys.Length); for (int i = 0; i < keys.Length; i++) { - if (!format.TryGetPrimitiveRecordValue((IRecord)keys[i], out object? key) - || !format.TryGetPrimitiveRecordValueOrNull((IRecord)values[i], out object? value)) + if (!format.TryGetPrimitiveRecordValue((IRecord)keys[i]!, out object? key) + || !format.TryGetPrimitiveRecordValueOrNull((IRecord)values[i]!, out object? value)) { return false; } @@ -473,7 +473,7 @@ static bool Get(BinaryFormattedObject format, [NotNullWhen(true)] out object? ex return false; } - exception = new NotSupportedException(classInfo["Message"].ToString()); + exception = new NotSupportedException(classInfo["Message"]!.ToString()); return true; } } @@ -507,7 +507,7 @@ public static bool TryGetFrameworkObject( /// public static IEnumerable GetStringValues(this BinaryFormattedObject format, ArraySingleString array, int count) => array.ArrayObjects.Take(count).Select(record => - format.Dereference((IRecord)record) switch + format.Dereference((IRecord)record!) switch { BinaryObjectString stringRecord => stringRecord.Value, _ => null diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryLibrary.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryLibrary.cs index 53a2890832e..067860b5f23 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryLibrary.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryLibrary.cs @@ -13,7 +13,7 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class BinaryLibrary : IRecord +internal sealed class BinaryLibrary : IRecord, IBinaryFormatParseable { public Id LibraryId { get; } public string LibraryName { get; } @@ -27,14 +27,13 @@ public BinaryLibrary(Id libraryId, string libraryName) public static RecordType RecordType => RecordType.BinaryLibrary; static BinaryLibrary IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) + BinaryFormattedObject.ParseState state) { BinaryLibrary record = new( - reader.ReadInt32(), - reader.ReadString()); + state.Reader.ReadInt32(), + state.Reader.ReadString()); - recordMap[record.LibraryId] = record; + state.RecordMap[record.LibraryId] = record; return record; } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryObjectString.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryObjectString.cs index 01114c7de2c..9fac3f4c138 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryObjectString.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/BinaryObjectString.cs @@ -13,7 +13,7 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class BinaryObjectString : IRecord +internal sealed class BinaryObjectString : IRecord, IBinaryFormatParseable { public Id ObjectId { get; } public string Value { get; } @@ -27,12 +27,11 @@ public BinaryObjectString(Id objectId, string value) } static BinaryObjectString IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) + BinaryFormattedObject.ParseState state) { - BinaryObjectString record = new(reader.ReadInt32(), reader.ReadString()); + BinaryObjectString record = new(state.Reader.ReadInt32(), state.Reader.ReadString()); - recordMap[record.ObjectId] = record; + state.RecordMap[record.ObjectId] = record; return record; } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassRecord.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassRecord.cs index f9b8a13626b..12bca0b7a6d 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassRecord.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassRecord.cs @@ -17,7 +17,8 @@ namespace System.Windows.Forms.BinaryFormat; internal abstract class ClassRecord : ObjectRecord { internal ClassInfo ClassInfo { get; } - public IReadOnlyList MemberValues { get; } + public IReadOnlyList MemberValues { get; } + public MemberTypeInfo MemberTypeInfo { get; } public string Name => ClassInfo.Name; public override Id ObjectId => ClassInfo.ObjectId; @@ -25,7 +26,7 @@ internal abstract class ClassRecord : ObjectRecord public IReadOnlyList MemberNames => ClassInfo.MemberNames; - public object this[string memberName] + public object? this[string memberName] { get { @@ -42,15 +43,10 @@ public object this[string memberName] } } - private protected ClassRecord(ClassInfo classInfo, IReadOnlyList memberValues) + private protected ClassRecord(ClassInfo classInfo, MemberTypeInfo memberTypeInfo, IReadOnlyList memberValues) { ClassInfo = classInfo; MemberValues = memberValues; - } - - private protected static List ReadDataFromClassInfo(BinaryReader reader, RecordMap recordMap, ClassInfo info) - { - // Not sure what gets us into this state yet. - return ReadRecords(reader, recordMap, info.MemberNames.Count); + MemberTypeInfo = memberTypeInfo; } } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassRecordExtensions.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassRecordExtensions.cs new file mode 100644 index 00000000000..721b01ccf77 --- /dev/null +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassRecordExtensions.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +namespace System.Windows.Forms.BinaryFormat; + +internal static class ClassRecordExtensions +{ + private static bool IsPrimitiveType(this SystemClassWithMembersAndTypes systemClass) => + systemClass.IsPrimitiveTypeClassName() && systemClass.MemberTypeInfo[0].Type == BinaryType.Primitive; + + private static bool IsPrimitiveTypeClassName(this SystemClassWithMembersAndTypes systemClass) => TypeInfo.GetPrimitiveType(systemClass.Name) switch + { + PrimitiveType.Boolean => true, + PrimitiveType.Byte => true, + PrimitiveType.Char => true, + PrimitiveType.Double => true, + PrimitiveType.Int32 => true, + PrimitiveType.Int64 => true, + PrimitiveType.SByte => true, + PrimitiveType.Single => true, + PrimitiveType.Int16 => true, + PrimitiveType.UInt16 => true, + PrimitiveType.UInt32 => true, + PrimitiveType.UInt64 => true, + _ => false, + }; + + internal static bool TryGetSystemPrimitive(this SystemClassWithMembersAndTypes systemClass, [NotNullWhen(true)] out object? value) + { + value = null; + + if (systemClass.IsPrimitiveType()) + { + value = systemClass.MemberValues[0]; + Debug.Assert(value is not null); + return true; + } + + if (systemClass.Name == typeof(TimeSpan).FullName) + { + value = new TimeSpan((long)systemClass.MemberValues[0]!); + return true; + } + + switch (systemClass.Name) + { + case TypeInfo.TimeSpanType: + value = new TimeSpan((long)systemClass.MemberValues[0]!); + return true; + case TypeInfo.DateTimeType: + ulong ulongValue = (ulong)systemClass["dateData"]!; + value = Unsafe.As(ref ulongValue); + return true; + case TypeInfo.DecimalType: + ReadOnlySpan bits = + [ + (int)systemClass["lo"]!, + (int)systemClass["mid"]!, + (int)systemClass["hi"]!, + (int)systemClass["flags"]! + ]; + + value = new decimal(bits); + return true; + case TypeInfo.IntPtrType: + // Rehydrating still throws even though casting doesn't any more + value = checked((nint)(long)systemClass.MemberValues[0]!); + return true; + case TypeInfo.UIntPtrType: + value = checked((nuint)(ulong)systemClass.MemberValues[0]!); + return true; + default: + return false; + } + } +} diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithId.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithId.cs index e02a4d30d00..5b9bd70492b 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithId.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithId.cs @@ -15,11 +15,12 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class ClassWithId : ClassRecord, IRecord +internal sealed class ClassWithId : ClassRecord, IRecord, IBinaryFormatParseable { private readonly ClassRecord _metadataClass; public override Id ObjectId { get; } + public override Id LibraryId { get; } /// /// The ObjectId of a prior , , @@ -27,24 +28,29 @@ internal sealed class ClassWithId : ClassRecord, IRecord /// public Id MetadataId { get; } - public ClassWithId(Id id, ClassRecord metadataClass, IReadOnlyList memberValues) - : base(metadataClass.ClassInfo, memberValues) + public ClassWithId(Id id, ClassRecord metadataClass, IReadOnlyList memberValues) + : base(metadataClass.ClassInfo, metadataClass.MemberTypeInfo, memberValues) { ObjectId = id; MetadataId = metadataClass.ObjectId; + LibraryId = metadataClass.LibraryId; _metadataClass = metadataClass; } + public ClassWithId(Id id, ClassRecord metadataClass, params object?[] memberValues) + : this(id, metadataClass, (IReadOnlyList)memberValues) + { + } + public static RecordType RecordType => RecordType.ClassWithId; static ClassWithId IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) + BinaryFormattedObject.ParseState state) { - Id objectId = reader.ReadInt32(); - Id metadataId = reader.ReadInt32(); + Id objectId = state.Reader.ReadInt32(); + Id metadataId = state.Reader.ReadInt32(); - if (recordMap[metadataId] is not ClassRecord referencedRecord) + if (state.RecordMap[metadataId] is not ClassRecord referencedRecord) { throw new SerializationException(); } @@ -52,23 +58,10 @@ static ClassWithId IBinaryFormatParseable.Parse( ClassWithId record = new( objectId, referencedRecord, - ReadDataFromRefId(reader, recordMap, referencedRecord)); - recordMap[record.ObjectId] = record; + ReadValuesFromMemberTypeInfo(state, referencedRecord.MemberTypeInfo)); + state.RecordMap[record.ObjectId] = record; return record; - - static IReadOnlyList ReadDataFromRefId(BinaryReader reader, RecordMap recordMap, ClassRecord record) => record switch - { - ClassWithMembersAndTypes classWithMembersAndTypes - => ReadValuesFromMemberTypeInfo(reader, recordMap, classWithMembersAndTypes.MemberTypeInfo), - SystemClassWithMembersAndTypes systemClassWithMembersAndTypes - => ReadValuesFromMemberTypeInfo(reader, recordMap, systemClassWithMembersAndTypes.MemberTypeInfo), - ClassWithMembers classWithMembers - => ReadRecords(reader, recordMap, classWithMembers.MemberValues.Count), - SystemClassWithMembers systemClassWithMembers - => ReadRecords(reader, recordMap, systemClassWithMembers.MemberValues.Count), - _ => throw new SerializationException(), - }; } public override void Write(BinaryWriter writer) diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithMembers.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithMembers.cs index f6dc8523ebe..25ecc5d319d 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithMembers.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithMembers.cs @@ -13,12 +13,12 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class ClassWithMembers : ClassRecord, IRecord +internal sealed class ClassWithMembers : ClassRecord, IRecord, IBinaryFormatParseable { public override Id LibraryId { get; } - public ClassWithMembers(ClassInfo classInfo, Id libraryId, IReadOnlyList memberValues) - : base(classInfo, memberValues) + private ClassWithMembers(ClassInfo classInfo, Id libraryId, MemberTypeInfo memberTypeInfo, IReadOnlyList memberValues) + : base(classInfo, memberTypeInfo, memberValues) { LibraryId = libraryId; } @@ -26,24 +26,26 @@ public ClassWithMembers(ClassInfo classInfo, Id libraryId, IReadOnlyList public static RecordType RecordType => RecordType.ClassWithMembers; static ClassWithMembers IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) + BinaryFormattedObject.ParseState state) { - ClassInfo classInfo = ClassInfo.Parse(reader, out _); + ClassInfo classInfo = ClassInfo.Parse(state.Reader, out _); + Id libraryId = state.Reader.ReadInt32(); + MemberTypeInfo memberTypeInfo = MemberTypeInfo.CreateFromClassInfoAndLibrary(state, classInfo, libraryId); ClassWithMembers record = new( classInfo, - reader.ReadInt32(), - ReadDataFromClassInfo(reader, recordMap, classInfo)); + libraryId, + memberTypeInfo, + ReadValuesFromMemberTypeInfo(state, memberTypeInfo)); // Index this record by the id of the embedded ClassInfo's object id. - recordMap[classInfo.ObjectId] = record; + state.RecordMap[classInfo.ObjectId] = record; return record; } public override void Write(BinaryWriter writer) { - writer.Write((byte)RecordType); - ClassInfo.Write(writer); - writer.Write(LibraryId); + // Really shouldn't be writing this record type. It isn't as safe as the typed variant + // and saves very little space. + throw new NotSupportedException(); } } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithMembersAndTypes.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithMembersAndTypes.cs index c9a9600e3df..3b8e921d23b 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithMembersAndTypes.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ClassWithMembersAndTypes.cs @@ -13,19 +13,20 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class ClassWithMembersAndTypes : ClassRecord, IRecord +internal sealed class ClassWithMembersAndTypes : + ClassRecord, + IRecord, + IBinaryFormatParseable { - public MemberTypeInfo MemberTypeInfo { get; } public override Id LibraryId { get; } public ClassWithMembersAndTypes( ClassInfo classInfo, Id libraryId, MemberTypeInfo memberTypeInfo, - IReadOnlyList memberValues) - : base(classInfo, memberValues) + IReadOnlyList memberValues) + : base(classInfo, memberTypeInfo, memberValues) { - MemberTypeInfo = memberTypeInfo; LibraryId = libraryId; } @@ -41,20 +42,19 @@ public ClassWithMembersAndTypes( public static RecordType RecordType => RecordType.ClassWithMembersAndTypes; static ClassWithMembersAndTypes IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) + BinaryFormattedObject.ParseState state) { - ClassInfo classInfo = ClassInfo.Parse(reader, out Count memberCount); - MemberTypeInfo memberTypeInfo = MemberTypeInfo.Parse(reader, memberCount); + ClassInfo classInfo = ClassInfo.Parse(state.Reader, out Count memberCount); + MemberTypeInfo memberTypeInfo = MemberTypeInfo.Parse(state.Reader, memberCount); ClassWithMembersAndTypes record = new( classInfo, - reader.ReadInt32(), + state.Reader.ReadInt32(), memberTypeInfo, - ReadValuesFromMemberTypeInfo(reader, recordMap, memberTypeInfo)); + ReadValuesFromMemberTypeInfo(state, memberTypeInfo)); // Index this record by the id of the embedded ClassInfo's object id. - recordMap[record.ClassInfo.ObjectId] = record; + state.RecordMap[record.ClassInfo.ObjectId] = record; return record; } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IBinaryArray.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IBinaryArray.cs new file mode 100644 index 00000000000..dca43c3c145 --- /dev/null +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IBinaryArray.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Windows.Forms.BinaryFormat; + +/// +/// Binary array record. +/// +/// +internal interface IBinaryArray : IRecord +{ + /// + /// Rank (dimensions) of the array. + /// + Count Rank { get; } + + /// + /// Type of the array. + /// + BinaryArrayType ArrayType { get; } + + /// + /// Lengths of the array (for each dimension). + /// + IReadOnlyList Lengths { get; } + + /// + /// Array element type information. + /// + MemberTypeInfo TypeInfo { get; } +} diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IBinaryFormatParseable.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IBinaryFormatParseable.cs index 7347211a0ba..d586bbb110c 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IBinaryFormatParseable.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IBinaryFormatParseable.cs @@ -11,8 +11,5 @@ internal interface IBinaryFormatParseable where T : IRecord /// /// Creates the type utilizaing the given . /// - /// - /// Record map for looking up referenced records. If this record has an id it will be added to the map. - /// - static abstract T Parse(BinaryReader reader, RecordMap recordMap); + static abstract T Parse(BinaryFormattedObject.ParseState state); } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IReadOnlyRecordMap.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IReadOnlyRecordMap.cs new file mode 100644 index 00000000000..7567b863ec8 --- /dev/null +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IReadOnlyRecordMap.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Windows.Forms.BinaryFormat; + +/// +/// Map of records. +/// +internal interface IReadOnlyRecordMap +{ + IRecord this[Id id] { get; } +} diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IRecord.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IRecord.cs index 7dba711576c..86a6d665103 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IRecord.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/IRecord.cs @@ -14,6 +14,6 @@ internal interface IRecord : IBinaryWriteable /// /// Typed record interface. /// -internal interface IRecord : IRecord, IBinaryFormatParseable where T : class, IRecord +internal interface IRecord : IRecord where T : class, IRecord { } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberPrimitiveTyped.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberPrimitiveTyped.cs index ad3c8e58eef..7ea037b38f7 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberPrimitiveTyped.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberPrimitiveTyped.cs @@ -13,7 +13,11 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class MemberPrimitiveTyped : Record, IRecord, IPrimitiveTypeRecord +internal sealed class MemberPrimitiveTyped : + Record, + IRecord, + IPrimitiveTypeRecord, + IBinaryFormatParseable { public PrimitiveType PrimitiveType { get; } public object Value { get; } @@ -40,13 +44,12 @@ internal MemberPrimitiveTyped(object value) public static RecordType RecordType => RecordType.MemberPrimitiveTyped; static MemberPrimitiveTyped IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) + BinaryFormattedObject.ParseState state) { - PrimitiveType primitiveType = (PrimitiveType)reader.ReadByte(); + PrimitiveType primitiveType = (PrimitiveType)state.Reader.ReadByte(); return new( primitiveType, - ReadPrimitiveType(reader, primitiveType)); + ReadPrimitiveType(state.Reader, primitiveType)); } public override void Write(BinaryWriter writer) diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberReference.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberReference.cs index d2c7a44ff0a..6bd635e5b11 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberReference.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberReference.cs @@ -13,7 +13,7 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class MemberReference : IRecord +internal sealed class MemberReference : IRecord, IBinaryFormatParseable { public Id IdRef { get; } @@ -22,8 +22,7 @@ internal sealed class MemberReference : IRecord public static RecordType RecordType => RecordType.MemberReference; static MemberReference IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) => new(reader.ReadInt32()); + BinaryFormattedObject.ParseState state) => new(state.Reader.ReadInt32()); public void Write(BinaryWriter writer) { diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberTypeInfo.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberTypeInfo.cs index dad6be7d973..37a9278fbce 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberTypeInfo.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MemberTypeInfo.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections; +using System.Reflection; using System.Runtime.Serialization; namespace System.Windows.Forms.BinaryFormat; @@ -16,7 +17,7 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal readonly struct MemberTypeInfo : IBinaryWriteable, IEnumerable<(BinaryType Type, object? Info)> +internal class MemberTypeInfo : IBinaryWriteable, IEnumerable<(BinaryType Type, object? Info)> { private readonly IList<(BinaryType Type, object? Info)> _info; @@ -24,8 +25,8 @@ namespace System.Windows.Forms.BinaryFormat; public MemberTypeInfo(params (BinaryType Type, object? Info)[] info) => _info = info; - public readonly (BinaryType Type, object? Info) this[int index] => _info[index]; - public readonly int Count => _info.Count; + public (BinaryType Type, object? Info) this[int index] => _info[index]; + public int Count => _info.Count; public static MemberTypeInfo Parse(BinaryReader reader, Count expectedCount) { @@ -68,7 +69,43 @@ public static MemberTypeInfo Parse(BinaryReader reader, Count expectedCount) return new MemberTypeInfo(info); } - public readonly void Write(BinaryWriter writer) + internal static MemberTypeInfo CreateFromClassInfoAndLibrary(BinaryFormattedObject.ParseState state, ClassInfo classInfo, Id libraryId) + { + Type type = state.GetType(classInfo.Name, libraryId); + + if (typeof(ISerializable).IsAssignableFrom(type)) + { + throw new SerializationException("Cannot intuit type information for ISerializable types."); + } + +#pragma warning disable SYSLIB0050 // Type or member is obsolete + MemberInfo[] memberInfo = FormatterServices.GetSerializableMembers(type); +#pragma warning restore SYSLIB0050 + IReadOnlyList memberNames = classInfo.MemberNames; + var info = new (BinaryType Type, object? Info)[memberInfo.Length]; + + for (int i = 0; i < memberNames.Count; i++) + { + // FormatterServices never returns anything other than FieldInfo. + FieldInfo? field = (FieldInfo?)memberInfo.FirstOrDefault(m => m.Name == memberNames[i]) + ?? throw new SerializationException($"Could not find member '{memberNames[i]}' on type '{classInfo.Name}'."); + + BinaryType binaryType = TypeInfo.GetBinaryType(field.FieldType); + + info[i] = (binaryType, binaryType switch + { + BinaryType.PrimitiveArray => TypeInfo.GetPrimitiveArrayType(field.FieldType), + BinaryType.Primitive => TypeInfo.GetPrimitiveType(field.FieldType), + BinaryType.SystemClass => field.FieldType, + BinaryType.Class => field.FieldType, + _ => null + }); + } + + return new(info); + } + + public void Write(BinaryWriter writer) { foreach ((BinaryType type, _) in this) { diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MessageEnd.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MessageEnd.cs index c19eabf3e3a..584c81a0c73 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MessageEnd.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/MessageEnd.cs @@ -6,7 +6,7 @@ namespace System.Windows.Forms.BinaryFormat; /// /// Record that marks the end of the binary format stream. /// -internal sealed class MessageEnd : IRecord +internal sealed class MessageEnd : IRecord, IBinaryFormatParseable { public static MessageEnd Instance { get; } = new(); @@ -15,8 +15,7 @@ private MessageEnd() { } public static RecordType RecordType => RecordType.MessageEnd; static MessageEnd IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) => Instance; + BinaryFormattedObject.ParseState state) => Instance; public void Write(BinaryWriter writer) => writer.Write((byte)RecordType); } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/NullRecord.ObjectNullMultiple.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/NullRecord.ObjectNullMultiple.cs index e4581efa1c6..729ae844e06 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/NullRecord.ObjectNullMultiple.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/NullRecord.ObjectNullMultiple.cs @@ -15,16 +15,18 @@ internal abstract partial class NullRecord /// /// /// - internal sealed class ObjectNullMultiple : NullRecord, IRecord + internal sealed class ObjectNullMultiple : + NullRecord, + IRecord, + IBinaryFormatParseable { public static RecordType RecordType => RecordType.ObjectNullMultiple; public ObjectNullMultiple(Count count) => NullCount = count; static ObjectNullMultiple IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) - => new(reader.ReadInt32()); + BinaryFormattedObject.ParseState state) + => new(state.Reader.ReadInt32()); public void Write(BinaryWriter writer) { diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/NullRecord.ObjectNullMultiple256.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/NullRecord.ObjectNullMultiple256.cs index 2de823c56b6..1b8d45505af 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/NullRecord.ObjectNullMultiple256.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/NullRecord.ObjectNullMultiple256.cs @@ -15,15 +15,17 @@ internal abstract partial class NullRecord /// /// /// - internal sealed class ObjectNullMultiple256 : NullRecord, IRecord + internal sealed class ObjectNullMultiple256 : + NullRecord, + IRecord, + IBinaryFormatParseable { public static RecordType RecordType => RecordType.ObjectNullMultiple256; public ObjectNullMultiple256(Count count) => NullCount = count; static ObjectNullMultiple256 IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) => new(reader.ReadByte()); + BinaryFormattedObject.ParseState state) => new(state.Reader.ReadByte()); public void Write(BinaryWriter writer) { diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ObjectNull.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ObjectNull.cs index 07bef61ba09..8ef6754a3bf 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ObjectNull.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/ObjectNull.cs @@ -13,19 +13,18 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class ObjectNull : NullRecord, IRecord +internal sealed class ObjectNull : NullRecord, IRecord, IBinaryFormatParseable { public static ObjectNull Instance { get; } = new(); private ObjectNull() { } - public override Count NullCount => Count.One; + public override Count NullCount => 1; public static RecordType RecordType => RecordType.ObjectNull; static ObjectNull IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) => Instance; + BinaryFormattedObject.ParseState state) => Instance; public void Write(BinaryWriter writer) => writer.Write((byte)RecordType); diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Record.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Record.cs index 0e15c494cac..9de4e927aae 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Record.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Record.cs @@ -35,78 +35,30 @@ internal abstract class Record : IRecord _ => throw new SerializationException($"Failure trying to read primitive '{primitiveType}'"), }; - /// - /// Reads primitives of from the given . - /// - private protected static IReadOnlyList ReadPrimitiveTypes(BinaryReader reader, PrimitiveType primitiveType, int count) - { - List values = new(Math.Min(count, BinaryFormattedObject.MaxNewCollectionSize)); - for (int i = 0; i < count; i++) - { - values.Add(ReadPrimitiveType(reader, primitiveType)); - } - - return values; - } - private protected static IReadOnlyList ReadPrimitiveTypes(BinaryReader reader, int count) where T : unmanaged { - // Special casing byte for performance. - if (typeof(T) == typeof(byte)) + // Special casing simple primitives for performance. + if (typeof(T) == typeof(bool) + || typeof(T) == typeof(byte) + || typeof(T) == typeof(sbyte) + || typeof(T) == typeof(char) + || typeof(T) == typeof(short) + || typeof(T) == typeof(ushort) + || typeof(T) == typeof(int) + || typeof(T) == typeof(uint) + || typeof(T) == typeof(long) + || typeof(T) == typeof(ulong) + || typeof(T) == typeof(float) + || typeof(T) == typeof(double)) { - byte[] bytes = reader.ReadBytes(count); - return (IReadOnlyList)(object)bytes; + return reader.ReadPrimitiveArray(count); } List values = new(Math.Min(count, BinaryFormattedObject.MaxNewCollectionSize)); for (int i = 0; i < count; i++) { - if (typeof(T) == typeof(bool)) - { - values.Add((T)(object)reader.ReadBoolean()); - } - else if (typeof(T) == typeof(sbyte)) - { - values.Add((T)(object)reader.ReadSByte()); - } - else if (typeof(T) == typeof(char)) - { - values.Add((T)(object)reader.ReadChar()); - } - else if (typeof(T) == typeof(short)) - { - values.Add((T)(object)reader.ReadInt16()); - } - else if (typeof(T) == typeof(ushort)) - { - values.Add((T)(object)reader.ReadUInt16()); - } - else if (typeof(T) == typeof(int)) - { - values.Add((T)(object)reader.ReadInt32()); - } - else if (typeof(T) == typeof(uint)) - { - values.Add((T)(object)reader.ReadUInt32()); - } - else if (typeof(T) == typeof(long)) - { - values.Add((T)(object)reader.ReadInt64()); - } - else if (typeof(T) == typeof(ulong)) - { - values.Add((T)(object)reader.ReadUInt64()); - } - else if (typeof(T) == typeof(float)) - { - values.Add((T)(object)reader.ReadSingle()); - } - else if (typeof(T) == typeof(double)) - { - values.Add((T)(object)reader.ReadDouble()); - } - else if (typeof(T) == typeof(decimal)) + if (typeof(T) == typeof(decimal)) { values.Add((T)(object)decimal.Parse(reader.ReadString(), CultureInfo.InvariantCulture)); } @@ -120,11 +72,12 @@ private protected static IReadOnlyList ReadPrimitiveTypes(BinaryReader rea } else { - throw new SerializationException($"Failure trying to read primitive '{typeof(T)}'"); + throw new SerializationException($"Invalid primitive type '{typeof(T)}'"); } } - return values; + // For simplicity in the deserialization model we'll always return an array. + return [.. values]; } /// @@ -187,17 +140,6 @@ private protected static unsafe void WritePrimitiveType(BinaryWriter writer, Pri } } - /// - /// Writes as to the given . - /// - private protected static void WritePrimitiveTypes(BinaryWriter writer, PrimitiveType primitiveType, IReadOnlyList values) - { - for (int i = 0; i < values.Count; i++) - { - WritePrimitiveType(writer, primitiveType, values[i]); - } - } - private protected static void WritePrimitiveTypes(BinaryWriter writer, IReadOnlyList values) where T : unmanaged { @@ -286,87 +228,86 @@ private protected static void WritePrimitiveTypes(BinaryWriter writer, IReadO } /// - /// Reads the next record from the given . + /// Reads the next record. /// /// Found a multidimensional array. /// Found a remote method invocation record. /// Unknown or corrupted data. - internal static IRecord ReadBinaryFormatRecord(BinaryReader reader, RecordMap recordMap) + internal static IRecord ReadBinaryFormatRecord(BinaryFormattedObject.ParseState state) { - RecordType recordType = (RecordType)reader.ReadByte(); + RecordType recordType = (RecordType)state.Reader.ReadByte(); return recordType switch { - RecordType.SerializedStreamHeader => ReadSpecificRecord(recordMap), - RecordType.ClassWithId => ReadSpecificRecord(recordMap), - RecordType.SystemClassWithMembers => ReadSpecificRecord(recordMap), - RecordType.ClassWithMembers => ReadSpecificRecord(recordMap), - RecordType.SystemClassWithMembersAndTypes => ReadSpecificRecord(recordMap), - RecordType.ClassWithMembersAndTypes => ReadSpecificRecord(recordMap), - RecordType.BinaryObjectString => ReadSpecificRecord(recordMap), - // The BinaryArray record is used for all types of arrays, but we currently only support single dimension. - RecordType.BinaryArray => ReadSpecificRecord(recordMap), - RecordType.MemberPrimitiveTyped => ReadSpecificRecord(recordMap), - RecordType.MemberReference => ReadSpecificRecord(recordMap), - RecordType.ObjectNull => ReadSpecificRecord(recordMap), - RecordType.MessageEnd => ReadSpecificRecord(recordMap), - RecordType.BinaryLibrary => ReadSpecificRecord(recordMap), - RecordType.ObjectNullMultiple256 => ReadSpecificRecord(recordMap), - RecordType.ObjectNullMultiple => ReadSpecificRecord(recordMap), - RecordType.ArraySinglePrimitive => ReadArraySinglePrimitive(recordMap), - RecordType.ArraySingleObject => ReadSpecificRecord(recordMap), - RecordType.ArraySingleString => ReadSpecificRecord(recordMap), + RecordType.SerializedStreamHeader => ReadSpecificRecord(state), + RecordType.ClassWithId => ReadSpecificRecord(state), + RecordType.SystemClassWithMembers => ReadSpecificRecord(state), + RecordType.ClassWithMembers => ReadSpecificRecord(state), + RecordType.SystemClassWithMembersAndTypes => ReadSpecificRecord(state), + RecordType.ClassWithMembersAndTypes => ReadSpecificRecord(state), + RecordType.BinaryObjectString => ReadSpecificRecord(state), + RecordType.BinaryArray => BinaryArray.Parse(state), + RecordType.MemberPrimitiveTyped => ReadSpecificRecord(state), + RecordType.MemberReference => ReadSpecificRecord(state), + RecordType.ObjectNull => ReadSpecificRecord(state), + RecordType.MessageEnd => ReadSpecificRecord(state), + RecordType.BinaryLibrary => ReadSpecificRecord(state), + RecordType.ObjectNullMultiple256 => ReadSpecificRecord(state), + RecordType.ObjectNullMultiple => ReadSpecificRecord(state), + RecordType.ArraySinglePrimitive => ReadArraySinglePrimitive(state), + RecordType.ArraySingleObject => ReadSpecificRecord(state), + RecordType.ArraySingleString => ReadSpecificRecord(state), RecordType.MethodCall => throw new NotSupportedException(), RecordType.MethodReturn => throw new NotSupportedException(), _ => throw new SerializationException("Invalid record type."), }; - IRecord ReadArraySinglePrimitive(RecordMap recordMap) + static IRecord ReadArraySinglePrimitive(BinaryFormattedObject.ParseState state) { // Special casing to avoid excessive boxing/unboxing. - Id id = ArrayInfo.Parse(reader, out Count length); - PrimitiveType primitiveType = (PrimitiveType)reader.ReadByte(); + Id id = ArrayInfo.Parse(state.Reader, out Count length); + PrimitiveType primitiveType = (PrimitiveType)state.Reader.ReadByte(); IRecord record = primitiveType switch { - PrimitiveType.Boolean => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.Byte => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.SByte => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.Char => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.Int16 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.UInt16 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.Int32 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.UInt32 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.Int64 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.UInt64 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.Single => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.Double => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.Decimal => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.DateTime => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - PrimitiveType.TimeSpan => new ArraySinglePrimitive(id, ReadPrimitiveTypes(reader, length)), - _ => throw new SerializationException($"Failure trying to read primitive '{primitiveType}'"), + PrimitiveType.Boolean => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.Byte => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.SByte => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.Char => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.Int16 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.UInt16 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.Int32 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.UInt32 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.Int64 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.UInt64 => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.Single => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.Double => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.Decimal => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.DateTime => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + PrimitiveType.TimeSpan => new ArraySinglePrimitive(id, ReadPrimitiveTypes(state.Reader, length)), + _ => throw new SerializationException($"Invalid primitive type '{primitiveType}'"), }; - recordMap[id] = record; + state.RecordMap[id] = record; return record; } - unsafe TRecord ReadSpecificRecord(RecordMap recordMap) where TRecord : class, IRecord + unsafe static TRecord ReadSpecificRecord(BinaryFormattedObject.ParseState state) where TRecord : class, IRecord, IBinaryFormatParseable { - return TRecord.Parse(reader, recordMap); + return TRecord.Parse(state); } } /// /// Reads records, expanding null records into individual entries. /// - private protected static List ReadRecords(BinaryReader reader, RecordMap recordMap, Count count) + private protected static List ReadRecords(BinaryFormattedObject.ParseState state, Count count) { List objects = new(Math.Min(count, BinaryFormattedObject.MaxNewCollectionSize)); for (int i = 0; i < count; i++) { - var record = ReadBinaryFormatRecord(reader, recordMap); + IRecord record = ReadBinaryFormatRecord(state); if (record is not NullRecord nullRecord) { objects.Add(record); @@ -395,7 +336,7 @@ private protected static List ReadRecords(BinaryReader reader, RecordMap /// /// contained an object that isn't a record. /// - private protected static void WriteRecords(BinaryWriter writer, IReadOnlyList objects) + private protected static void WriteRecords(BinaryWriter writer, IReadOnlyList objects) { int nullCount = 0; @@ -407,7 +348,7 @@ private protected static void WriteRecords(BinaryWriter writer, IReadOnlyList. /// private protected static IReadOnlyList ReadValuesFromMemberTypeInfo( - BinaryReader reader, - RecordMap recordMap, + BinaryFormattedObject.ParseState state, MemberTypeInfo memberTypeInfo) { - List memberValues = new(memberTypeInfo.Count); + List memberValues = new(Math.Min(memberTypeInfo.Count, BinaryFormattedObject.MaxNewCollectionSize)); foreach ((BinaryType type, object? info) in memberTypeInfo) { - memberValues.Add(ReadValue(reader, recordMap, type, info)); + memberValues.Add(ReadValue(state, type, info)); + } + + return memberValues; + } + + /// + /// Reads a count of object member values of with optional clarifying . + /// + /// was unexpected. + private protected static IReadOnlyList ReadValues( + BinaryFormattedObject.ParseState state, + BinaryType type, + object? typeInfo, + int count) + { + List memberValues = new(Math.Min(count, BinaryFormattedObject.MaxNewCollectionSize)); + for (int i = 0; i < count; i++) + { + object value = ReadValue(state, type, typeInfo); + if (value is not NullRecord nullRecord) + { + memberValues.Add(value); + } + else + { + i += nullRecord.NullCount - 1; + if (i >= count) + { + throw new SerializationException(); + } + + for (int j = 0; j < nullRecord.NullCount; j++) + { + memberValues.Add(null); + } + } } return memberValues; @@ -450,19 +426,18 @@ private protected static IReadOnlyList ReadValuesFromMemberTypeInfo( /// /// was unexpected. private protected static object ReadValue( - BinaryReader reader, - RecordMap recordMap, + BinaryFormattedObject.ParseState state, BinaryType type, object? typeInfo) { if (type == BinaryType.Primitive) { - return ReadPrimitiveType(reader, (PrimitiveType)typeInfo!); + return ReadPrimitiveType(state.Reader, (PrimitiveType)typeInfo!); } if (type == BinaryType.String) { - return ReadBinaryFormatRecord(reader, recordMap); + return ReadBinaryFormatRecord(state); } // BinaryLibrary records can be dumped in front of any member reference. @@ -480,7 +455,7 @@ or BinaryType.StringArray or BinaryType.PrimitiveArray or BinaryType.Class or BinaryType.SystemClass - or BinaryType.ObjectArray => ReadBinaryFormatRecord(reader, recordMap), + or BinaryType.ObjectArray => ReadBinaryFormatRecord(state), _ => throw new SerializationException("Invalid binary type."), }; } @@ -491,7 +466,7 @@ or BinaryType.SystemClass private protected static void WriteValuesFromMemberTypeInfo( BinaryWriter writer, MemberTypeInfo memberTypeInfo, - IReadOnlyList memberValues) + IReadOnlyList memberValues) { for (int i = 0; i < memberTypeInfo.Count; i++) { @@ -499,7 +474,7 @@ private protected static void WriteValuesFromMemberTypeInfo( switch (type) { case BinaryType.Primitive: - WritePrimitiveType(writer, (PrimitiveType)info!, memberValues[i]); + WritePrimitiveType(writer, (PrimitiveType)info!, memberValues[i]!); break; case BinaryType.String: case BinaryType.Object: @@ -508,7 +483,7 @@ private protected static void WriteValuesFromMemberTypeInfo( case BinaryType.Class: case BinaryType.SystemClass: case BinaryType.ObjectArray: - ((IRecord)memberValues[i]).Write(writer); + ((IRecord)memberValues[i]!).Write(writer); break; default: throw new ArgumentException("Invalid binary type.", nameof(memberTypeInfo)); diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/RecordMap.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/RecordMap.cs index a4ff524f4d2..ec38784fbd2 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/RecordMap.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/RecordMap.cs @@ -6,7 +6,7 @@ namespace System.Windows.Forms.BinaryFormat; /// /// Map of records that ensures that IDs are only entered once. /// -internal class RecordMap +internal class RecordMap : IReadOnlyRecordMap { private readonly Dictionary _records = []; diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SerializationHeader.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SerializationHeader.cs index 7a9771a2bd3..06744793514 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SerializationHeader.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SerializationHeader.cs @@ -13,7 +13,7 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class SerializationHeader : IRecord +internal sealed class SerializationHeader : IRecord, IBinaryFormatParseable { /// /// The id of the root object record. @@ -45,13 +45,12 @@ internal sealed class SerializationHeader : IRecord }; static SerializationHeader IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) => new() + BinaryFormattedObject.ParseState state) => new() { - RootId = reader.ReadInt32(), - HeaderId = reader.ReadInt32(), - MajorVersion = reader.ReadInt32(), - MinorVersion = reader.ReadInt32(), + RootId = state.Reader.ReadInt32(), + HeaderId = state.Reader.ReadInt32(), + MajorVersion = state.Reader.ReadInt32(), + MinorVersion = state.Reader.ReadInt32() }; public void Write(BinaryWriter writer) diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Support/SerializationExtensions.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Support/SerializationExtensions.cs index 89132feb8ff..256ef3c4159 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Support/SerializationExtensions.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Support/SerializationExtensions.cs @@ -1,7 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; using System.Runtime.Serialization; namespace System.Windows.Forms.BinaryFormat; @@ -23,4 +25,21 @@ public static SerializationException ConvertToSerializationException(this Except : (SerializationException)ExceptionDispatchInfo.SetRemoteStackTrace( new SerializationException(ex.Message, ex), ex.StackTrace ?? string.Empty); + + /// + /// Gets a span over any array, including multi-dimensional arrays. + /// + public static Span GetArrayData(this Array array) + { + if (array.GetType().UnderlyingSystemType.IsAssignableFrom(typeof(T))) + { + throw new InvalidCastException($"Cannot cast array of type {array.GetType().UnderlyingSystemType} to {typeof(T)}."); + } + + Span data = MemoryMarshal.CreateSpan(ref Unsafe.As( + ref MemoryMarshal.GetArrayDataReference(array)), + checked((int)array.LongLength)); + + return data; + } } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Support/TypeInfo.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Support/TypeInfo.cs index 6d70f0541bd..a6869e64670 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Support/TypeInfo.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/Support/TypeInfo.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Reflection; + namespace System.Windows.Forms.BinaryFormat; internal static class TypeInfo @@ -34,6 +36,14 @@ public const string MscorlibAssemblyName public const string SystemDrawingAssemblyName = "System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a"; + private static Assembly? s_mscorlibFacadeAssembly; + + internal static Assembly MscorlibAssembly => s_mscorlibFacadeAssembly + ??= Assembly.Load("mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"); + + internal static Assembly CorelibAssembly { get; } = typeof(string).Assembly; + internal static string CorelibAssemblyString { get; } = CorelibAssembly.FullName!; + /// /// Returns the for the given . /// @@ -62,7 +72,7 @@ public const string SystemDrawingAssemblyName /// Returns the for the given . /// /// or if not a . - internal static PrimitiveType GetPrimitiveType(Type type) => Type.GetTypeCode(type) switch + internal static PrimitiveType GetPrimitiveType(Type type) => type.IsEnum ? default : Type.GetTypeCode(type) switch { TypeCode.Boolean => PrimitiveType.Boolean, TypeCode.Char => PrimitiveType.Char, @@ -85,6 +95,51 @@ public const string SystemDrawingAssemblyName _ => type == typeof(TimeSpan) ? PrimitiveType.TimeSpan : default, }; + /// + /// Returns the for the given if it is a simple primitive array. + /// + /// or if not a primitive array. + internal static PrimitiveType GetPrimitiveArrayType(Type type) => type.IsSZArray + ? GetPrimitiveType(type.GetElementType()!) + : default; + + /// + /// Get the proper for the given . + /// + internal static BinaryType GetBinaryType(Type type) + { + if (type == typeof(string)) + { + return BinaryType.String; + } + else if (type == typeof(object)) + { + return BinaryType.Object; + } + else if (type == typeof(object[])) + { + return BinaryType.ObjectArray; + } + else if (type == typeof(string[])) + { + return BinaryType.StringArray; + } + else if (GetPrimitiveArrayType(type) != default) + { + return BinaryType.PrimitiveArray; + } + else + { + PrimitiveType primitiveType = GetPrimitiveType(type); + if (primitiveType == default) + { + return type.Assembly == MscorlibAssembly ? BinaryType.SystemClass : BinaryType.Class; + } + + return BinaryType.Primitive; + } + } + /// /// Returns the for the given . /// @@ -108,4 +163,28 @@ public const string SystemDrawingAssemblyName PrimitiveType.TimeSpan => typeof(TimeSpan), _ => throw new NotSupportedException(), }; + + /// + /// Returns the for the given . + /// + internal static Type GetArrayPrimitiveTypeType(this PrimitiveType primitiveType) => primitiveType switch + { + PrimitiveType.Boolean => typeof(bool[]), + PrimitiveType.Char => typeof(char[]), + PrimitiveType.SByte => typeof(sbyte[]), + PrimitiveType.Byte => typeof(byte[]), + PrimitiveType.Int16 => typeof(short[]), + PrimitiveType.UInt16 => typeof(ushort[]), + PrimitiveType.Int32 => typeof(int[]), + PrimitiveType.UInt32 => typeof(uint[]), + PrimitiveType.Int64 => typeof(long[]), + PrimitiveType.UInt64 => typeof(ulong[]), + PrimitiveType.Single => typeof(float[]), + PrimitiveType.Double => typeof(double[]), + PrimitiveType.Decimal => typeof(decimal[]), + PrimitiveType.DateTime => typeof(DateTime[]), + PrimitiveType.String => typeof(string[]), + PrimitiveType.TimeSpan => typeof(TimeSpan[]), + _ => throw new NotSupportedException(), + }; } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SystemClassWithMembers.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SystemClassWithMembers.cs index 8c29a202b19..96a62bb0a90 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SystemClassWithMembers.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SystemClassWithMembers.cs @@ -13,30 +13,32 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class SystemClassWithMembers : ClassRecord, IRecord +internal sealed class SystemClassWithMembers : ClassRecord, IRecord, IBinaryFormatParseable { - public SystemClassWithMembers(ClassInfo classInfo, IReadOnlyList memberValues) - : base(classInfo, memberValues) { } + private SystemClassWithMembers(ClassInfo classInfo, MemberTypeInfo memberTypeInfo, IReadOnlyList memberValues) + : base(classInfo, memberTypeInfo, memberValues) { } public static RecordType RecordType => RecordType.SystemClassWithMembers; static SystemClassWithMembers IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) + BinaryFormattedObject.ParseState state) { - ClassInfo classInfo = ClassInfo.Parse(reader, out _); + ClassInfo classInfo = ClassInfo.Parse(state.Reader, out _); + MemberTypeInfo memberTypeInfo = MemberTypeInfo.CreateFromClassInfoAndLibrary(state, classInfo, Id.Null); SystemClassWithMembers record = new( classInfo, - ReadDataFromClassInfo(reader, recordMap, classInfo)); + memberTypeInfo, + ReadValuesFromMemberTypeInfo(state, memberTypeInfo)); // Index this record by the id of the embedded ClassInfo's object id. - recordMap[record.ClassInfo.ObjectId] = record; + state.RecordMap[record.ClassInfo.ObjectId] = record; return record; } public override void Write(BinaryWriter writer) { - writer.Write((byte)RecordType); - ClassInfo.Write(writer); + // Really shouldn't be writing this record type. It isn't as safe as the typed variant + // and saves very little space. + throw new NotSupportedException(); } } diff --git a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SystemClassWithMembersAndTypes.cs b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SystemClassWithMembersAndTypes.cs index eac360f9d47..a6245a9cfc9 100644 --- a/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SystemClassWithMembersAndTypes.cs +++ b/src/System.Private.Windows.Core/src/System/Windows/Forms/BinaryFormat/SystemClassWithMembersAndTypes.cs @@ -13,43 +13,42 @@ namespace System.Windows.Forms.BinaryFormat; /// /// /// -internal sealed class SystemClassWithMembersAndTypes : ClassRecord, IRecord +internal sealed class SystemClassWithMembersAndTypes : + ClassRecord, + IRecord, + IBinaryFormatParseable { - public MemberTypeInfo MemberTypeInfo { get; } - public SystemClassWithMembersAndTypes( ClassInfo classInfo, MemberTypeInfo memberTypeInfo, - IReadOnlyList memberValues) - : base(classInfo, memberValues) + IReadOnlyList memberValues) + : base(classInfo, memberTypeInfo, memberValues) { - MemberTypeInfo = memberTypeInfo; } public SystemClassWithMembersAndTypes( ClassInfo classInfo, MemberTypeInfo memberTypeInfo, - params object[] memberValues) - : this(classInfo, memberTypeInfo, (IReadOnlyList)memberValues) + params object?[] memberValues) + : this(classInfo, memberTypeInfo, (IReadOnlyList)memberValues) { } public static RecordType RecordType => RecordType.SystemClassWithMembersAndTypes; static SystemClassWithMembersAndTypes IBinaryFormatParseable.Parse( - BinaryReader reader, - RecordMap recordMap) + BinaryFormattedObject.ParseState state) { - ClassInfo classInfo = ClassInfo.Parse(reader, out Count memberCount); - MemberTypeInfo memberTypeInfo = MemberTypeInfo.Parse(reader, memberCount); + ClassInfo classInfo = ClassInfo.Parse(state.Reader, out Count memberCount); + MemberTypeInfo memberTypeInfo = MemberTypeInfo.Parse(state.Reader, memberCount); SystemClassWithMembersAndTypes record = new( classInfo, memberTypeInfo, - ReadValuesFromMemberTypeInfo(reader, recordMap, memberTypeInfo)); + ReadValuesFromMemberTypeInfo(state, memberTypeInfo)); // Index this record by the id of the embedded ClassInfo's object id. - recordMap[record.ClassInfo.ObjectId] = record; + state.RecordMap[record.ClassInfo.ObjectId] = record; return record; } diff --git a/src/System.Windows.Forms.Primitives/tests/TestUtilities/System/Windows/Forms/BinaryFormat/BinaryFormatTestExtensions.cs b/src/System.Windows.Forms.Primitives/tests/TestUtilities/System/Windows/Forms/BinaryFormat/BinaryFormatTestExtensions.cs index 6f321089385..46469fbf15c 100644 --- a/src/System.Windows.Forms.Primitives/tests/TestUtilities/System/Windows/Forms/BinaryFormat/BinaryFormatTestExtensions.cs +++ b/src/System.Windows.Forms.Primitives/tests/TestUtilities/System/Windows/Forms/BinaryFormat/BinaryFormatTestExtensions.cs @@ -12,7 +12,11 @@ internal static class BinaryFormatTestExtensions /// /// Serializes the object using the and reads it into a . /// - public static BinaryFormattedObject SerializeAndParse(this object source) => new(source.Serialize()); + public static BinaryFormattedObject SerializeAndParse(this object source) + { + using Stream stream = source.Serialize(); + return new(stream); + } /// /// Serializes the object using the . diff --git a/src/System.Windows.Forms.Primitives/tests/UnitTests/System/Windows/Forms/BinaryFormat/BinaryFormattedObjectTests.cs b/src/System.Windows.Forms.Primitives/tests/UnitTests/System/Windows/Forms/BinaryFormat/BinaryFormattedObjectTests.cs index 157ba609d06..4f444fe0efe 100644 --- a/src/System.Windows.Forms.Primitives/tests/UnitTests/System/Windows/Forms/BinaryFormat/BinaryFormattedObjectTests.cs +++ b/src/System.Windows.Forms.Primitives/tests/UnitTests/System/Windows/Forms/BinaryFormat/BinaryFormattedObjectTests.cs @@ -115,14 +115,14 @@ public void ReadHashTableWithStringPair() ArraySingleObject array = (ArraySingleObject)format[2]; array.ArrayInfo.ObjectId.Should().Be(2); array.ArrayInfo.Length.Should().Be(1); - BinaryObjectString value = (BinaryObjectString)array.ArrayObjects[0]; + BinaryObjectString value = (BinaryObjectString)array.ArrayObjects[0]!; value.ObjectId.Should().Be(4); value.Value.Should().Be("This"); array = (ArraySingleObject)format[3]; array.ArrayInfo.ObjectId.Should().Be(3); array.ArrayInfo.Length.Should().Be(1); - value = (BinaryObjectString)array.ArrayObjects[0]; + value = (BinaryObjectString)array.ArrayObjects[0]!; value.ObjectId.Should().Be(5); value.Value.Should().Be("That"); } @@ -169,7 +169,7 @@ public void ReadHashTableWithNullValues() new MemberReference(3) }); - ArrayRecord array = (ArrayRecord)format[(MemberReference)systemClass.MemberValues[5]]; + ArrayRecord array = (ArrayRecord)format[(MemberReference)systemClass.MemberValues[5]!]; array.ArrayInfo.ObjectId.Should().Be(2); array.ArrayInfo.Length.Should().Be(3); @@ -177,7 +177,7 @@ public void ReadHashTableWithNullValues() value.ObjectId.Should().Be(4); value.Value.Should().BeOneOf("Yowza", "Youza", "Meeza"); - array = (ArrayRecord)format[(MemberReference)systemClass["Values"]]; + array = (ArrayRecord)format[(MemberReference)systemClass["Values"]!]; array.ArrayInfo.ObjectId.Should().Be(3); array.ArrayInfo.Length.Should().Be(3); array.ArrayObjects[0].Should().BeOfType(); @@ -309,7 +309,7 @@ public void ReadStringArray() ArraySingleString array = (ArraySingleString)format[1]; array.ArrayInfo.ObjectId.Should().Be(1); array.ArrayInfo.Length.Should().Be(3); - BinaryObjectString value = (BinaryObjectString)array.ArrayObjects[0]; + BinaryObjectString value = (BinaryObjectString)array.ArrayObjects[0]!; } [Fact] @@ -328,7 +328,7 @@ public void ReadStringArrayWithNulls() ObjectNull.Instance, ObjectNull.Instance }); - BinaryObjectString value = (BinaryObjectString)array.ArrayObjects[0]; + BinaryObjectString value = (BinaryObjectString)array.ArrayObjects[0]!; } [Fact] @@ -338,8 +338,8 @@ public void ReadDuplicatedStringArray() ArraySingleString array = (ArraySingleString)format[1]; array.ArrayInfo.ObjectId.Should().Be(1); array.ArrayInfo.Length.Should().Be(3); - BinaryObjectString value = (BinaryObjectString)array.ArrayObjects[0]; - MemberReference reference = (MemberReference)array.ArrayObjects[2]; + BinaryObjectString value = (BinaryObjectString)array.ArrayObjects[0]!; + MemberReference reference = (MemberReference)array.ArrayObjects[2]!; reference.IdRef.Should().Be(value.ObjectId); } diff --git a/src/System.Windows.Forms.Primitives/tests/UnitTests/System/Windows/Forms/BinaryFormat/ListTests.cs b/src/System.Windows.Forms.Primitives/tests/UnitTests/System/Windows/Forms/BinaryFormat/ListTests.cs index 34fcf1d43fd..0017be9b3a8 100644 --- a/src/System.Windows.Forms.Primitives/tests/UnitTests/System/Windows/Forms/BinaryFormat/ListTests.cs +++ b/src/System.Windows.Forms.Primitives/tests/UnitTests/System/Windows/Forms/BinaryFormat/ListTests.cs @@ -38,7 +38,7 @@ public void BinaryFormattedObject_ParsePrimitivesArrayList(object value) systemClass.MemberTypeInfo[0].Should().Be((BinaryType.ObjectArray, null)); ArraySingleObject array = (ArraySingleObject)format[2]; - MemberPrimitiveTyped primitve = (MemberPrimitiveTyped)array[0]; + MemberPrimitiveTyped primitve = (MemberPrimitiveTyped)array[0]!; primitve.Value.Should().Be(value); } @@ -57,7 +57,7 @@ public void BinaryFormattedObject_ParseStringArrayList() systemClass.MemberTypeInfo[0].Should().Be((BinaryType.ObjectArray, null)); ArraySingleObject array = (ArraySingleObject)format[2]; - BinaryObjectString binaryString = (BinaryObjectString)array[0]; + BinaryObjectString binaryString = (BinaryObjectString)array[0]!; binaryString.Value.Should().Be("JarJar"); } diff --git a/src/System.Windows.Forms/src/System/Resources/ResXDataNode.cs b/src/System.Windows.Forms/src/System/Resources/ResXDataNode.cs index 563cf1f8c8e..16a565057c7 100644 --- a/src/System.Windows.Forms/src/System/Resources/ResXDataNode.cs +++ b/src/System.Windows.Forms/src/System/Resources/ResXDataNode.cs @@ -427,7 +427,7 @@ Type ResolveTypeName(string typeName) try { - BinaryFormattedObject format = new(stream, leaveOpen: true); + BinaryFormattedObject format = new(stream); if (format.TryGetObject(out object? value)) { return value; diff --git a/src/System.Windows.Forms/src/System/Windows/Forms/ActiveX/AxHost.PropertyBagStream.cs b/src/System.Windows.Forms/src/System/Windows/Forms/ActiveX/AxHost.PropertyBagStream.cs index 20fca66c990..5f1e42e7419 100644 --- a/src/System.Windows.Forms/src/System/Windows/Forms/ActiveX/AxHost.PropertyBagStream.cs +++ b/src/System.Windows.Forms/src/System/Windows/Forms/ActiveX/AxHost.PropertyBagStream.cs @@ -23,7 +23,7 @@ internal PropertyBagStream(Stream stream) long position = stream.Position; try { - BinaryFormattedObject format = new(stream, leaveOpen: true); + BinaryFormattedObject format = new(stream); if (format.TryGetPrimitiveHashtable(out _bag!)) { return; diff --git a/src/System.Windows.Forms/src/System/Windows/Forms/ActiveX/Control.ActiveXImpl.PropertyBagStream.cs b/src/System.Windows.Forms/src/System/Windows/Forms/ActiveX/Control.ActiveXImpl.PropertyBagStream.cs index 29d2b891d94..fe8b942de09 100644 --- a/src/System.Windows.Forms/src/System/Windows/Forms/ActiveX/Control.ActiveXImpl.PropertyBagStream.cs +++ b/src/System.Windows.Forms/src/System/Windows/Forms/ActiveX/Control.ActiveXImpl.PropertyBagStream.cs @@ -22,7 +22,7 @@ private class PropertyBagStream : IPropertyBag.Interface, IManagedWrapper ComboBoxItemAccessibleObject_ScrollIntoView_ { int itemsCount = 41; - for (int index = 0; index < itemsCount; index++) + for (int index = 0; index < itemsCount; index += 10) { yield return new object[] { comboBoxStyle, scrollingDown, index, itemsCount }; } diff --git a/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/AccessibleObjects/ToolStripMenuItem.ToolStripMenuItemAccessibleObjectTests.cs b/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/AccessibleObjects/ToolStripMenuItem.ToolStripMenuItemAccessibleObjectTests.cs index c2651e4bee9..742eaf0b322 100644 --- a/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/AccessibleObjects/ToolStripMenuItem.ToolStripMenuItemAccessibleObjectTests.cs +++ b/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/AccessibleObjects/ToolStripMenuItem.ToolStripMenuItemAccessibleObjectTests.cs @@ -36,7 +36,7 @@ public void ToolStripMenuItemAccessibleObject_InvokePattern_Invoke_NoPopUpDialog } [ActiveIssue("https://github.com/dotnet/winforms/issues/10244")] - [WinFormsFact] + [WinFormsFact(Skip = "https://github.com/dotnet/winforms/issues/10244")] [SkipOnArchitecture(TestArchitectures.X86 | TestArchitectures.X64, "InvokePattern.Invoke blocks on a menu item")] public void ToolStripMenuItemAccessibleObject_InvokePattern_Invoke_WithPopUpDialog() diff --git a/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/BinaryFormat/WinFormsBinaryFormattedObjectTests.cs b/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/BinaryFormat/WinFormsBinaryFormattedObjectTests.cs index e5825b0ada4..e2ec9f07de1 100644 --- a/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/BinaryFormat/WinFormsBinaryFormattedObjectTests.cs +++ b/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/BinaryFormat/WinFormsBinaryFormattedObjectTests.cs @@ -37,7 +37,7 @@ public void BinaryFormattedObject_Bitmap_RoundTrip() WinFormsBinaryFormatWriter.WriteBitmap(stream, bitmap); stream.Position = 0; - BinaryFormattedObject binary = new(stream, leaveOpen: true); + BinaryFormattedObject binary = new(stream); binary.TryGetBitmap(out object? result).Should().BeTrue(); using Bitmap deserialized = result.Should().BeOfType().Which; @@ -99,7 +99,7 @@ public void BinaryFormattedObject_ImageListStreamer_RoundTrip() using MemoryStream memoryStream = new(); WinFormsBinaryFormatWriter.WriteImageListStreamer(memoryStream, stream); memoryStream.Position = 0; - BinaryFormattedObject binary = new(memoryStream, leaveOpen: true); + BinaryFormattedObject binary = new(memoryStream); binary.TryGetImageListStreamer(out object? result).Should().BeTrue(); using ImageListStreamer deserialized = result.Should().BeOfType().Which; diff --git a/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/MonthCalendar.HitTestInfoTests.cs b/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/MonthCalendar.HitTestInfoTests.cs new file mode 100644 index 00000000000..71903625527 --- /dev/null +++ b/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/MonthCalendar.HitTestInfoTests.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using static System.Windows.Forms.MonthCalendar; +using Point = System.Drawing.Point; + +namespace System.Windows.Forms.Tests; + +public class MonthCalendarHitTestInfoTests +{ + [WinFormsTheory] + [InlineData(HitArea.Date, true, "2022/01/01")] + [InlineData(HitArea.WeekNumbers, true, "2022/01/01")] + [InlineData(HitArea.Nowhere, false, null)] + public void HitTestInfo_Constructor_SetsPropertiesCorrectly(HitArea hitArea, bool hasDateTime, string dateTimeStr) + { + Point point = new(5, 5); + DateTime? time = hasDateTime ? DateTime.Parse(dateTimeStr) : null; + + HitTestInfo hitTestInfo = time.HasValue + ? new HitTestInfo(point, hitArea, time.Value) + : new HitTestInfo(point, hitArea); + + hitTestInfo.Point.Should().Be(point); + hitTestInfo.HitArea.Should().Be(hitArea); + hitTestInfo.Time.Should().Be(time ?? DateTime.MinValue); + } + + [WinFormsTheory] + [InlineData(HitArea.Date, true)] + [InlineData(HitArea.WeekNumbers, true)] + [InlineData(HitArea.Nowhere, false)] + public void HitTestInfo_HitAreaHasValidDateTime_ReturnsExpectedResult(HitArea hitArea, bool expectedResult) + { + bool result = HitTestInfo.HitAreaHasValidDateTime(hitArea); + + result.Should().Be(expectedResult); + } + + [WinFormsTheory] + [InlineData(null)] + public void HitTestInfo_Constructor_WithNullDateTime_ThrowsException(DateTime? time) + { + Point point = new(5, 5); + HitArea hitArea = HitArea.Date; + + Action action = () => new HitTestInfo(point, hitArea, time!.Value); + + action.Should().Throw(); + } +} diff --git a/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/MonthCalendarTests.cs b/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/MonthCalendarTests.cs index 5b2250639a1..b4434cab384 100644 --- a/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/MonthCalendarTests.cs +++ b/src/System.Windows.Forms/tests/UnitTests/System/Windows/Forms/MonthCalendarTests.cs @@ -5,6 +5,7 @@ using System.Drawing; using System.Globalization; using System.Windows.Forms.TestUtilities; +using static System.Windows.Forms.MonthCalendar; using Point = System.Drawing.Point; using Size = System.Drawing.Size; @@ -4157,7 +4158,7 @@ public void MonthCalendar_SetSelectionRange_DateGreaterThanMaxDate_ThrowsArgumen public static IEnumerable MonthCalendar_FillMonthDayStates_ReturnsExpected_TestData() { - // This test set of dates is designed for a specifict test case: + // This test set of dates is designed for a specific test case: // when a calendar has 12 fully visible months + 2 not fully visible. // This test calendar has (08/29/2021 - 09/10/2022) dates range. @@ -4253,6 +4254,203 @@ public void SelectionRange_CopyConstructor_CopiesStartAndEndDatesCorrectly() copiedRange.End.Should().Be(endDate); } + [WinFormsFact] + public void GetFocused_Returns_FocusedCellAccessibleObject() + { + using MonthCalendar monthCalendar = new(); + + MonthCalendarAccessibleObject accessibleObject = new(monthCalendar); + var result = accessibleObject.GetFocused(); + + result.Should().Be(accessibleObject.FocusedCell); + } + + [WinFormsFact] + public void MonthCalendarAccessibleObject_Help_ReturnsExpected() + { + using MonthCalendar monthCalendar = new(); + + monthCalendar.CreateControl(); + MonthCalendarAccessibleObject accessibleObject = new(monthCalendar); + string expectedHelp = "MonthCalendar(Control)"; + + accessibleObject.Help.Should().Be(expectedHelp); + } + + [WinFormsTheory] + [InlineData("42")] + public void CalendarWeekNumberCellAccessibleObject_Name_ReturnsExpected(string weekNumber) + { + using MonthCalendar monthCalendar = new(); + + monthCalendar.CreateControl(); + MonthCalendarAccessibleObject monthCalendarAccessibleObject = new(monthCalendar); + CalendarAccessibleObject calendarAccessibleObject = new(monthCalendarAccessibleObject, 1, "Main Calendar"); + CalendarBodyAccessibleObject calendarBodyAccessibleObject = new(calendarAccessibleObject, monthCalendarAccessibleObject, 1); + CalendarRowAccessibleObject calendarRowAccessibleObject = new(calendarBodyAccessibleObject, monthCalendarAccessibleObject, 1, 1); + + CalendarWeekNumberCellAccessibleObject accessibleObject = new( + calendarRowAccessibleObject, + calendarBodyAccessibleObject, + monthCalendarAccessibleObject, + calendarIndex: 0, + rowIndex: 0, + columnIndex: 0, + weekNumber); + + string name = accessibleObject.Name; + + name.Should().Be($"Week {weekNumber}"); + } + + [WinFormsFact] + public void CalendarTodayLinkAccessibleObject_Bounds_ReturnsExpected() + { + using MonthCalendar monthCalendar = new(); + + monthCalendar.CreateControl(); + var controlAccessibleObject = (MonthCalendarAccessibleObject)monthCalendar.AccessibilityObject; + CalendarTodayLinkAccessibleObject todayLinkAccessibleObject = new(controlAccessibleObject); + Rectangle actual = todayLinkAccessibleObject.Bounds; + Rectangle expected = controlAccessibleObject.GetCalendarPartRectangle(MCGRIDINFO_PART.MCGIP_FOOTER); + + actual.Should().Be(expected); + } + + [WinFormsFact] + public void CalendarPreviousButtonAccessibleObject_Bounds_ReturnsExpected() + { + using MonthCalendar monthCalendar = new(); + + monthCalendar.CreateControl(); + var controlAccessibleObject = (MonthCalendarAccessibleObject)monthCalendar.AccessibilityObject; + CalendarPreviousButtonAccessibleObject previousButtonAccessibleObject = new(controlAccessibleObject); + Rectangle bounds = previousButtonAccessibleObject.Bounds; + Rectangle expectedBounds = new(13, 42, 16, 16); + + bounds.Should().Be(expectedBounds); + } + + private (CalendarRowAccessibleObject, CalendarCellAccessibleObject) CreateCalendarObjects(MonthCalendar control, int calendarIndex = 0, int rowIndex = 0, int columnIndex = 0) + { + MonthCalendarAccessibleObject controlAccessibleObject = (MonthCalendarAccessibleObject)control.AccessibilityObject; + CalendarAccessibleObject calendarAccessibleObject = new(controlAccessibleObject, calendarIndex, "Main Calendar"); + CalendarBodyAccessibleObject bodyAccessibleObject = new(calendarAccessibleObject, controlAccessibleObject, calendarIndex); + CalendarRowAccessibleObject rowAccessibleObject = new(bodyAccessibleObject, controlAccessibleObject, calendarIndex, rowIndex); + CalendarCellAccessibleObject cellAccessibleObject = new(rowAccessibleObject, bodyAccessibleObject, controlAccessibleObject, calendarIndex, rowIndex, columnIndex); + + return (rowAccessibleObject, cellAccessibleObject); + } + + [WinFormsFact] + public void CalendarRowAccessibleObject_Bounds_ReturnsExpected() + { + using MonthCalendar monthCalendar = new(); + + monthCalendar.CreateControl(); + var controlAccessibleObject = (MonthCalendarAccessibleObject)monthCalendar.AccessibilityObject; + var (rowAccessibleObject, _) = CreateCalendarObjects(monthCalendar); + Rectangle actual = rowAccessibleObject.Bounds; + Rectangle expected = controlAccessibleObject.GetCalendarPartRectangle(MCGRIDINFO_PART.MCGIP_CALENDARROW, 0, 0); + + actual.Should().Be(expected); + } + + [WinFormsTheory] + [InlineData(0, "Week 23, Monday")] + [InlineData(1, null)] + public void CalendarCellAccessibleObject_Description_ReturnsExpected(int view, string expected) + { + using MonthCalendar monthCalendar = new(); + + monthCalendar.CreateControl(); + monthCalendar.FirstDayOfWeek = Day.Monday; + monthCalendar.SelectionStart = new DateTime(2021, 6, 16); + PInvoke.SendMessage(monthCalendar, PInvoke.MCM_SETCURRENTVIEW, 0, view); + + var (_, cellAccessibleObject) = CreateCalendarObjects(monthCalendar); + + cellAccessibleObject.Description.Should().Be(expected); + } + + [WinFormsFact] + public void CalendarCellAccessibleObject_Select_SetsSelectionRange() + { + using MonthCalendar monthCalendar = new(); + + monthCalendar.CreateControl(); + var (_, cellAccessibleObject) = CreateCalendarObjects(monthCalendar); + + DateTime startDate = new(2022, 10, 1); + DateTime endDate = new(2022, 10, 7); + cellAccessibleObject.TestAccessor().Dynamic._dateRange = new SelectionRange(startDate, endDate); + + cellAccessibleObject.Select(AccessibleSelection.TakeSelection); + + monthCalendar.SelectionStart.Should().Be(startDate); + monthCalendar.SelectionEnd.Should().Be(endDate); + } + + [WinFormsTheory] + [InlineData(AccessibleStates.Focusable | AccessibleStates.Selectable | AccessibleStates.Selected)] + public void CalendarCellAccessibleObject_State_ReturnsExpected(AccessibleStates expectedState) + { + using MonthCalendar monthCalendar = new(); + + monthCalendar.CreateControl(); + var controlAccessibleObject = (MonthCalendarAccessibleObject)monthCalendar.AccessibilityObject; + var (_, cellAccessibleObject) = CreateCalendarObjects(monthCalendar); + + bool shouldFocus = expectedState.HasFlag(AccessibleStates.Focused); + bool selectExactRange = expectedState.HasFlag(AccessibleStates.Selected); + + if (shouldFocus) + { + monthCalendar.Focus(); + } + + if (selectExactRange) + { + monthCalendar.SetSelectionRange(cellAccessibleObject.DateRange.Start, cellAccessibleObject.DateRange.End); + } + else + { + monthCalendar.SetSelectionRange(cellAccessibleObject.DateRange.Start.AddDays(-1), cellAccessibleObject.DateRange.End.AddDays(1)); + } + + cellAccessibleObject.State.Should().Be(expectedState); + } + + [WinFormsTheory] + [InlineData(AccessibleSelection.AddSelection)] + [InlineData(AccessibleSelection.RemoveSelection)] + [InlineData(AccessibleSelection.TakeSelection)] + public void CalendarCellAccessibleObject_Select_Invoke_SetsSelectionRange(AccessibleSelection selectionFlag) + { + using MonthCalendar monthCalendar = new(); + monthCalendar.CreateControl(); + var (_, cellAccessibleObject) = CreateCalendarObjects(monthCalendar); + + DateTime expectedStart = cellAccessibleObject.DateRange.Start; + DateTime expectedEnd = cellAccessibleObject.DateRange.End; + + cellAccessibleObject.Select(selectionFlag); + monthCalendar.SelectionStart.Should().Be(expectedStart); + monthCalendar.SelectionEnd.Should().Be(expectedEnd); + } + + [WinFormsFact] + public void CalendarCellAccessibleObject_Role_ReturnsExpected() + { + using MonthCalendar monthCalendar = new(); + + monthCalendar.CreateControl(); + var controlAccessibleObject = (MonthCalendarAccessibleObject)monthCalendar.AccessibilityObject; + var (_, cellAccessibleObject) = CreateCalendarObjects(monthCalendar); + + cellAccessibleObject.Role.Should().Be(AccessibleRole.Cell); + } + private class SubMonthCalendar : MonthCalendar { public new bool CanEnableIme => base.CanEnableIme;