Integrate infra changes from nightly (#1277)

.dockerignore

@@ -4,3 +4,4 @@
 **/.vscode
 **/.vs
 .dotnet
+.Microsoft.DotNet.ImageBuilder

build-and-test.ps1

@@ -4,13 +4,13 @@ param(
     [string]$VersionFilter = "*",
     [string]$OSFilter = "*",
     [string]$ArchitectureFilter = "amd64",
-    [string]$ImageBuilderCustomArgs,
+    [string]$OptionalImageBuilderArgs,
     [switch]$SkipTesting = $false
 )
 
 & ./eng/common/build-and-test.ps1 `
     -VersionFilter $VersionFilter `
     -OSFilter $OSFilter `
     -ArchitectureFilter $ArchitectureFilter `
-    -ImageBuilderCustomArgs $ImageBuilderCustomArgs `
+    -OptionalImageBuilderArgs $OptionalImageBuilderArgs `
     -SkipTesting:$SkipTesting

eng/Get-TagsDocumentation.ps1

@@ -17,10 +17,16 @@ if (!$Branch) {
         $coreRepoName = "core"
     }
 }
+
 $gitRepo = "https://github.com/dotnet/dotnet-docker"
 
-& $PSScriptRoot/common/Invoke-ReadmeGeneration.ps1 dotnet/$coreRepoName/runtime-deps README.runtime-deps.md manifest.json $Branch $gitRepo
-& $PSScriptRoot/common/Invoke-ReadmeGeneration.ps1 dotnet/$coreRepoName/runtime README.runtime.md manifest.json $Branch $gitRepo -ReuseImageBuilderImage
-& $PSScriptRoot/common/Invoke-ReadmeGeneration.ps1 dotnet/$coreRepoName/aspnet README.aspnet.md manifest.json $Branch $gitRepo -ReuseImageBuilderImage
-& $PSScriptRoot/common/Invoke-ReadmeGeneration.ps1 dotnet/$coreRepoName/sdk README.sdk.md manifest.json $Branch $gitRepo -ReuseImageBuilderImage
-& $PSScriptRoot/common/Invoke-ReadmeGeneration.ps1 dotnet/core/samples README.samples.md manifest.samples.json master $gitRepo -ReuseImageBuilderImage
+& $PSScriptRoot/common/Invoke-ReadmeGeneration.ps1 `
+    dotnet/$coreRepoName/runtime-deps README.runtime-deps.md manifest.json $gitRepo $Branch
+& $PSScriptRoot/common/Invoke-ReadmeGeneration.ps1 `
+    dotnet/$coreRepoName/runtime README.runtime.md manifest.json $gitRepo $Branch -ReuseImageBuilderImage
+& $PSScriptRoot/common/Invoke-ReadmeGeneration.ps1 `
+    dotnet/$coreRepoName/aspnet README.aspnet.md manifest.json $gitRepo $Branch -ReuseImageBuilderImage
+& $PSScriptRoot/common/Invoke-ReadmeGeneration.ps1 `
+    dotnet/$coreRepoName/sdk README.sdk.md manifest.json $gitRepo $Branch -ReuseImageBuilderImage
+& $PSScriptRoot/common/Invoke-ReadmeGeneration.ps1 `
+    dotnet/core/samples README.samples.md manifest.samples.json $gitRepo -ReuseImageBuilderImage

eng/common/Invoke-CleanupDocker.ps1

@@ -5,11 +5,17 @@ docker ps -a -q | ForEach-Object { docker rm -f $_ }
 
 docker volume prune -f
 
-# Windows base images are large, preserve them to avoid the overhead of pulling each time.
-docker images |
+# Preserve the Windows base images and the common eng infra images (e.g. ImageBuilder)
+# to avoid the expense of having to repull continuously.
+$engInfraImages = Get-Content ./eng/common/templates/variables/docker-images.yml |
+    Where-Object { $_.Trim() -notlike 'variables:' } |
+    ForEach-Object { $_.Split(' ', [System.StringSplitOptions]::RemoveEmptyEntries)[1] }
+
+docker images --format "{{.Repository}}:{{.Tag}} {{.ID}}"|
     Where-Object {
-        -Not ($_.StartsWith("mcr.microsoft.com/windows")`
-        -Or $_.StartsWith("REPOSITORY ")) } |
-    ForEach-Object { $_.Split(' ', [System.StringSplitOptions]::RemoveEmptyEntries)[2] } |
+        $localImage = $_
+        -Not ($localImage.StartsWith("mcr.microsoft.com/windows")`
+            -Or ($engInfraImages.Where({ $localImage.StartsWith($_) }, 'First').Count -gt 0)) } |
+    ForEach-Object { $_.Split(' ', [System.StringSplitOptions]::RemoveEmptyEntries)[1] } |
     Select-Object -Unique |
     ForEach-Object { docker rmi -f $_ }

eng/common/Invoke-ReadmeGeneration.ps1

@@ -3,8 +3,8 @@ param (
     [string] $Repo,
     [string] $ReadmePath,
     [string] $Manifest,
-    [string] $Branch,
     [string] $GitRepo,
+    [string] $Branch = "master",
     [switch] $ReuseImageBuilderImage
 )
 

eng/common/build-and-test.ps1

@@ -4,23 +4,47 @@ param(
     [string]$VersionFilter = "*",
     [string]$OSFilter = "*",
     [string]$ArchitectureFilter = "amd64",
-    [string]$ImageBuilderCustomArgs,
-    [switch]$SkipTesting = $false
+    [string]$OptionalImageBuilderArgs,
+    [string]$OptionalTestArgs,
+    [switch]$SkipTesting = $false,
+    [switch]$ExcludeArchitecture = $false
 )
 
 Set-StrictMode -Version Latest
 $ErrorActionPreference = 'Stop'
 
+function Log {
+    param ([string] $Message)
+
+    Write-Output $Message
+}
+
+function Exec {
+    param ([string] $Cmd)
+
+    Log "Executing: '$Cmd'"
+    Invoke-Expression $Cmd
+    if ($LASTEXITCODE -ne 0) {
+        throw "Failed: '$Cmd'"
+    }
+}
+
 pushd $PSScriptRoot/../..
 try {
-    ./eng/common/Invoke-ImageBuilder.ps1 "build --path '$VersionFilter/*/$OSFilter/$ArchitectureFilter' $ImageBuilderCustomArgs"
+    $pathFilter = "$VersionFilter/*/$OSFilter"
+    if (-not $ExcludeArchitecture) {
+        $pathFilter += "/$ArchitectureFilter"
+    }
+
+    ./eng/common/Invoke-ImageBuilder.ps1 "build --path '$pathFilter' $OptionalImageBuilderArgs"
 
     if (-not $SkipTesting) {
         if (Test-Path ./tests/run-tests.ps1) {
-            ./tests/run-tests.ps1 `
-                -VersionFilter $VersionFilter `
-                -ArchitectureFilter $ArchitectureFilter `
-                -OSFilter $OSFilter -IsLocalRun
+            if (-not $ExcludeArchitecture) {
+                $OptionalTestArgs += " -ArchitectureFilter $ArchitectureFilter"
+            }
+
+            Exec "./tests/run-tests.ps1 -VersionFilter $VersionFilter -OSFilter $OSFilter $OptionalTestArgs"
         } else {
           Write-Warning "Test script file './tests/run-tests.ps1' not found."
         }

eng/common/templates/jobs/build-images.yml

@@ -20,6 +20,7 @@ jobs:
       imageBuilderBuildArgs: --registry-override $(acr.server) --repo-prefix $(stagingRepoPrefix) --push --username $(acr.userName) --password $(BotAccount-dotnet-docker-acr-bot-password) $(imageBuilder.queueArgs)
     ${{ if eq(variables['System.TeamProject'], 'public') }}:
       imageBuilderBuildArgs: $(imageBuilder.queueArgs)
+    imageBuilderDockerRunExtraOptions: $(build.imageBuilderDockerRunExtraOptions)
   steps:
   # This script is necessary to workaround there not being a matching architecture when pulling images
   # on an aarch64 machine. By using the multi-arch tag for the images referenced in the sample

eng/common/templates/jobs/publish.yml

@@ -63,6 +63,18 @@ jobs:
       --git-repo versions
       --git-branch master
       --git-path build-info/docker/image-info.json
+    displayName: Publish Image Info (legacy)
+    condition: and(succeeded(), eq(variables['publishRepoPrefix'], 'public/'))
+  - script: >
+      $(runImageBuilderCmd) publishImageInfo
+      $(dotnetBot-userName)
+      $(dotnetBot-email)
+      $(dotnet-bot-user-repo-adminrepohook-pat)
+      $(artifactsPath)/image-info.json
+      --git-owner dotnet
+      --git-repo versions
+      --git-branch master
+      --git-path build-info/docker/image-info.$(Build.Repository.Name)-$(publicSourceBranch).json
     displayName: Publish Image Info
     condition: and(succeeded(), eq(variables['publishRepoPrefix'], 'public/'))
   - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0

eng/common/templates/jobs/test-images-linux-client.yml

@@ -3,13 +3,15 @@ parameters:
   pool: {}
   matrix: {}
   useRemoteDockerServer: false
+  testJobTimeout: 60
 jobs:
 - job: ${{ parameters.name }}
   condition: and(succeeded(), ${{ parameters.matrix }})
   dependsOn: GenerateTestMatrix
   pool: ${{ parameters.pool }}
   strategy:
     matrix: $[ ${{ parameters.matrix }} ]
+  timeoutInMinutes: ${{ parameters.testJobTimeout }}
   steps:
   - template: ../steps/test-images-linux-client.yml
     parameters:

eng/common/templates/jobs/test-images-windows-client.yml

@@ -2,12 +2,14 @@ parameters:
   name: null
   pool: {}
   matrix: {}
+  testJobTimeout: 60
 jobs:
 - job: ${{ parameters.name }}
   condition: and(succeeded(), ${{ parameters.matrix }})
   dependsOn: GenerateTestMatrix
   pool: ${{ parameters.pool }}
   strategy:
     matrix: $[ ${{ parameters.matrix }} ]
+  timeoutInMinutes: ${{ parameters.testJobTimeout }}
   steps:
   - template: ../steps/test-images-windows-client.yml

eng/common/templates/stages/build-test-publish-repo.yml

@@ -2,11 +2,17 @@ parameters:
   buildMatrixType: platformDependencyGraph
   testMatrixType: platformVersionedOs
   customBuildLegGrouping: ""
+
   linuxAmdBuildJobTimeout: 60
   linuxArmBuildJobTimeout: 60
   windowsAmdBuildJobTimeout: 60
   windowsArmBuildJobTimeout: 60
 
+  linuxAmdTestJobTimeout: 60
+  linuxArmTestJobTimeout: 60
+  windowsAmdTestJobTimeout: 60
+  windowsArmTestJobTimeout: 60
+
 stages:
 
 ################################################################################
@@ -197,6 +203,7 @@ stages:
         pool: # linuxAmd64Pool
           name: Hosted Ubuntu 1604
         matrix: dependencies.GenerateTestMatrix.outputs['${{ format('matrix.{0}MatrixLinuxAmd64', parameters.testMatrixType) }}']
+        testJobTimeout: ${{ parameters.linuxAmdTestJobTimeout }}
     - template: ../jobs/test-images-linux-client.yml
       parameters:
         name: Test_Linux_arm64v8
@@ -207,6 +214,7 @@ stages:
           - RemoteDockerServerOS -equals linux
           - RemoteDockerServerArch -equals aarch64
         matrix: dependencies.GenerateTestMatrix.outputs['${{ format('matrix.{0}MatrixLinuxArm64v8', parameters.testMatrixType) }}']
+        testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }}
         useRemoteDockerServer: true
     - template: ../jobs/test-images-linux-client.yml
       parameters:
@@ -218,6 +226,7 @@ stages:
           - RemoteDockerServerOS -equals linux
           - RemoteDockerServerArch -equals aarch64
         matrix: dependencies.GenerateTestMatrix.outputs['${{ format('matrix.{0}MatrixLinuxArm32v7', parameters.testMatrixType) }}']
+        testJobTimeout: ${{ parameters.linuxArmTestJobTimeout }}
         useRemoteDockerServer: true
     - template: ../jobs/test-images-windows-client.yml
       parameters:
@@ -226,13 +235,15 @@ stages:
           name: DotNetCore-Docker
           demands: VSTS_OS -equals Windows_Server_2016_Data_Center_RS4
         matrix: dependencies.GenerateTestMatrix.outputs['${{ format('matrix.{0}MatrixNanoserver1803Amd64', parameters.testMatrixType) }}']
+        testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }}
     - template: ../jobs/test-images-windows-client.yml
       parameters:
         name: Test_NanoServer1809_amd64
         pool: # windows1809Amd64
           name: DotNetCore-Docker
           demands: VSTS_OS -equals Windows_Server_2019_Data_Center_RS5
         matrix: dependencies.GenerateTestMatrix.outputs['${{ format('matrix.{0}MatrixNanoserver1809Amd64', parameters.testMatrixType) }}']
+        testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }}
     - template: ../jobs/test-images-linux-client.yml
       parameters:
         name: Test_NanoServer1809_arm32
@@ -243,6 +254,7 @@ stages:
           - RemoteDockerServerOS -equals nanoserver-1809
           - RemoteDockerServerArch -equals arm32
         matrix: dependencies.GenerateTestMatrix.outputs['${{ format('matrix.{0}MatrixNanoserver1809Arm32', parameters.testMatrixType) }}']
+        testJobTimeout: ${{ parameters.windowsArmTestJobTimeout }}
         useRemoteDockerServer: true
     - template: ../jobs/test-images-windows-client.yml
       parameters:
@@ -252,6 +264,7 @@ stages:
           name: DotNetCore-Docker
           demands: VSTS_OS -equals Windows_Server_2019_Data_Center_1903
         matrix: dependencies.GenerateTestMatrix.outputs['${{ format('matrix.{0}MatrixNanoserver1903Amd64', parameters.testMatrixType) }}']
+        testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }}
     - template: ../jobs/test-images-windows-client.yml
       parameters:
         name: Test_WindowsServerCoreLtsc2016_amd64
@@ -260,6 +273,7 @@ stages:
           name: DotNetCore-Docker
           demands: VSTS_OS -equals Windows_Server_2016_Data_Center_with_Containers
         matrix: dependencies.GenerateTestMatrix.outputs['${{ format('matrix.{0}MatrixWindowsservercoreLtsc2016Amd64', parameters.testMatrixType) }}']
+        testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }}
     - template: ../jobs/test-images-windows-client.yml
       parameters:
         name: Test_WindowsServerCore1803_amd64
@@ -268,6 +282,7 @@ stages:
           name: DotNetCore-Docker
           demands: VSTS_OS -equals Windows_Server_2016_Data_Center_RS4
         matrix: dependencies.GenerateTestMatrix.outputs['${{ format('matrix.{0}MatrixWindowsservercore1803Amd64', parameters.testMatrixType) }}']
+        testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }}
     - template: ../jobs/test-images-windows-client.yml
       parameters:
         name: Test_WindowsServerCoreLtsc2019_amd64
@@ -276,6 +291,7 @@ stages:
           name: DotNetCore-Docker
           demands: VSTS_OS -equals Windows_Server_2019_Data_Center_RS5
         matrix: dependencies.GenerateTestMatrix.outputs['${{ format('matrix.{0}MatrixWindowsservercoreLtsc2019Amd64', parameters.testMatrixType) }}']
+        testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }}
     - template: ../jobs/test-images-windows-client.yml
       parameters:
         name: Test_WindowsServerCore1903_amd64
@@ -284,6 +300,7 @@ stages:
           name: DotNetCore-Docker
           demands: VSTS_OS -equals Windows_Server_2019_Data_Center_1903
         matrix: dependencies.GenerateTestMatrix.outputs['${{ format('matrix.{0}MatrixWindowsservercore1903Amd64', parameters.testMatrixType) }}']
+        testJobTimeout: ${{ parameters.windowsAmdTestJobTimeout }}
 
   ################################################################################
   # Publish Images

eng/common/templates/steps/cleanup-docker-linux.yml

@@ -11,12 +11,12 @@ steps:
   # owned by this build.
   ################################################################################
 - ${{ if and(eq(parameters.cleanupRemoteDockerServer, 'true'), eq(parameters.initCleanup, 'false')) }}:
-  - script: docker run --rm $(dockerArmRunArgs) --entrypoint docker $(dockerClientImage) system prune -f --volumes
+  - script: docker run --rm $(dockerArmRunOptions) --entrypoint docker $(dockerClientImage) system prune -f --volumes
     displayName: Cleanup Remote Docker Server (basic)
     condition: always()
     continueOnError: true
   - script: >
-      docker run --rm $(dockerArmRunArgs) --entrypoint /bin/bash $(dockerClientImage) -c
+      docker run --rm $(dockerArmRunOptions) --entrypoint /bin/bash $(dockerClientImage) -c
       'docker rmi -f $(docker images --format "{{.Repository}}:{{.Tag}}" | grep -v -e ^arm32v7/ -e ^arm64v8/ -e ^mcr\\.microsoft\\.com/windows)'
     displayName: Cleanup Remote Docker Server (images)
     condition: always()

eng/common/templates/steps/init-docker-linux.yml

@@ -29,12 +29,12 @@ steps:
   ################################################################################
 - ${{ if eq(parameters.setupRemoteDockerServer, 'true') }}:
   - script: >
-      echo "##vso[task.setvariable variable=dockerArmRunArgs]
+      echo "##vso[task.setvariable variable=dockerArmRunOptions]
       -v $(DOCKER_CERT_PATH):/docker-certs
       -e DOCKER_CERT_PATH=/docker-certs
       -e DOCKER_TLS_VERIFY=1
       -e DOCKER_HOST=tcp://$(DOCKER_HOST_IP):2376"
-    displayName: Define dockerArmRunArgs Variable
+    displayName: Define dockerArmRunOptions Variable
 
   ################################################################################
   # Setup Image Builder (Optional)
@@ -54,7 +54,8 @@ steps:
       -v /var/run/docker.sock:/var/run/docker.sock
       -v $(Build.ArtifactStagingDirectory):$(artifactsPath)
       -w /repo
-      $(dockerArmRunArgs)
+      $(dockerArmRunOptions)
+      $(imageBuilderDockerRunExtraOptions)
       $(imageNames.imageBuilder.withrepo)"
     displayName: Define runImageBuilderCmd Variable
   - ${{ if eq(parameters.setupRemoteDockerServer, 'true') }}:

eng/common/templates/steps/test-images-linux-client.yml

@@ -26,7 +26,7 @@ steps:
     docker run -t -d
     -v /var/run/docker.sock:/var/run/docker.sock
     -v $(Build.ArtifactStagingDirectory):$(artifactsPath)
-    -w /repo $(dockerArmRunArgs)
+    -w /repo $(dockerArmRunOptions)
     -e RUNNING_TESTS_IN_CONTAINER=true 
     --name $(testRunner.container)
     $(imageNames.testRunner.withrepo)

eng/common/templates/variables/common.yml

@@ -7,6 +7,10 @@ variables:
   value: true
 - name: skipComponentGovernanceDetection
   value: true
+- name: build.imageBuilderDockerRunExtraOptions
+  value: ""
+- name: imageBuilderDockerRunExtraOptions
+  value: ""
 - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
   - group: DotNet-Docker-Common
   - group: DotNet-Docker-Secrets

eng/common/templates/variables/docker-images.yml

@@ -1,6 +1,6 @@
 variables:
-  imageNames.imageBuilder.linux: mcr.microsoft.com/dotnet-buildtools/image-builder:debian-20190807123248
-  imageNames.imageBuilder.windows: mcr.microsoft.com/dotnet-buildtools/image-builder:nanoserver-20190807123248
+  imageNames.imageBuilder.linux: mcr.microsoft.com/dotnet-buildtools/image-builder:debian-20190821150534
+  imageNames.imageBuilder.windows: mcr.microsoft.com/dotnet-buildtools/image-builder:nanoserver-20190821150534
   imageNames.imageBuilder.withrepo: imagebuilder-withrepo:$(Build.BuildId)-$(System.JobId)
-  imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-docker-testrunner-63f2145-20184325094343
+  imageNames.testRunner: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-slim-docker-testrunner-d61254f-20190807161111
   imageNames.testRunner.withrepo: testrunner-withrepo:$(Build.BuildId)-$(System.JobId)