azure-aks-ci.yml

parameters:
- name: workspace
  displayName: Pipeline Environment / Terraform Workspace
  type: string
  default: ci
  values:
  - ci
  - ci1
  - ci2
  - ci3
  - cd
  - cd1
  - cd2
  - cd3
- name: clear
  displayName: Clear state
  type: boolean
  default: false
- name: deploy
  displayName: Deploy applications
  type: boolean
  default: true
- name: destroy
  displayName: Destroy infrastructure
  type: string
  default: Always
  values:
  - Always
  - Never 
  - 'On failure'
  - 'On success'
- name: testReentrance
  displayName: Test Terraform re-entrance (apply twice)
  type: boolean
  default: true
- name: unpinTerraform
  displayName: Unpin Terraform version
  type: boolean
  default: false
- name: unpinTerraformProviders
  displayName: Unpin Terraform provider versions
  type: string
  default: No
  values:
  - No
  - Yes
  - Strategy
- name: useDefaultK8sVersion
  displayName: Use default Kubernetes version
  type: boolean
  default: true

name: $(Date:yyyyMMdd)$(Rev:.r)-$(Build.DefinitionVersion)-$(SourceBranchName)-${{ parameters.workspace }}-$(Build.BuildId)

trigger: none

pr:
  branches:
    include:
    - '*'
  paths:
    exclude:
    - '.devcontainer/**'  
    - 'visuals/**'  
    - '*.md'  

schedules:
- cron: '0 1 * * Mon,Wed,Fri'
  displayName: 'Bi-Nightly build (UTC)'
  # Run if there are no changes
  always: 'true'
  branches:
    include:
    - main

resources:
  repositories:
  - repository: azure-identity-scripts
    type: github
    endpoint: github.com # Service Connection name
    name: geekzter/azure-identity-scripts

variables:
- group: 'aks-ci'
- name: 'jobTimeOutMinutes'
  value: 180
- name: 'repository'
  value: 'azure-aks'
- name: AZURE_CORE_ONLY_SHOW_ERRORS
  value: 'true'
- name: AZURE_EXTENSION_USE_DYNAMIC_INSTALL
  value: 'yes_without_prompt'
- name: 'TF_IN_AUTOMATION'
  value: 'true'
- name: 'TF_INPUT'
  value: 0
- name: 'TF_WORKSPACE'
  value: ${{ parameters.workspace }}
- name: 'identityScriptDirectory'
  value: '$(Build.SourcesDirectory)/azure-identity-scripts/scripts/azure-devops'
- name: 'manifestDirectory'
  value: '$(Build.SourcesDirectory)/azure-aks/manifests'
- name: 'scriptDirectory'
  value: '$(Build.SourcesDirectory)/azure-aks/scripts'
- name: 'terraformDirectory'
  value: '$(Build.SourcesDirectory)/azure-aks/terraform'
- name: 'TF_VAR_run_id'
  value: '$(Build.BuildId)'

jobs:
- job: 'Provision'
  ${{ if and(eq(parameters.destroy, 'Always'),parameters.deploy) }}:
    displayName: 'Provision (${{ parameters.workspace }}), Deploy, Test & Destroy'
  ${{ if and(eq(parameters.destroy, 'Never'),parameters.deploy) }}:
    displayName: 'Provision (${{ parameters.workspace }}), Deploy & Test'
  ${{ if and(eq(parameters.destroy, 'On failure'),parameters.deploy) }}:
    displayName: 'Provision (${{ parameters.workspace }}), Deploy, Test & Destroy (${{ lower(parameters.destroy) }})'
  ${{ if and(eq(parameters.destroy, 'On success'),parameters.deploy) }}:
    displayName: 'Provision (${{ parameters.workspace }}), Deploy, Test & Destroy (${{ lower(parameters.destroy) }})'
  ${{ if and(eq(parameters.destroy, 'Always'),not(parameters.deploy)) }}:
    displayName: 'Provision (${{ parameters.workspace }}) & Destroy'
  ${{ if and(eq(parameters.destroy, 'Never'),not(parameters.deploy)) }}:
    displayName: 'Provision (${{ parameters.workspace }})'
  ${{ if and(eq(parameters.destroy, 'On failure'),not(parameters.deploy)) }}:
    displayName: 'Provision (${{ parameters.workspace }}) & Destroy (${{ lower(parameters.destroy) }})'
  ${{ if and(eq(parameters.destroy, 'On success'),not(parameters.deploy)) }}:
    displayName: 'Provision (${{ parameters.workspace }}) & Destroy (${{ lower(parameters.destroy) }})'
  condition: succeeded()
  timeoutInMinutes: $[ variables['jobTimeOutMinutes'] ]

  pool:
    name: '$(pool)'
    vmImage: $(vmImage)

  ${{ if or(eq(parameters.unpinTerraformProviders, 'Strategy'),not(eq(variables['Build.Reason'], 'Manual'))) }}:
    strategy:
      matrix:
        pinTerraformProviders:
          randomSeed: $(Build.BuildId)0
          resourceGroup: '$(TF_VAR_resource_prefix)-${{ parameters.workspace }}a-$(Build.BuildId)'
          substituteAlternateVariables: false
          terraformArtifactName: 'terraformPrimary'
          TF_VAR_resource_suffix: '$(Build.BuildId)'
          TF_WORKSPACE: '${{ parameters.workspace }}a'
          unpinTerraformProviders: ${{ lower(eq(parameters.unpinTerraformProviders, 'Yes')) }}
        unpinTerraformProviders:
          randomSeed: $(Build.BuildId)5
          resourceGroup: '$(TF_VAR_resource_prefix)-${{ parameters.workspace }}b-$(Build.BuildId)'
          substituteAlternateVariables: true
          terraformArtifactName: 'terraformAlternate'
          TF_VAR_resource_suffix: '$(Build.BuildId)'
          TF_WORKSPACE: '${{ parameters.workspace }}b'
          unpinTerraformProviders: ${{ lower(or(eq(parameters.unpinTerraformProviders, 'Yes'),eq(parameters.unpinTerraformProviders, 'Strategy'),ne(variables['Build.Reason'], 'Manual'))) }}
      maxParallel: 2

  variables:
    ${{ if not(or(eq(parameters.unpinTerraformProviders, 'Strategy'),not(eq(variables['Build.Reason'], 'Manual')))) }}:
      # Not runnig as strategy
      terraformArtifactName: 'terraformPrimary'
      ${{ if parameters.clear }}:  # Don't reset suffix if we want to keep existing resources
        TF_VAR_resource_suffix: '$(Build.BuildId)'
      TF_WORKSPACE: '${{ parameters.workspace }}'
      unpinTerraformProviders: ${{ eq(parameters.unpinTerraformProviders, 'Yes') }}
      randomSeed: $(Build.BuildId)
      resourceGroup: '$(TF_VAR_resource_prefix)-$(TF_WORKSPACE)-$(TF_VAR_resource_suffix)'

  workspace:
    clean: all

  steps:
    - checkout: self
    - checkout: azure-identity-scripts

    - ${{ if parameters.clear }}:
      - task: AzureCLI@2
        name: clear
        displayName: 'Remove conflicting resources from previous runs'
        inputs:
          azureSubscription: '$(subscriptionConnection)'
          scriptType: pscore
          scriptLocation: inlineScript
          inlineScript: |
            # Remove VNet peerings
            if ($env:PIPELINE_DEMO_AGENT_VIRTUAL_NETWORK_ID) {
              $peeringIDs = $(az network vnet show --ids $env:PIPELINE_DEMO_AGENT_VIRTUAL_NETWORK_ID --query "virtualNetworkPeerings[?starts_with(name,'k8s-${env:TF_WORKSPACE}-')].id" -o tsv 2>$null)
              if ($peeringIDs) {
                Write-Host "Removing virtual network peerings `"${peeringIDs}`"..."
                &{ # az writes information to stderr
                  $ErrorActionPreference = 'SilentlyContinue'
                  az resource delete --ids $peeringIDs 2>&1
                }
              } else {
                Write-Host "No virtual network peerings to remove"
              }
            }
          useGlobalConfig: true
          failOnStandardError: true
          workingDirectory: '$(terraformDirectory)'

    - ${{ if not(parameters.unpinTerraform) }}:
      - pwsh: |
          $terraformVersion = (Get-Content .terraform-version)
          Write-Host "##vso[task.setvariable variable=version;isOutput=true]${terraformVersion}"
          Copy-Item backend.tf.sample backend.tf
        name: terraformConfig
        displayName: 'Prepare Terraform config'
        workingDirectory: '$(terraformDirectory)'

    - ${{ if parameters.unpinTerraform }}:
      - pwsh: |
          (Get-Content ./provider.tf) -replace "required_version *= `" *(~>|=) +",'required_version = ">= ' | Out-File provider.tf
          Get-Content ./provider.tf
          Write-Host "##vso[task.setvariable variable=version;isOutput=true]latest"
          Copy-Item backend.tf.sample backend.tf
        name: terraformConfig
        displayName: 'Prepare Terraform config (latest version)'
        workingDirectory: '$(terraformDirectory)'

    - task: TerraformInstaller@1
      displayName: 'Install terraform'
      inputs:
        terraformVersion: '$(terraformConfig.version)'

    - ${{ if or(eq(parameters.unpinTerraformProviders, 'Yes'),eq(parameters.unpinTerraformProviders, 'Strategy'),not(eq(variables['Build.Reason'], 'Manual'))) }}:
      # Unpin version e.g. "= 2.56" -> "~> 2.56"
      - pwsh: |
          (Get-Content ./provider.tf) -replace " = `" *= +",' = "~> ' | Out-File provider.tf
          Get-Content ./provider.tf
          if (Test-Path .terraform.lock.hcl) {
            Remove-Item .terraform.lock.hcl -Force
          }
        displayName: 'Unpin Terraform provider versions'
        # condition required as '- ${{ if ' template expression is not evaluated when using a strategy
        condition: and(succeeded(), eq(variables['unpinTerraformProviders'],'true'))
        workingDirectory: '$(terraformDirectory)'

    - task: TerraformCLI@1
      displayName: 'Terraform init'
      inputs:
        command: 'init'
        workingDirectory: '$(terraformDirectory)'
        backendType: 'azurerm'
        backendServiceArm: '$(subscriptionConnection)'
        backendAzureRmResourceGroupName: '$(TF_STATE_RESOURCE_GROUP_NAME)'
        backendAzureRmStorageAccountName: '$(TF_STATE_STORAGE_ACCOUNT_NAME)'
        backendAzureRmContainerName: '$(TF_STATE_CONTAINER_NAME)'
        backendAzureRmKey: 'terraform.tfstate'
        allowTelemetryCollection: true

    - publish: $(terraformDirectory)
      displayName: 'Publish Terraform workspace'
      artifact: $(terraformArtifactName)

    - ${{ if parameters.clear }}:
      - task: AzureCLI@2
        name: cleanup
        displayName: 'Clear Terraform state'
        inputs:
          azureSubscription: '$(subscriptionConnection)'
          scriptType: pscore
          scriptLocation: inlineScript
          inlineScript: |
            $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1

            $terraformState = (terraform state pull | ConvertFrom-Json)
            if ($terraformState.resources) {
              Write-Host "Clearing Terraform state in workspace ${env:TF_WORKSPACE}..."
              $terraformState.outputs = New-Object PSObject # Empty output
              $terraformState.resources = @() # No resources
              $terraformState.serial++
              $terraformState | ConvertTo-Json | terraform state push -
            } else {
              Write-Host "No resources in Terraform state in workspace ${env:TF_WORKSPACE}..."
            }
            terraform state pull 
          addSpnToEnvironment: true
          useGlobalConfig: true
          failOnStandardError: true
          workingDirectory: '$(terraformDirectory)'

    - ${{ if not(parameters.clear) }}:
      - task: AzureCLI@2
        displayName: 'Restore kubeconfig'
        inputs:
          azureSubscription: '$(subscriptionConnection)'
          scriptType: pscore
          scriptLocation: inlineScript
          inlineScript: |
            if ((${env:system.debug} -eq "true") -or ($env:system_debug -eq "true") -or ($env:SYSTEM_DEBUG -eq "true")) {
              $DebugPreference       = "Continue"
              $InformationPreference = "Continue"
              $VerbosePreference     = "Continue"
              Set-PSDebug -Trace 2
            }
            ./kube_config.ps1
          addSpnToEnvironment: true
          useGlobalConfig: true
          failOnStandardError: true
          workingDirectory: '$(scriptDirectory)'

    - task: AzureCLI@2
      displayName: 'Prepare Terraform variables'
      inputs:
        azureSubscription: '$(subscriptionConnection)'
        scriptType: pscore
        scriptLocation: inlineScript
        inlineScript: |
          if ($${{ parameters.useDefaultK8sVersion }} -and !($env:TF_VAR_kubernetes_version -or $env:TF_VAR_KUBERNETES_VERSION)) {
            # Use Azure CLI default (Terraform may use a different one)
            az aks get-versions -l $(TF_VAR_location) --query "orchestrators[?default].orchestratorVersion" -o tsv | Set-Item env:TF_VAR_kubernetes_version
          }

          # Use pipeline agent VNet as network to peer from
          $env:TF_VAR_peer_network_id ??= $env:PIPELINE_DEMO_AGENT_VIRTUAL_NETWORK_ID

          if ($env:TF_VAR_ADDRESS_SPACE) {
            Remove-Item env:TF_VAR_ADDRESS_SPACE 2>$null # Override below
          }
          if ($${{ not(parameters.clear) }} -and (!((terraform output resource_suffix 2>&1) -match "Warning"))) {
            # Keep random values of previous runs
            $env:TF_VAR_address_space   = "$(terraform output -raw address_space 2>$null)"
            $env:TF_VAR_RESOURCE_SUFFIX = $null
            $env:TF_VAR_resource_suffix = "$(terraform output -raw resource_suffix 2>$null)"
          }
          # Set random CIDR (to reduce the risk of clashing VNet peerings with agent VNet)
          $env:TF_VAR_address_space   ??= "$([IPAddress]::Parse([String] (167772160 + (65536*(Get-Random -Minimum 0 -Maximum 255 -SetSeed $(randomSeed))))) | Select-Object -ExpandProperty IPAddressToString)/16"

          # List environment variables
          Get-ChildItem -Path Env: -Recurse -Include ARM_*,AZURE_*,PIPELINE_*,TF_* | Sort-Object -Property Name

          # Convert uppercased Terraform environment variables to .auto.tfvars file
          foreach ($tfvar in $(Get-ChildItem -Path Env: -Recurse -Include TF_VAR_*)) {
            $terraformVariableName = $tfvar.Name.Substring(7).ToLowerInvariant()
            $terraformVariableValue = $tfVar.Value

            if ($terraformVariableValue -imatch "^\W*(true|false|\[[^\]]*\]|\{[^\}]*\})\W*$") {
              # Boolean or List, write as-is
              Write-Output "${terraformVariableName} = ${terraformVariableValue}" | Out-File ci.auto.tfvars -Append -Force
            } else {
              Write-Output "${terraformVariableName} = `"${terraformVariableValue}`"" | Out-File ci.auto.tfvars -Append -Force
            }
          } 
          Write-Host "Contents of ci.auto.tfvars:"
          Get-Content ci.auto.tfvars

        useGlobalConfig: true
        failOnStandardError: true
        workingDirectory: '$(terraformDirectory)'

    - task: AzureCLI@2
      displayName: 'Terraform plan & apply'
      name: terraformApply
      inputs:
        azureSubscription: '$(subscriptionConnection)'
        scriptType: pscore
        scriptLocation: inlineScript
        inlineScript: |
          $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1
          ./deploy.ps1 -apply -force
        addSpnToEnvironment: true
        useGlobalConfig: true
        failOnStandardError: true
        retryCountOnTaskFailure: 3
        workingDirectory: '$(scriptDirectory)'

    - ${{ if parameters.deploy }}:
      - task: KubectlInstaller@0
        displayName: 'Install kubectl'
        condition: and(succeeded(), not(eq(coalesce(variables['terraformApply.aks_name'],'null'),'null')), eq(variables['terraformApply.peered_network'],'true'))
        inputs:
          kubectlVersion: 'latest'

    - ${{ if or(parameters.deploy, parameters.testReentrance) }}:
      - task: AzureCLI@2
        displayName: 'Wait for Application Gateway'
        condition: and(succeeded(), not(eq(coalesce(variables['terraformApply.application_gateway_id'],'null'),'null')))
        inputs:
          azureSubscription: '$(subscriptionConnection)'
          scriptType: pscore
          scriptLocation: scriptPath
          scriptPath: $(scriptDirectory)/wait_for_app_gw.ps1
          arguments:
            -AksName $(terraformApply.aks_name) `
            -ApplicationGatewayName "$(terraformApply.application_gateway_id)".Split("/")[-1] `
            -ResourceGroupName $(terraformApply.resource_group)
          useGlobalConfig: true
          failOnStandardError: true
          powerShellIgnoreLASTEXITCODE: false
          workingDirectory: '$(scriptDirectory)'

    - ${{ if parameters.deploy }}:
      - task: AzureCLI@2
        condition: and(succeeded(), not(eq(coalesce(variables['terraformApply.aks_name'],'null'),'null')), eq(variables['terraformApply.peered_network'],'true'))
        displayName: 'Deploy & Test applications'
        inputs:
          azureSubscription: '$(subscriptionConnection)'
          scriptType: pscore
          scriptLocation: inlineScript
          inlineScript: |
            # Diagnostics
            if ((${env:system.debug} -eq "true") -or ($env:system_debug -eq "true") -or ($env:SYSTEM_DEBUG -eq "true")) {
              $DebugPreference       = "Continue"
              $InformationPreference = "Continue"
              $VerbosePreference     = "Continue"
              Set-PSDebug -Trace 2
            }
            
            $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1
            ./kube_config.ps1
            ./deploy_app.ps1
          addSpnToEnvironment: true
          useGlobalConfig: true
          failOnStandardError: true
          powerShellIgnoreLASTEXITCODE: false
          workingDirectory: '$(scriptDirectory)'
        env:
          KUBECONFIG: variables['KUBE_CONFIG_PATH']

    - ${{ if parameters.testReentrance }}:
      - task: AzureCLI@2
        displayName: 'Terraform plan & apply (re-entrance test)'
        inputs:
          azureSubscription: '$(subscriptionConnection)'
          scriptType: pscore
          scriptLocation: inlineScript
          inlineScript: |
            $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1
            ./deploy.ps1 -apply -force
          addSpnToEnvironment: true
          useGlobalConfig: true
          failOnStandardError: true
          retryCountOnTaskFailure: 3
          workingDirectory: '$(scriptDirectory)'
          
    - ${{ if not(eq(parameters.destroy, 'Never')) }}:
      - pwsh: |
          Write-Host "Indicating success for job '$(Agent.JobName)'"
          Write-Host "##vso[task.setvariable variable=result;isOutput=true]success"
        name: provisioningResult
        displayName: 'Indicate provisioning success'
        condition: succeeded()

    - ${{ if not(eq(parameters.destroy, 'Never')) }}:
      - task: TerraformCLI@1
        displayName: 'Terraform destroy (${{ lower(parameters.destroy) }})'
        ${{ if eq(parameters.destroy, 'Always') }}:
          condition: succeededOrFailed()
        ${{ if eq(parameters.destroy, 'On failure') }}:
          condition: failed()
        ${{ if eq(parameters.destroy, 'On success') }}:
          condition: succeeded()
        continueOnError: true # Treat failure as warning during destroy, we will clean up anyway
        inputs:
          command: 'destroy'
          workingDirectory: '$(terraformDirectory)'
          environmentServiceName: '$(subscriptionConnection)'
          runAzLogin: true
          allowTelemetryCollection: true
          retryCountOnTaskFailure: 3
        env:
          KUBECONFIG: variables['KUBE_CONFIG_PATH']

    - ${{ if ne(parameters.destroy, 'Never') }}:
      - task: AzureCLI@2
        name: teardown
        displayName: 'Tear down remaining resources'
        ${{ if eq(parameters.destroy, 'Always') }}:
          condition: or(always(),canceled())
        ${{ if eq(parameters.destroy, 'On failure') }}:
          condition: not(eq(variables['provisioningResult.result'],'success'))
        ${{ if eq(parameters.destroy, 'On success') }}:
          condition: eq(variables['provisioningResult.result'],'success')
        inputs:
          azureSubscription: '$(subscriptionConnection)'
          scriptType: pscore
          scriptLocation: inlineScript
          inlineScript: |
            $ErrorActionPreference = "Continue" # Continue to remove resources if remove by resoyrce group fails
            # Build JMESPath expression
            $tagQuery = "[?tags.repository == '$(repository)' && tags.workspace == '${env:TF_WORKSPACE}' && tags.runid == '$(Build.BuildId)' && properties.provisioningState != 'Deleting'].id"
            Write-Host "Removing resources identified by `"$tagQuery`"..."

            # Remove resource groups
            $resourceGroupIDs = $(az group list --query "${tagQuery}" -o tsv)
            if ($resourceGroupIDs) {
              Write-Host "Removing resource group(s) `"${resourceGroupIDs}`"..."
              &{ # az writes information to stderr
                $ErrorActionPreference = 'SilentlyContinue'
                az resource delete --ids $resourceGroupIDs 2>&1
              }
            } else {
              Write-Host "No resource groups to remove"
            }

            # Remove (remaining) resources
            $resourceIDs = $(az resource list --query "${tagQuery}" -o tsv)
            if ($resourceIDs) {
              Write-Host "Removing resources `"${resourceIDs}`"..."
              &{ # az writes information to stderr
                $ErrorActionPreference = 'SilentlyContinue'
                az resource delete --ids $resourceIDs 2>&1
              }
            } else {
              Write-Host "No resources to remove"
            }

            # Remove VNet peerings
            ./cleanup_peerings.ps1
          addSpnToEnvironment: true
          useGlobalConfig: true
          failOnStandardError: true
          workingDirectory: '$(scriptDirectory)'

    - ${{ if ne(parameters.destroy, 'Never') }}:
      - task: AzureCLI@2
        name: cleanup
        displayName: 'Clean up Terraform state'
        ${{ if eq(parameters.destroy, 'Always') }}:
          condition: or(always(),canceled())
        ${{ if eq(parameters.destroy, 'On failure') }}:
          condition: not(eq(variables['provisioningResult.result'],'success'))
        ${{ if eq(parameters.destroy, 'On success') }}:
          condition: eq(variables['provisioningResult.result'],'success')
        inputs:
          azureSubscription: '$(subscriptionConnection)'
          scriptType: pscore
          scriptLocation: inlineScript
          inlineScript: |
            $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1

            $terraformState = (terraform state pull | ConvertFrom-Json)
            if ($terraformState.resources) {
              Write-Host "Clearing Terraform state in workspace ${env:TF_WORKSPACE}..."
              $terraformState.outputs = New-Object PSObject # Empty output
              $terraformState.resources = @() # No resources
              $terraformState.serial++
              $terraformState | ConvertTo-Json | terraform state push -
            } else {
              Write-Host "No resources in Terraform state in workspace ${env:TF_WORKSPACE}..."
            }
            terraform state pull 
          addSpnToEnvironment: true
          useGlobalConfig: true
          failOnStandardError: true
          workingDirectory: '$(terraformDirectory)'