Test Azure-Samples/cargotracker-liberty-aks periodically (#5)

* add scheduler and alert.

* fix "You have an error in your yaml syntax on line 43"

* fix yaml format

* fix resource group name

* set default value to inputs.

* remove testing parameter.

* fix job order

* fix yml format

* fix "You have an error in your yaml syntax on line 301"

* remove incorrect env variables.

* remove wls prefix

* set default value to `location`.

.github/workflows/setup.sh

@@ -127,7 +127,7 @@ msg "Subscription id is $SUBSCRIPTION_ID"
 ### AZ ACTION CREATE
 # --sdk-auth will be deprecated
 SP_SECRET=$(az ad sp create-for-rbac --display-name ${SERVICE_PRINCIPAL_NAME} --only-show-errors --query "password" --output tsv)
-SP_OBJECT_ID=$(az ad sp list --display-name ${SERVICE_PRINCIPAL_NAME} --query "[0].appId" --output tsv)
+SP_OBJECT_ID=$(az ad sp list --display-name ${SERVICE_PRINCIPAL_NAME} --query \[0\].appId --output tsv)
 TENANT_ID=$(az account show --query tenantId --output tsv --only-show-errors)
 az role assignment create --assignee ${SP_OBJECT_ID} --role "User Access Administrator" --scope "/subscriptions/${SUBSCRIPTION_ID}"
 az role assignment create --assignee ${SP_OBJECT_ID} --role "Contributor" --scope "/subscriptions/${SUBSCRIPTION_ID}"

.github/workflows/setupOpenLibertyAks.yml

@@ -10,31 +10,61 @@ on:
         description: "Azure region for resources"
         required: true
         default: eastus
+      deleteResource:
+        description: "True to delete all the Azure resources after the deployments. False to keep Azure resources."
+        default: "true"
+        required: true
   repository_dispatch:
+  schedule:
+    - cron: '0 0 27 * *' # run the workflow at the end of 27th monthly.
 
 env:
   refArmttk: c11a62d4ae011ee96fdecc76d76d811c5b5a99ce
   refJavaee: 931ce903b85a4bf1fc79c763a92ac3666f10a1e0
   appInsightsName: appinsights${{ github.run_id }}${{ github.run_number }}
   azCliVersion: 2.40.0
   azureCredentials: ${{ secrets.AZURE_CREDENTIALS }}
-  dbName: wlsdb${{ github.run_id }}${{ github.run_number }}
+  dbName: libertydb${{ github.run_id }}${{ github.run_number }}
   dbAdminUser: liberty
   dbPassword: ${{ secrets.DB_PASSWORD }}
-  location: ${{ github.event.inputs.region }}
   logAnalyticsName: loga${{ github.run_id }}${{ github.run_number }}
   namespace: default
   replicas: 3
-  resourceGroupForDB: wlsd-db-${{ github.run_id }}-${{ github.run_number }}
-  resourceGroupForOpenLibertyAks: ol-aks-${{ github.event.inputs.disambiguationSuffix }}-${{ github.run_number }}
+  resourceGroupForDB: liberty-db-${{ github.run_id }}-${{ github.run_number }}
   aksRepoUserName: WASdev
   aksRepoBranchName: 5c3f60fffdfd1219036bac2e50c51a53a97f21e3
 
 jobs:
   # Make it so the bicep file that causes Liberty on AKS to be deployed is available to this workflow.
   preflight:
     runs-on: ubuntu-20.04
-    steps:
+    outputs:
+        deleteResource: ${{steps.setup-env-variables.outputs.deleteResource}}
+        location: ${{ steps.setup-env-variables.outputs.location }}
+        resourceGroupForOpenLibertyAks: ol-aks-${{ steps.setup-env-variables.outputs.disambiguationSuffix }}-${{ github.run_number }}
+    steps:      
+      - name: Set up ENV variables
+        id: setup-env-variables
+        run: |
+          deleteResource=${{ github.event.inputs.deleteResource }}
+          if [ ${{ github.event.inputs.deleteResource }} == '' ]; then
+            deleteResource='true'
+          fi
+          echo "deleteResource: ${deleteResource}"
+
+          location=${{ github.event.inputs.region }}
+          if [[ -z "${location}" ]];then
+            location=eastus
+          fi
+
+          disambiguationSuffix=${{ github.event.inputs.disambiguationSuffix }}
+          if [[ -z "${disambiguationSuffix}" ]];then
+            disambiguationSuffix=$(date +%m%d)
+          fi
+
+          echo "deleteResource=${deleteResource}" >> $GITHUB_OUTPUT
+          echo "location=${location}" >> $GITHUB_OUTPUT
+          echo "disambiguationSuffix=${disambiguationSuffix}" >> $GITHUB_OUTPUT
       - name: Set up bicep
         run: |
           curl -Lo bicep https://github.com/Azure/bicep/releases/download/v0.11.1/bicep-linux-x64
@@ -92,6 +122,9 @@ jobs:
     needs: preflight
     runs-on: ubuntu-latest
     steps:
+      - name: Set variables
+        run: |
+          echo "location=${{ needs.preflight.outputs.location }}" >> $GITHUB_ENV
       - uses: azure/login@v1
         id: azure-login
         with:
@@ -102,7 +135,7 @@ jobs:
           azcliversion: ${{ env.azCliVersion }}
           inlineScript: |
             echo "create resource group" ${{ env.resourceGroupForDB }}
-            az group create --verbose --name ${{ env.resourceGroupForDB }} --location ${{ env.location }}
+            az group create --verbose --name ${{ env.resourceGroupForDB }} --location ${location}
       - name: Set Up Azure Postgresql to Test dbTemplate
         id: setup-postgresql
         uses: azure/CLI@v1
@@ -113,7 +146,7 @@ jobs:
             az postgres server create \
             --resource-group ${{ env.resourceGroupForDB }} \
             --name ${{ env.dbName }}  \
-            --location ${{ env.location }} \
+            --location ${location} \
             --admin-user ${{ env.dbAdminUser }} \
             --ssl-enforcement Disabled \
             --public-network-access Enabled \
@@ -131,6 +164,10 @@ jobs:
     needs: preflight
     runs-on: ubuntu-20.04
     steps:
+      - name: Set variables
+        run: |
+          echo "location=${{ needs.preflight.outputs.location }}" >> $GITHUB_ENV
+          echo "resourceGroupForOpenLibertyAks=${{ needs.preflight.outputs.resourceGroupForOpenLibertyAks }}" >> $GITHUB_ENV
       - name: Checkout ${{ env.aksRepoUserName }}/azure.liberty.aks
         uses: actions/checkout@v2
         with:
@@ -161,8 +198,8 @@ jobs:
         with:
           azcliversion: ${{ env.azCliVersion }}
           inlineScript: |
-            echo "create resource group" ${{ env.resourceGroupForOpenLibertyAks }}
-            az group create --verbose --name ${{ env.resourceGroupForOpenLibertyAks }} --location ${{ env.location }}
+            echo "create resource group" ${resourceGroupForOpenLibertyAks}
+            az group create --verbose --name ${resourceGroupForOpenLibertyAks} --location ${location}
       - name: Checkout cargotracker
         uses: actions/checkout@v2
         with:
@@ -188,7 +225,7 @@ jobs:
           inlineScript: |
             az deployment group validate \
             --debug \
-            --resource-group ${{ env.resourceGroupForOpenLibertyAks }} \
+            --resource-group ${resourceGroupForOpenLibertyAks} \
             --name openliberty-on-aks \
             --parameters @cargotracker/src/test/aks/parameters.json \
             --template-file ${artifactName}/mainTemplate.json
@@ -200,15 +237,19 @@ jobs:
           inlineScript: |
             az deployment group create \
             --verbose \
-            --resource-group ${{ env.resourceGroupForOpenLibertyAks }} \
+            --resource-group ${resourceGroupForOpenLibertyAks} \
             --name openliberty-on-aks \
             --parameters @cargotracker/src/test/aks/parameters.json \
             --template-file ${artifactName}/mainTemplate.json
   # Set up monitoring resources
   deploy-azure-monitor:
-    needs: [deploy-openliberty-on-aks]
+    needs: [preflight, deploy-openliberty-on-aks]
     runs-on: ubuntu-20.04
     steps:
+      - name: Set variables
+        run: |
+          echo "location=${{ needs.preflight.outputs.location }}" >> $GITHUB_ENV
+          echo "resourceGroupForOpenLibertyAks=${{ needs.preflight.outputs.resourceGroupForOpenLibertyAks }}" >> $GITHUB_ENV
       - uses: azure/login@v1
         id: azure-login
         with:
@@ -220,22 +261,22 @@ jobs:
           azcliversion: ${{ env.azCliVersion }}
           inlineScript: |
             az monitor log-analytics workspace create \
-              --resource-group ${{ env.resourceGroupForOpenLibertyAks }} \
+              --resource-group ${resourceGroupForOpenLibertyAks} \
               --workspace-name ${{ env.logAnalyticsName }} \
-              --location ${{ env.location }}
+              --location ${location}
       - name: Enable Container Insights
         id: enable-container-insights
         uses: azure/CLI@v1
         with:
           azcliversion: ${{ env.azCliVersion }}
           inlineScript: |
-            workspaceId=$(az monitor log-analytics workspace list -g ${{ env.resourceGroupForOpenLibertyAks }} --query '[0].id' -o tsv)
-            aksClusterName=$(az aks list  -g ${{env.resourceGroupForOpenLibertyAks}} --query [0].name -o tsv)
+            workspaceId=$(az monitor log-analytics workspace list -g ${resourceGroupForOpenLibertyAks} --query '[0].id' -o tsv)
+            aksClusterName=$(az aks list  -g ${resourceGroupForOpenLibertyAks} --query [0].name -o tsv)
 
             az aks enable-addons \
               --addons monitoring \
               --name ${aksClusterName} \
-              --resource-group ${{ env.resourceGroupForOpenLibertyAks }} \
+              --resource-group ${resourceGroupForOpenLibertyAks} \
               --workspace-resource-id ${workspaceId}
       - name: Provision Application Insights
         id: provision-app-insights
@@ -245,17 +286,21 @@ jobs:
           inlineScript: |
             az extension add --upgrade -n application-insights
 
-            workspaceId=$(az monitor log-analytics workspace list -g ${{ env.resourceGroupForOpenLibertyAks }} --query '[0].id' -o tsv)
+            workspaceId=$(az monitor log-analytics workspace list -g ${resourceGroupForOpenLibertyAks} --query '[0].id' -o tsv)
             az monitor app-insights component create \
-              --resource-group ${{ env.resourceGroupForOpenLibertyAks }} \
+              --resource-group ${resourceGroupForOpenLibertyAks} \
               --app ${{ env.appInsightsName }} \
-              --location ${{ env.location }} \
+              --location ${location} \
               --workspace ${workspaceId}
   # Build app, push to ACR and apply it to Open Liberty servers running on AKS.
   deploy-cargo-tracker:
-    needs: [deploy-db,deploy-azure-monitor]
+    needs: [preflight, deploy-db,deploy-azure-monitor]
     runs-on: ubuntu-20.04
     steps:
+      - name: Set variables
+        run: |
+          echo "location=${{ needs.preflight.outputs.location }}" >> $GITHUB_ENV
+          echo "resourceGroupForOpenLibertyAks=${{ needs.preflight.outputs.resourceGroupForOpenLibertyAks }}" >> $GITHUB_ENV
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
@@ -272,26 +317,26 @@ jobs:
         id: prepare_variables
         run: |
           echo "Query AKS cluster name"
-          aksClusterName=$(az aks list  -g ${{env.resourceGroupForOpenLibertyAks}} --query [0].name -o tsv)
+          aksClusterName=$(az aks list  -g ${resourceGroupForOpenLibertyAks} --query [0].name -o tsv)
           if [ -z "${aksClusterName}" ] ; then
             echo "Failed to obtain AKS. Exiting."
             exit 1
           fi
           echo "aksClusterName=${aksClusterName}" >> $GITHUB_ENV
 
-          acrName=$(az acr list  -g ${{env.resourceGroupForOpenLibertyAks}} --query [0].name -o tsv)
+          acrName=$(az acr list  -g ${resourceGroupForOpenLibertyAks} --query [0].name -o tsv)
           if [ -z "${acrName}" ] ; then
             echo "Falied to obtain ACR. Exiting."
             exit 1
           fi
 
-          azureACRServer=$(az acr show -n $acrName -g ${{env.resourceGroupForOpenLibertyAks}} --query 'loginServer' -o tsv)
-          azureACRUserName=$(az acr credential show -n $acrName -g ${{env.resourceGroupForOpenLibertyAks}} --query 'username' -o tsv)
-          azureACRPassword=$(az acr credential show -n $acrName -g ${{env.resourceGroupForOpenLibertyAks}} --query 'passwords[0].value' -o tsv)
+          azureACRServer=$(az acr show -n $acrName -g ${resourceGroupForOpenLibertyAks} --query 'loginServer' -o tsv)
+          azureACRUserName=$(az acr credential show -n $acrName -g ${resourceGroupForOpenLibertyAks} --query 'username' -o tsv)
+          azureACRPassword=$(az acr credential show -n $acrName -g ${resourceGroupForOpenLibertyAks} --query 'passwords[0].value' -o tsv)
           
           az extension add --upgrade -n application-insights
           appInsightsConnectionString=$(az monitor app-insights component show \
-            --resource-group ${{env.resourceGroupForOpenLibertyAks}} \
+            --resource-group ${resourceGroupForOpenLibertyAks} \
             --query '[0].connectionString' -o tsv)
 
           echo "azureACRServer=${azureACRServer}" >> $GITHUB_OUTPUT
@@ -341,7 +386,7 @@ jobs:
       - name: Connect to AKS cluster
         run: |
           echo "connect to cluster"
-          az aks get-credentials --resource-group ${{ env.resourceGroupForOpenLibertyAks }} --name ${aksClusterName}
+          az aks get-credentials --resource-group ${resourceGroupForOpenLibertyAks} --name ${aksClusterName}
       - name: Apply deployment files
         run: |
           echo "record time before restart"
@@ -387,7 +432,7 @@ jobs:
       - name: Query Application URL
         run: |
           gatewayPublicIPId=$(az network application-gateway list \
-            --resource-group ${{env.resourceGroupForOpenLibertyAks}} \
+            --resource-group ${resourceGroupForOpenLibertyAks} \
             --query '[0].frontendIPConfigurations[0].publicIPAddress.id' -o tsv)
 
           gatewayUrl=$(az network public-ip show --ids ${gatewayPublicIPId} --query 'dnsSettings.fqdn' -o tsv)
@@ -448,4 +493,45 @@ jobs:
       # Print app URL to the pipeline summary page.
       - name: Print app URL
         run: |
-          echo "${appURL}" >> $GITHUB_STEP_SUMMARY
+          echo "${appURL}" >> $GITHUB_STEP_SUMMARY
+ # Delete azure resources
+  cleanup:
+    if: always()
+    needs: [preflight, deploy-db, deploy-openliberty-on-aks, deploy-azure-monitor, deploy-cargo-tracker]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: azure/login@v1
+        id: azure-login
+        with:
+          creds: ${{ env.azureCredentials }}
+      - name: Delete Azure resources.
+        uses: azure/CLI@v1
+        if: ${{ needs.preflight.outputs.deleteResource == 'true' }}
+        with:
+          azcliversion: ${{ env.azCliVersion }}
+          inlineScript: |
+            echo "delete... " ${{ needs.preflight.outputs.resourceGroupForOpenLibertyAks }}
+            az group delete --yes --no-wait --verbose --name ${{ needs.preflight.outputs.resourceGroupForOpenLibertyAks }}
+
+            echo "delete... " ${{ env.resourceGroupForDB }}
+            az group delete --yes --no-wait --verbose --name ${{ env.resourceGroupForDB }}
+# File error message to specified Teams channel.
+  alert:
+    if: failure()
+    needs: [preflight, deploy-db, deploy-openliberty-on-aks, deploy-azure-monitor, deploy-cargo-tracker, cleanup]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send Teams message
+        if: ${{ github.repository_owner == 'azure-javaee' }}
+        run: |
+          echo "Job failed, send notification to Teams"
+          # https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook?tabs=dotnet
+          curl ${{ secrets.MSTEAMS_WEBHOOK }} \
+            -H 'Content-Type: application/json' \
+            --data-binary @- << EOF
+            {
+            "@context":"http://schema.org/extensions",
+            "@type":"MessageCard",
+            "text":"Workflow failed in cargotracker-liberty-aks repository, please take a look at: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${{ github.run_id }}"
+            }
+          EOF

.github/workflows/teardown.sh

@@ -47,7 +47,7 @@ SERVICE_PRINCIPAL_NAME=${DISAMBIG_PREFIX}sp
 # Execute commands
 msg "${GREEN}(1/3) Delete service principal ${SERVICE_PRINCIPAL_NAME}"
 SUBSCRIPTION_ID=$(az account show --query id --output tsv --only-show-errors)
-SP_OBJECT_ID_ARRAY=$(az ad sp list --display-name ${SERVICE_PRINCIPAL_NAME} --query "[].appId") || true
+SP_OBJECT_ID_ARRAY=$(az ad sp list --display-name ${SERVICE_PRINCIPAL_NAME} --query \[\].appId) || true
 # remove whitespace
 SP_OBJECT_ID_ARRAY=$(echo ${SP_OBJECT_ID_ARRAY} | xargs) || true
 SP_OBJECT_ID_ARRAY=${SP_OBJECT_ID_ARRAY//[/}