e2e-tests.yaml

name: E2E Tests

on: 
  pull_request: {}
  # workflow_run:
  #   workflows: ["BuildnPush"]
  #   types:
  #     - completed
concurrency: dev_environment

env:
  OCI_CLI_USER: ${{ secrets.OCI_CLI_USER }}
  OCI_CLI_TENANCY: ${{ secrets.OCI_CLI_TENANCY }}
  OCI_CLI_FINGERPRINT: ${{ secrets.OCI_CLI_FINGERPRINT }}
  OCI_CLI_KEY_CONTENT: ${{ secrets.OCI_CLI_KEY_CONTENT }}
  OCI_CLI_REGION: ${{ secrets.OCI_CLI_REGION }}
  
  # OCI_CLUSTER_ID: ${{ vars.CLUSTER_ID }}
  # OCI_VAULT_ID: ${{ vars.VAULT_ID }}
  OCI_VAULT_SECRET_VALUE: ${{ vars.VAULT_SECRET_VALUE }}
#   OCI_DEBUG: "--debug"

jobs:
  build:
    uses: ./.github/workflows/build-n-push.yaml
    with: 
      IMAGE_REGISTRY: ${{ vars.IMAGE_REGISTRY }}
    secrets: inherit
  setup-vault:
    runs-on: ubuntu-latest
    name: Setup Vault and secrets
    needs: [ build ]
    env:
      OCI_VAULT_ID: ${{ vars.VAULT_ID }}
      OCI_VAULT_KEY_ID: ${{ vars.VAULT_KEY_ID }}
    outputs:
      OCI_VAULT_ID: ${{ env.OCI_VAULT_ID }}
      OCI_VAULT_KEY_ID: ${{ env.OCI_VAULT_KEY_ID }}
      VAULT_SECRET_NAME: ${{ vars.SECRET_NAME_PREFIX }}-${{ env.OCI_RANDOM }}
      VAULT_SECRET_OCID: ${{ steps.extract-secret-ocid.outputs.VAULT_SECRET_OCID }}
    steps: 
      - name: create env with random
        id: gen-random
        run: echo "OCI_RANDOM=${RANDOM}" >> $GITHUB_ENV
        
      - name: Create vault if doesn't exist
        if: ${{ vars.USE_EXISTING_VAULT != 'true' }}
        uses: oracle-actions/run-oci-cli-command@v1.1
        id: create-vault
        with:
          silent: false
          command: "kms management vault create --compartment-id ${{ vars.COMPARTMENT_ID }} --display-name ${{ vars.VAULT_NAME_PREFIX }}-${{ env.OCI_RANDOM }} --vault-type default"
          query: "data.id"
      
      - name: get vault from previous output
        if: ${{ vars.USE_EXISTING_VAULT != 'true' }}
        run: echo "OCI_VAULT_ID=${{ steps.create-vault.outputs.raw_output }}" >> $GITHUB_ENV
      
      - name: create key if doesn't exist
        if: ${{ vars.USE_EXISTING_VAULT != 'true' }}
        uses: oracle-actions/run-oci-cli-command@v1.1
        id: create-vault-key
        with:
          silent: false
          command: "kms management key create --endpoint ${{ vars.VAULT_MGMT_ENDPOINT }} --compartment-id ${{ vars.COMPARTMENT_ID }}  --display-name key-${RANDOM} --key-shape '{ \"algorithm\" : \"AES\", \"length\" : 32 }'"
          query: "data.id"

      - name: create env for key id from create-vault-key output
        if: ${{ vars.USE_EXISTING_VAULT != 'true' }}
        run: echo "OCI_VAULT_KEY_ID=${{ steps.create-vault-key.outputs.raw_output }}" >> $GITHUB_ENV

      - name: create secret in vault
        uses: oracle-actions/run-oci-cli-command@v1.1
        id: create-secret
        with:
          silent: false
          command: vault secret create-base64 --compartment-id ${{ vars.COMPARTMENT_ID }} --vault-id ${{ env.OCI_VAULT_ID }} --key-id ${{ env.OCI_VAULT_KEY_ID }} --secret-name ${{ vars.SECRET_NAME_PREFIX }}-${{ env.OCI_RANDOM }} --secret-content-content ${{ env.OCI_VAULT_SECRET_VALUE }}"
#           query: "data.id"
      
      - name: extract secret id
        id: extract-secret-ocid
        run: echo VAULT_SECRET_OCID=`echo ${{ steps.create-secret.outputs.output }} | jq -r ".data.id"` >> $GITHUB_OUTPUT

  setup-cluster:
    runs-on: ubuntu-latest
    name: Setup Cluster
    needs: [ build ]
    env:
      OCI_CLUSTER_ID: ${{ vars.CLUSTER_ID }}
    outputs:
      OCI_CLUSTER_ID: ${{ steps.print-cluster-id.outputs.clusterId }}
    steps:
      # - name: create vcn if doesn't exist
      # - name: get vcn id from previous output or existing var
      # - name: create cluster
      #   if: ${{ vars.USE_EXISTING_CLUSTER != 'true' }} 
      #   uses: oracle-actions/run-oci-cli-command@v1.1
      #   id: create-cluster
      #   with:
      #     silent: false
      #     command: "ce cluster create --compartment-id ${{ vars.COMPARTMENT_ID }} 
      #     --vcn-id ${{ vars.VCN_ID }} --kubernetes-version ${{ vars.K8S_VERSION }} 
      #     --wait-for-state succeeded"
      #     query: "data.secret-name"

      # - name: create env for key id from create-vault-key output
      #   if: ${{ vars.USE_EXISTING_CLUSTER != 'true' }}
      #   run: echo "OCI_CLUSTER_ID=${{ steps.create-cluster.outputs.raw_output }}" >> $GITHUB_ENV

      # - name: create nodepool
      #   if: ${{ vars.USE_EXISTING_CLUSTER != 'true' }}

      # - name: get kubeconfig
      #   uses: oracle-actions/run-oci-cli-command@v1.1
      #   id: get-kube-config
      #   with:
      #     silent: false
      #     command: "ce cluster create-kubeconfig --cluster-id ${{ env.OCI_CLUSTER_ID }} --file $HOME/.kube/config --region ${{ env.OCI_CLI_REGION }} --token-version 2.0.0  --kube-endpoint PUBLIC_ENDPOINT"
      
      - name: print cluster id from vars
        id: print-cluster-id
        run: echo "clusterId=${{ env.OCI_CLUSTER_ID }}" >> $GITHUB_OUTPUT
        
  deploy-provider:
    runs-on: ubuntu-latest
    name: Deploy Provider and Run Tests
    needs: [ setup-vault , setup-cluster , build ]
    env:
      OCI_VAULT_ID: ${{ needs.setup-vault.outputs.OCI_VAULT_ID }}
      OCI_VAULT_SECRET_NAME: ${{ needs.setup-vault.outputs.VAULT_SECRET_NAME }}
      OCI_VAULT_SECRET_OCID: ${{ needs.setup-vault.outputs.VAULT_SECRET_OCID }}
      OCI_VAULT_SECRET_OCID_1: ${{ needs.setup-vault.outputs.VAULT_SECRET_OCID_1 }}
      OCI_CLUSTER_ID: ${{ needs.setup-cluster.outputs.OCI_CLUSTER_ID }}
      PROVIDER_NAMESPACE: ${{ vars.PROVIDER_NAMESPACE }}
      IMAGE_PATH : ${{ needs.build.outputs.IMAGE_PATH }}
    outputs:
      OCI_VAULT_SECRET_NAME: ${{ needs.setup-vault.outputs.VAULT_SECRET_NAME }}
      OCI_VAULT_SECRET_OCID: ${{ needs.setup-vault.outputs.VAULT_SECRET_OCID }}
      OCI_CLUSTER_ID:        ${{ needs.setup-cluster.outputs.OCI_CLUSTER_ID }}
    steps:
      - name: Configure Kubectl
        uses: oracle-actions/configure-kubectl-oke@v1.3.1
        id: test-configure-kubectl-oke-action
        with:
          cluster: ${{ env.OCI_CLUSTER_ID }}

      - name: test cluster access
        run: kubectl get nodes -A
        
      - name: create namespace in the cluster
        continue-on-error: true
        run: kubectl create namespace ${{ env.PROVIDER_NAMESPACE }}
      
      # - name: Install Helm
      #   uses: azure/setup-helm@v3

      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: split image path into repo and tag
        id: split-image-path
        run: |
          echo PROVIDER_IMAGE_REPO=`echo ${{ env.IMAGE_PATH }} | sed -e "s/:.*$//"` >> $GITHUB_OUTPUT
          echo PROVIDER_IMAGE_TAG=`echo ${{ env.IMAGE_PATH }} | sed -e "s/.*://"` >> $GITHUB_OUTPUT
      
      - name: print image values
        run: |
          echo ${{ steps.split-image-path.outputs.PROVIDER_IMAGE_REPO }}
          echo ${{ steps.split-image-path.outputs.PROVIDER_IMAGE_TAG }}    

      - name: Deploy Helm chart
        run: |
          helm upgrade --install oci-provider charts/oci-secrets-store-csi-driver-provider \
            --namespace ${{ env.PROVIDER_NAMESPACE }} \
            --set "provider.image.repository=${{ steps.split-image-path.outputs.PROVIDER_IMAGE_REPO }},provider.image.tag=${{ steps.split-image-path.outputs.PROVIDER_IMAGE_TAG }}"
      
      - name: list pods
        run: |
          kubectl get daemonset --namespace oci-provider \
            --selector='app.kubernetes.io/name in (oci-secrets-store-csi-driver-provider, secrets-store-csi-driver)'

      - name: user principal authentication - update auth file with correct values
        run: | 
          sed -e 's/region:.*/region: ${{ env.OCI_CLI_REGION }}/' \
          -e 's/tenancy:.*/tenancy: ${{ env.OCI_CLI_TENANCY }}/' \
          -e 's/user:.*/user: ${{ env.OCI_CLI_USER }}/' \
          -e 's/fingerprint:.*/fingerprint: ${{ env.OCI_CLI_FINGERPRINT }}/' e2e/example/user-principal/user-auth-config-example.yaml > e2e/example/user-principal/user-auth-config-example.yaml.tmp
      
      # - name: print updated yaml file
      #   run: cat e2e/example/user-auth-config-example.yaml.tmp

      - name: user principal authentication - create namespace
        # continue-on-error: true
        run: kubectl create namespace app-user
       
      - name: user principal authentication - delete secret if exists
        continue-on-error: true
        run: kubectl delete secret oci-config -n app-user

      - name: user principal authentication - create kubernetes secret for user auth config
        run: |
          kubectl create secret generic oci-config \
          --from-file=config=e2e/example/user-principal/user-auth-config-example.yaml.tmp \
          --from-literal=private-key="${{ env.OCI_CLI_KEY_CONTENT }}" -n app-user

      - name: user principal authentication - update spc file with correct values
        run: | 
          sed -e 's/vaultId:.*/vaultId: ${{ env.OCI_VAULT_ID }}/' \
          -e 's/authType:.*/authType: user/' \
          -e 's/- name:.*/- name: ${{ env.OCI_VAULT_SECRET_NAME }}/' e2e/example/user-principal/secret-provider-class.yaml > e2e/example/user-principal/secret-provider-class.yaml.tmp

      - name: user principal authentication - update deployment file with secret name
        run: | 
          sed -e 's/testingSecretName:.*/testingSecretName: ${{ env.OCI_VAULT_SECRET_NAME }}/' \
            e2e/example/user-principal/app.deployment.yaml > e2e/example/user-principal/app.deployment.yaml.tmp

      - name: user principal authentication - print updated yaml file
        run: cat e2e/example/user-principal/secret-provider-class.yaml.tmp

      - name: user principal authentication - deploy spc
        run: kubectl apply -f e2e/example/user-principal/secret-provider-class.yaml.tmp -n app-user

      - name: user principal authentication - deploy workload
        run: kubectl apply -f e2e/example/user-principal/app.deployment.yaml.tmp -n app-user

      - name: user principal authentication - Wait for pod to run
        id: wait-on-pod-user
        # run: kubectl wait --for=jsonpath='{.status.phase}'=Running pods/${{ env.POD_NAME }} --timeout=90s
        run: sleep 90

      - name: user principal authentication - Verify pods are running
        id: pod-names-user
        run: kubectl get pods -l testingSecretName=${{ env.OCI_VAULT_SECRET_NAME }} -o='custom-columns=PodName:.metadata.name' --no-headers -n app-user

      - name: user principal authentication - capture pod name into env
        run: echo "POD_NAME=`kubectl get pods -l testingSecretName=${{ env.OCI_VAULT_SECRET_NAME }} -o='custom-columns=PodName:.metadata.name' --no-headers -n app-user`" >> $GITHUB_ENV

      - name: user principal authentication - print secret value
        id: print-secret-content-user
        run: echo "SECRET_CONTENT=`kubectl exec -n app-user -it ${{ env.POD_NAME }} -- cat /mnt/secrets-store/${{ env.OCI_VAULT_SECRET_NAME }} 2> /dev/null | base64`" >> $GITHUB_ENV

      # - name: convert to base64
      #   id: convert-to-base64
      #   run: echo -n ${{ steps.print-secret-content.outputs.output }} | base64

      - name: user principal authentication - print values
        run: echo "${{ env.SECRET_CONTENT }} == ${{ env.OCI_VAULT_SECRET_VALUE}}"

      - name: user principal authentication - verify value
        run: if [ "${{ env.SECRET_CONTENT }}" == "${{ env.OCI_VAULT_SECRET_VALUE}}" ]; then exit 0; else exit 1; fi

      #
      # End of user principal
      #
      - name: workload identity principal authentication - update spc file with correct values
        run: | 
          sed -e 's/vaultId:.*/vaultId: ${{ env.OCI_VAULT_ID }}/' \
          -e 's/authType:.*/authType: workload/' \
          -e 's/- name:.*/- name: ${{ env.OCI_VAULT_SECRET_NAME }}/' e2e/example/workload-identity/secret-provider-class.yaml > e2e/example/workload-identity/secret-provider-class.yaml.tmp

      - name: workload identity principal authentication - update deployment file with secret name
        run: | 
          sed -e 's/testingSecretName:.*/testingSecretName: workload-${{ env.OCI_VAULT_SECRET_NAME }}/' \
            e2e/example/workload-identity/app.deployment.yaml > e2e/example/workload-identity/app.deployment.yaml.tmp

      - name: workload identity principal authentication - print updated yaml file
        run: cat e2e/example/workload-identity/secret-provider-class.yaml.tmp

      - name: workload identity principal authentication - create namespace
        # continue-on-error: true
        run: kubectl create namespace app-workload

      - name: workload identity principal authentication - deploy namespace and service account
        run: kubectl apply -f e2e/example/workload-identity/service-account.yaml -n app-workload

      - name: workload identity principal authentication - deploy spc
        run: kubectl apply -f e2e/example/workload-identity/secret-provider-class.yaml.tmp  -n app-workload

      - name: workload identity principal authentication - deploy workload
        run: kubectl apply -f e2e/example/workload-identity/app.deployment.yaml.tmp  -n app-workload

      - name: workload identity principal authentication - Wait for pod to run
        id: wait-on-pod-workload
        # run: kubectl wait --for=jsonpath='{.status.phase}'=Running pods/${{ env.POD_NAME }} --timeout=90s
        run: sleep 90

      - name: workload identity principal authentication - Verify pods are running
        id: pod-names-workload
        run: kubectl get pods -l testingSecretName=workload-${{ env.OCI_VAULT_SECRET_NAME }} -o='custom-columns=PodName:.metadata.name' --no-headers  -n app-workload

      - name: workload identity principal authentication - capture pod name into env
        run: echo "POD_NAME=`kubectl get pods -l testingSecretName=workload-${{ env.OCI_VAULT_SECRET_NAME }} -o='custom-columns=PodName:.metadata.name' --no-headers  -n app-workload`" >> $GITHUB_ENV

      - name: workload identity principal authentication - print secret value
        id: print-secret-content-workload
        run: echo "SECRET_CONTENT=`kubectl exec -n app-workload -it ${{ env.POD_NAME }} -- cat /mnt/secrets-store/${{ env.OCI_VAULT_SECRET_NAME }} 2> /dev/null | base64`" >> $GITHUB_ENV

      - name: workload identity principal authentication - print values
        run: echo "${{ env.SECRET_CONTENT }} == ${{ env.OCI_VAULT_SECRET_VALUE}}"

      - name: workload identity principal authentication - verify value
        run: if [ "${{ env.SECRET_CONTENT }}" == "${{ env.OCI_VAULT_SECRET_VALUE}}" ]; then exit 0; else exit 1; fi  

      #
      # End of workload identity principal
      #
      - name: instance principal authentication - update spc file with correct values
        run: | 
          sed -e 's/vaultId:.*/vaultId: ${{ env.OCI_VAULT_ID }}/' \
          -e 's/authType:.*/authType: instance/' \
          -e 's/- name:.*/- name: ${{ env.OCI_VAULT_SECRET_NAME }}/' e2e/example/instance-principal/secret-provider-class.yaml > e2e/example/instance-principal/secret-provider-class.yaml.tmp

      - name: instance principal authentication - update deployment file with secret name
        run: | 
          sed -e 's/testingSecretName:.*/testingSecretName: instance-${{ env.OCI_VAULT_SECRET_NAME }}/' \
            e2e/example/instance-principal/app.deployment.yaml > e2e/example/instance-principal/app.deployment.yaml.tmp

      - name: instance principal authentication - print updated yaml file
        run: cat e2e/example/instance-principal/secret-provider-class.yaml.tmp

      - name: instance principal authentication - create namespace
        # continue-on-error: true
        run: kubectl create namespace app-instance

      - name: instance principal authentication - deploy spc
        run: kubectl apply -f e2e/example/instance-principal/secret-provider-class.yaml.tmp -n app-instance

      - name: instance principal authentication - deploy workload
        run: kubectl apply -f e2e/example/instance-principal/app.deployment.yaml.tmp -n app-instance

      - name: instance principal authentication - Wait for pod to run
        id: wait-on-pod-instance
        # run: kubectl wait --for=jsonpath='{.status.phase}'=Running pods/${{ env.POD_NAME }} --timeout=90s
        run: sleep 90

      - name: instance principal authentication - Verify pods are running
        id: pod-names-instance
        run: kubectl get pods -l testingSecretName=instance-${{ env.OCI_VAULT_SECRET_NAME }} -o='custom-columns=PodName:.metadata.name' --no-headers  -n app-instance

      - name: instance principal authentication - capture pod name into env
        run: echo "POD_NAME=`kubectl get pods -l testingSecretName=instance-${{ env.OCI_VAULT_SECRET_NAME }} -o='custom-columns=PodName:.metadata.name' --no-headers  -n app-instance`" >> $GITHUB_ENV

      - name: instance principal authentication - print secret value
        id: print-secret-content-instance
        run: echo "SECRET_CONTENT=`kubectl exec  -n app-instance -it ${{ env.POD_NAME }} -- cat /mnt/secrets-store/${{ env.OCI_VAULT_SECRET_NAME }} 2> /dev/null | base64`" >> $GITHUB_ENV

      - name: instance principal authentication - print values
        run: echo "${{ env.SECRET_CONTENT }} == ${{ env.OCI_VAULT_SECRET_VALUE}}"

      - name: instance principal authentication - verify value
        run: if [ "${{ env.SECRET_CONTENT }}" == "${{ env.OCI_VAULT_SECRET_VALUE}}" ]; then exit 0; else exit 1; fi    

      # cleanup
      - name: remove deployment - user principal authentication
        if: ${{ always() }} 
        run: |
          kubectl delete -f e2e/example/user-principal/app.deployment.yaml.tmp \
            -f e2e/example/user-principal/secret-provider-class.yaml.tmp -n app-user

      - name: remove deployment - workload identity authentication
        if: ${{ always() }} 
        run: |
          kubectl delete -f e2e/example/workload-identity/secret-provider-class.yaml.tmp \
            -f e2e/example/workload-identity/app.deployment.yaml.tmp \
            -f e2e/example/workload-identity/service-account.yaml -n app-workload

      - name: remove deployment - instance principal authentication
        if: ${{ always() }} 
        run: |
          kubectl delete -f e2e/example/instance-principal/secret-provider-class.yaml.tmp \
            -f e2e/example/instance-principal/app.deployment.yaml.tmp -n app-instance         

      - name: delete secret
        if: ${{ always() }} 
        run: kubectl delete secret oci-config -n app-user
      
      - name: uninstall provider
        if: ${{ always() }}
        run: helm uninstall oci-provider -n ${{ env.PROVIDER_NAMESPACE }}

      - name: delete namespaces 
        if: ${{ always() }}
        run: kubectl delete namespace app-user app-workload app-instance 

  cleanup:
    runs-on: ubuntu-latest
    needs: [deploy-provider]
    name: Cleanup resources
    env:
      OCI_VAULT_SECRET_NAME: ${{ needs.deploy-provider.outputs.OCI_VAULT_SECRET_NAME }}
      OCI_VAULT_SECRET_OCID: ${{ needs.deploy-provider.outputs.OCI_VAULT_SECRET_OCID }}
      OCI_CLUSTER_ID: ${{ needs.deploy-provider.outputs.OCI_CLUSTER_ID }}
    steps:
      - name: delete cluster
        if: ${{ vars.USE_EXISTING_CLUSTER != 'true' }}
        uses: oracle-actions/run-oci-cli-command@v1.1
        with:
          command: "ce cluster delete --cluster-id ${{ env.OCI_CLUSTER_ID }} --wait-for-state  SUCCEEDED"

      # - name: get secret id
      #   id: get-secret-ocid
      #   uses: oracle-actions/run-oci-cli-command@v1.1
      #   with:
      #     command: "vault secret list --name ${{ env.OCI_VAULT_SECRET_NAME }} --compartment-id ${{ vars.COMPARTMENT_ID }}"
      #     query: data[0].id

      - name: delete secrets
        uses: oracle-actions/run-oci-cli-command@v1.1
        with:
          command: "vault secret schedule-secret-deletion --secret-id ${{ env.OCI_VAULT_SECRET_OCID }}"
  #     - name: delete vcn if created
  #     - name: delete vault if created