testcharts.yaml

name: Check changes to charts do not break user experience

# Only runs when charts have changed

# Test changes to charts against released images
# TODO: test if changes to kustomize have been replicated for charts
# NOTE: charts/controller is being tested in all of the http and grpc tests

on:
  pull_request:
    branches:
      - master
    paths:
      - charts/**

# Kind versions used to test Iter8 on different versions of Kubernetes
# From: https://github.com/kubernetes-sigs/kind/releases
env:
  versions: |
    kindest/node:v1.28.0@sha256:b7a4cad12c197af3ba43202d3efe03246b3f0793f162afb40a33c923952d5b31
    kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
    kindest/node:v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb
    kindest/node:v1.25.11@sha256:227fa11ce74ea76a0474eeefb84cb75d8dad1b08638371ecf0e86259b35be0c8
    kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab

jobs:
  # Get the different Kind versions
  get_versions:
    runs-on: ubuntu-latest

    steps:
    - name: Get the different Kind versions
      id: set-matrix
      run: |
        # Serialize versions into JSON array
        jsonVersions=$(jq -ncR '[inputs]' <<< "$versions")
        echo $jsonVersions

        # Output serialized jsonVersions
        echo "matrix=$jsonVersions" | sed -e "s/,\"\"//" >> $GITHUB_OUTPUT

    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}

  # TODO: add check to verify when a change is made to kustomize, a similar change is made to charts

  http:
    name: HTTP performance test
    needs: get_versions
    runs-on: ubuntu-latest
    strategy:
      matrix:
        version: ${{ fromJson(needs.get_versions.outputs.matrix) }}

    steps:    
    - name: Check out code
      uses: actions/checkout@v3

    - name: Get modified files in the charts/iter8 folder
      id: modified-files
      uses: tj-actions/changed-files@v35
      with:
        files: charts/iter8

    - name: Start kind cluster ${{ matrix.version }}
      uses: helm/kind-action@v1.5.0
      if: steps.modified-files.outputs.any_modified == 'true'
      with:
        wait: 300s
        node_image: ${{ matrix.version }}

    - name: Create httpbin application
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl create deployment httpbin --image=kennethreitz/httpbin
        kubectl expose deployment httpbin --type=ClusterIP --port=80
        kubectl wait --for=condition=available --timeout=60s deploy/httpbin

    - name: Install controller
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm install iter8 charts/controller --set logLevel=trace
        kubectl rollout status --watch --timeout=60s statefulset/iter8

    - name: Start performance test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm upgrade --install httpbin-test charts/iter8 \
        --set "tasks={http}" \
        --set http.url="http://httpbin.default/get" \
        --set logLevel=trace
        kubectl wait --for=condition=complete --timeout=180s job/httpbin-test-1-job

    - name: Get Kubernetes status
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl get all

    - name: View test logs and delete test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl logs -l iter8.tools/test=httpbin-test
        helm delete httpbin-test

    - name: Expose metrics service
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl port-forward service/iter8 8080:8080 &

    - name: Check GET /httpDashboard
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        curl "http://localhost:8080/httpDashboard?namespace=default&test=httpbin-test" -f

  http-payload:
    name: HTTP performance test with payload
    needs: get_versions
    runs-on: ubuntu-latest
    strategy:
      matrix:
        version: ${{ fromJson(needs.get_versions.outputs.matrix) }}

    steps:    
    - name: Check out code
      uses: actions/checkout@v3

    - name: Get modified files in the charts/iter8 folder
      id: modified-files
      uses: tj-actions/changed-files@v35
      with:
        files: charts/iter8

    - name: Start kind cluster ${{ matrix.version }}
      uses: helm/kind-action@v1.5.0
      if: steps.modified-files.outputs.any_modified == 'true'
      with:
        wait: 300s
        node_image: ${{ matrix.version }}

    - name: Create httpbin application
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl create deployment httpbin --image=kennethreitz/httpbin
        kubectl expose deployment httpbin --type=ClusterIP --port=80
        kubectl wait --for=condition=available --timeout=60s deploy/httpbin

    - name: Install controller
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm install iter8 charts/controller --set logLevel=trace
        kubectl rollout status --watch --timeout=60s statefulset/iter8

    - name: Start performance test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm upgrade --install httpbin-test charts/iter8 \
        --set "tasks={http}" \
        --set http.url="http://httpbin.default/post" \
        --set http.payloadStr=hello \
        --set logLevel=trace
        kubectl wait --for=condition=complete --timeout=180s job/httpbin-test-1-job

    - name: Get Kubernetes status
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl get all

    - name: View test logs and delete test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl logs -l iter8.tools/test=httpbin-test
        helm delete httpbin-test

    - name: Expose metrics service
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl port-forward service/iter8 8080:8080 &

    - name: Check GET /httpDashboard
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        curl "http://localhost:8080/httpDashboard?namespace=default&test=httpbin-test" -f

  http-multiple:
    name: HTTP performance test with multiple endpoints
    needs: get_versions
    runs-on: ubuntu-latest
    strategy:
      matrix:
        version: ${{ fromJson(needs.get_versions.outputs.matrix) }}

    steps:    
    - name: Check out code
      uses: actions/checkout@v3

    - name: Get modified files in the charts/iter8 folder
      id: modified-files
      uses: tj-actions/changed-files@v35
      with:
        files: charts/iter8

    - name: Start kind cluster ${{ matrix.version }}
      uses: helm/kind-action@v1.5.0
      if: steps.modified-files.outputs.any_modified == 'true'
      with:
        wait: 300s
        node_image: ${{ matrix.version }}

    - name: Create httpbin application
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl create deployment httpbin --image=kennethreitz/httpbin
        kubectl expose deployment httpbin --type=ClusterIP --port=80
        kubectl wait --for=condition=available --timeout=60s deploy/httpbin

    - name: Install controller
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm install iter8 charts/controller --set logLevel=trace
        kubectl rollout status --watch --timeout=60s statefulset/iter8

    - name: Start performance test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm upgrade --install httpbin-test charts/iter8 \
        --set "tasks={http}" \
        --set http.endpoints.get.url=http://httpbin.default/get \
        --set http.endpoints.getAnything.url=http://httpbin.default/anything \
        --set http.endpoints.post.url=http://httpbin.default/post \
        --set http.endpoints.post.payloadStr=hello \
        --set logLevel=trace
        kubectl wait --for=condition=complete --timeout=180s job/httpbin-test-1-job

    - name: Get Kubernetes status
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl get all

    - name: View test logs and delete test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl logs -l iter8.tools/test=httpbin-test
        helm delete httpbin-test

    - name: Expose metrics service
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl port-forward service/iter8 8080:8080 &

    - name: Check GET /httpDashboard
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        curl "http://localhost:8080/httpDashboard?namespace=default&test=httpbin-test" -f

  grpc:
    name: gRPC performance test
    needs: get_versions
    runs-on: ubuntu-latest
    strategy:
      matrix:
        version: ${{ fromJson(needs.get_versions.outputs.matrix) }}

    steps:
    - name: Check out code
      uses: actions/checkout@v3

    - name: Get modified files in the charts/iter8 folder
      id: modified-files
      uses: tj-actions/changed-files@v35
      with:
        files: charts/iter8
    
    - name: Start kind cluster ${{ matrix.version }}
      uses: helm/kind-action@v1.5.0
      if: steps.modified-files.outputs.any_modified == 'true'
      with:
        wait: 300s
        node_image: ${{ matrix.version }}

    - name: Create routeguide application
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl create deployment routeguide --image=golang --port=50051 \
        -- bash -c "git clone -b v1.52.0 --depth 1 https://github.com/grpc/grpc-go; cd grpc-go/examples/route_guide; sed -i "''" "'"s/localhost//"'" server/server.go; go run server/server.go"
        kubectl expose deployment routeguide --port=50051
        kubectl wait --for=condition=available --timeout=60s deployment/routeguide

    - name: Test gRPC service with grpcurl
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        curl -sO https://gist.githubusercontent.com/kalantar/510737f0fd58c0432a08e5b6e45ec97f/raw/524d6660284bf653ce0f29f3a25ed0e913c3df80/grpcurl-routeguide.yaml
        kubectl apply -f grpcurl-routeguide.yaml
        sleep 180
        kubectl logs deploy/sleep

    - name: Install controller
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm install iter8 charts/controller --set logLevel=trace
        kubectl rollout status --watch --timeout=60s statefulset/iter8

    - name: Start performance test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm upgrade --install routeguide-test charts/iter8 \
        --set "tasks={ready,grpc}" \
        --set ready.deploy=routeguide \
        --set ready.service=routeguide \
        --set ready.timeout=60s \
        --set grpc.host=routeguide.default:50051 \
        --set grpc.protoURL=https://raw.githubusercontent.com/grpc/grpc-go/v1.52.0/examples/route_guide/routeguide/route_guide.proto \
        --set grpc.call=routeguide.RouteGuide.GetFeature \
        --set grpc.dataURL=https://raw.githubusercontent.com/iter8-tools/docs/v0.13.13/samples/grpc-payload/unary.json \
        --set logLevel=trace
        kubectl wait --for=condition=complete --timeout=180s job/routeguide-test-1-job

    - name: Get Kubernetes status
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl get all

    - name: View test logs and delete test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl logs -l iter8.tools/test=routeguide-test
        helm delete routeguide-test
  
    - name: Expose metrics service
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl port-forward service/iter8 8080:8080 &

    - name: Check GET /grpcDashboard
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        curl "http://localhost:8080/grpcDashboard?namespace=default&test=routeguide-test" -f

  grpc-multiple:
    name: gRPC performance test with multiple endpoints
    needs: get_versions
    runs-on: ubuntu-latest
    strategy:
      matrix:
        version: ${{ fromJson(needs.get_versions.outputs.matrix) }}

    steps:
    - name: Check out code
      uses: actions/checkout@v3

    - name: Get modified files in the charts/iter8 folder
      id: modified-files
      uses: tj-actions/changed-files@v35
      with:
        files: charts/iter8
    
    - name: Start kind cluster ${{ matrix.version }}
      uses: helm/kind-action@v1.5.0
      if: steps.modified-files.outputs.any_modified == 'true'
      with:
        wait: 300s
        node_image: ${{ matrix.version }}

    - name: Create routeguide application
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl create deployment routeguide --image=golang --port=50051 \
        -- bash -c "git clone -b v1.52.0 --depth 1 https://github.com/grpc/grpc-go; cd grpc-go/examples/route_guide; sed -i "''" "'"s/localhost//"'" server/server.go; go run server/server.go"
        kubectl expose deployment routeguide --port=50051
        kubectl wait --for=condition=available --timeout=60s deployment/routeguide

    - name: Test gRPC service with grpcurl
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        curl -sO https://gist.githubusercontent.com/kalantar/510737f0fd58c0432a08e5b6e45ec97f/raw/524d6660284bf653ce0f29f3a25ed0e913c3df80/grpcurl-routeguide.yaml
        kubectl apply -f grpcurl-routeguide.yaml
        sleep 180
        kubectl logs deploy/sleep

    - name: Install controller
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm install iter8 charts/controller --set logLevel=trace
        kubectl rollout status --watch --timeout=60s statefulset/iter8

    - name: Start performance test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm upgrade --install routeguide-test charts/iter8 \
        --set "tasks={ready,grpc}" \
        --set ready.deploy=routeguide \
        --set ready.service=routeguide \
        --set ready.timeout=60s \
        --set grpc.host=routeguide.default:50051 \
        --set grpc.protoURL=https://raw.githubusercontent.com/grpc/grpc-go/v1.52.0/examples/route_guide/routeguide/route_guide.proto \
        --set grpc.endpoints.getFeature.call=routeguide.RouteGuide.GetFeature \
        --set grpc.endpoints.getFeature.dataURL=https://raw.githubusercontent.com/iter8-tools/docs/v0.13.13/samples/grpc-payload/unary.json \
        --set grpc.endpoints.listFeatures.call=routeguide.RouteGuide.ListFeatures \
        --set grpc.endpoints.listFeatures.dataURL=https://raw.githubusercontent.com/iter8-tools/docs/v0.13.13/samples/grpc-payload/server.json \
        --set logLevel=trace
        kubectl wait --for=condition=complete --timeout=180s job/routeguide-test-1-job

    - name: Get Kubernetes status
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl get all

    - name: View test logs and delete test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl logs -l iter8.tools/test=routeguide-test
        helm delete routeguide-test
    
    - name: Expose metrics service
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl port-forward service/iter8 8080:8080 &

    - name: Check GET /grpcDashboard
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        curl "http://localhost:8080/grpcDashboard?namespace=default&test=routeguide-test" -f

  grpc2:
    name: gRPC performance test 2
    needs: get_versions
    runs-on: ubuntu-latest
    strategy:
      matrix:
        version: ${{ fromJson(needs.get_versions.outputs.matrix) }}

    steps:
    - name: Check out code
      uses: actions/checkout@v3

    - name: Get modified files in the charts/iter8 folder
      id: modified-files
      uses: tj-actions/changed-files@v35
      with:
        files: charts/iter8
    
    - name: Start kind cluster ${{ matrix.version }}
      uses: helm/kind-action@v1.5.0
      if: steps.modified-files.outputs.any_modified == 'true'
      with:
        wait: 300s
        node_image: ${{ matrix.version }}

    - name: Create hello application
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl create deploy hello --image=docker.io/grpc/java-example-hostname:latest --port=50051
        kubectl expose deploy hello --port=50051
        kubectl wait --for=condition=available --timeout=60s deploy/hello

    - name: Install controller
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm install iter8 charts/controller --set logLevel=trace
        kubectl rollout status --watch --timeout=60s statefulset/iter8

    - name: Start performance test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        helm upgrade --install hello-test charts/iter8 \
        --set "tasks={grpc}" \
        --set grpc.host="hello.default:50051" \
        --set grpc.call="helloworld.Greeter.SayHello" \
        --set grpc.protoURL="https://raw.githubusercontent.com/grpc/grpc-go/master/examples/helloworld/helloworld/helloworld.proto" \
        --set logLevel=trace
        kubectl wait --for=condition=complete --timeout=180s job/hello-test-1-job

    - name: Get Kubernetes status
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl get all

    - name: View test logs and delete test
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl logs -l iter8.tools/test=hello-test
        helm delete hello-test
      
    - name: Expose metrics service
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        kubectl port-forward service/iter8 8080:8080 &

    - name: Check GET /grpcDashboard
      if: steps.modified-files.outputs.any_modified == 'true'
      run: |
        curl "http://localhost:8080/grpcDashboard?namespace=default&test=hello-test" -f