[temp][test_sensubility] Add more detail to the healthcheck

Author: elfiesmelfie

roles/test_sensubility/tasks/test_health_status.yml

@@ -32,6 +32,8 @@
   changed_when: false
   failed_when: container_nodes.stdout_lines|length != 0
 
+
+  # issue might be that the value is aggregated over 10m, and we only check for 2
 - name: RHELOSP-176036 Check that health status of container changed to 0
   ansible.builtin.shell:
     cmd: /usr/bin/curl -k {{ prom_auth_string }} -g https://{{ prom_url }}/api/v1/query? --data-urlencode 'query=last_over_time(sensubility_container_health_status{process="logrotate_crond",host="ceph-0.redhat.local"}[10m])' | grep -oP '(?<="value":).*' | awk -F, '{ print $2 }' | grep -o '[0-9]\+' | grep 0
@@ -41,6 +43,52 @@
   delay: 10
   until: "output.stdout_lines | length == 1"
 
+- name: NEW RHELOSP-176036 but check over a smaller time
+  ansible.builtin.shell:
+    cmd: /usr/bin/curl -k {{ prom_auth_string }} -g https://{{ prom_url }}/api/v1/query? --data-urlencode 'query=last_over_time(sensubility_container_health_status{process="logrotate_crond",host="ceph-0.redhat.local"}[2m])' | grep -oP '(?<="value":).*' | awk -F, '{ print $2 }' | grep -o '[0-9]\+' | grep 0
+  register: output
+  changed_when: false
+  retries: 12
+  delay: 10
+  until: "output.stdout_lines | length == 1"
+
+- name: NEW RHELOSP-176036 but check over a smaller time
+  ansible.builtin.shell:
+    cmd: /usr/bin/curl -k {{ prom_auth_string }} -g https://{{ prom_url }}/api/v1/query? --data-urlencode 'query=last_over_time(sensubility_container_health_status{process="logrotate_crond",host="ceph-0.redhat.local"}[2m])' | grep -oP '(?<="value":).*' | awk -F, '{ print $2 }' | grep -o '[0-9]\+' | grep 0
+  register: output
+  changed_when: false
+  retries: 12
+  delay: 10
+  until: "output.stdout_lines | length == 1"
+
+- name: NEW RHELOSP-176036 but wait for up to 10 minutes
+  ansible.builtin.shell:
+    cmd: /usr/bin/curl -k {{ prom_auth_string }} -g https://{{ prom_url }}/api/v1/query? --data-urlencode 'query=last_over_time(sensubility_container_health_status{process="logrotate_crond",host="ceph-0.redhat.local"}[10m])' | grep -oP '(?<="value":).*' | awk -F, '{ print $2 }' | grep -o '[0-9]\+' | grep 0
+  register: output
+  changed_when: false
+  retries: 60
+  delay: 10
+  until: "output.stdout_lines | length == 1"
+
+- name: NEW RHELOSP-176036 but don't "grep 0"
+  ansible.builtin.shell:
+    cmd: /usr/bin/curl -k {{ prom_auth_string }} -g https://{{ prom_url }}/api/v1/query? --data-urlencode 'query=last_over_time(sensubility_container_health_status{process="logrotate_crond",host="ceph-0.redhat.local"}[10m])' | grep -oP '(?<="value":).*' | awk -F, '{ print $2 }' | grep -o '[0-9]\+'
+  register: output
+  changed_when: false
+
+- name: NEW RHELOSP-176036 but skip the awk
+  ansible.builtin.shell:
+    cmd: /usr/bin/curl -k {{ prom_auth_string }} -g https://{{ prom_url }}/api/v1/query? --data-urlencode 'query=last_over_time(sensubility_container_health_status{process="logrotate_crond",host="ceph-0.redhat.local"}[10m])' | grep -oP '(?<="value":).*'
+  register: output
+  changed_when: false
+
+- name: NEW RHELOSP-176036 but check the whole output
+  ansible.builtin.shell:
+    cmd: /usr/bin/curl -k {{ prom_auth_string }} -g https://{{ prom_url }}/api/v1/query? --data-urlencode 'query=last_over_time(sensubility_container_health_status{process="logrotate_crond",host="ceph-0.redhat.local"}[10m])'
+  register: output
+  changed_when: false
+
+
 - name: RHELOSP-176035 Start logrotate_crond container
   ansible.builtin.shell:
     cmd: |