enhancement: add monitoring test to check targets are up and running

Author: CFSNM

ods_ci/tests/Resources/Page/ODH/Prometheus/Prometheus.robot

@@ -244,6 +244,22 @@ Wait Until Alert Is Not Firing    # robocop: disable:too-many-arguments
     Wait Until Keyword Succeeds    ${timeout}    30s
     ...    Alert Should Not Be Firing    ${pm_url}    ${pm_token}    ${rule_group}    ${alert}    ${alert-duration}
 
+Get Target Pools
+    [Documentation]     Returns list of Targets scrape pools
+    [Arguments]        ${pm_url}    ${pm_token}    ${username}    ${password}
+    ${links}=    Run  curl --silent -X GET -H "Authorization:Bearer ${pm_token}" -u ${username}:${password} -k ${pm_url}/api/v1/targets | jq '.data.activeTargets[] | .scrapePool'       #robocop:disable
+    ${links}=    Replace String    ${links}    "    ${EMPTY}
+    @{links}=    Split String  ${links}  \n
+    RETURN    ${links}
+
+Get Target Pools Which Have State Up
+    [Documentation]     Returns list of Targets scrape pools which have state as "UP"
+    [Arguments]        ${pm_url}    ${pm_token}    ${username}    ${password}
+    ${links}=    Run  curl --silent -X GET -H "Authorization:Bearer ${pm_token}" -u ${username}:${password} -k ${pm_url}/api/v1/targets | jq '.data.activeTargets[] | select(.health == "up") | .scrapePool'       #robocop:disable
+    ${links}=    Replace String    ${links}    "    ${EMPTY}
+    @{links}=    Split String  ${links}  \n
+    RETURN    ${links}
+
 Get Target Endpoints
     [Documentation]     Returns list of Endpoint URLs
     [Arguments]         ${target_name}    ${pm_url}    ${pm_token}    ${username}    ${password}

ods_ci/tests/Tests/0100__platform/0102__monitor_and_manage/0101__metrics/0101__metrics.robot

@@ -94,6 +94,36 @@ Test Metric Existence For "Rhods_Aggregate_Availability" On ODS Prometheus
     Should Contain    ${list_values}    ${resp.json()["data"]["result"][0]["value"][-1]}
 
 
+Test Targets Are Available And Up In RHOAI Prometheus
+    [Documentation]   Verifies the expected targets in Prometheus are available and up running
+    [Tags]    Sanity
+    ...       Tier1
+    ...       ODS-179
+    ...       RHOAIENG-13066
+    ...       Monitoring
+    Skip If RHODS Is Self-Managed
+    @{targets} =    Prometheus.Get Target Pools Which Have State Up
+    ...    pm_url=${RHODS_PROMETHEUS_URL}
+    ...    pm_token=${RHODS_PROMETHEUS_TOKEN}
+    ...    username=${OCP_ADMIN_USER.USERNAME}
+    ...    password=${OCP_ADMIN_USER.PASSWORD}
+    List Should Contain Value    ${targets}    CodeFlare Operator
+    List Should Contain Value    ${targets}    Data Science Pipelines Operator
+    List Should Contain Value    ${targets}    Federate Prometheus
+    List Should Contain Value    ${targets}    Kserve Controller Manager
+    List Should Contain Value    ${targets}    KubeRay Operator
+    List Should Contain Value    ${targets}    Kubeflow Notebook Controller Service Metrics
+    List Should Contain Value    ${targets}    Kueue Operator
+    List Should Contain Value    ${targets}    Modelmesh Controller
+    List Should Contain Value    ${targets}    ODH Model Controller
+    List Should Contain Value    ${targets}    ODH Notebook Controller Service Metrics
+    List Should Contain Value    ${targets}    TrustyAI Controller Manager
+    List Should Contain Value    ${targets}    user_facing_endpoints_status_codeflare
+    List Should Contain Value    ${targets}    user_facing_endpoints_status_dsp
+    List Should Contain Value    ${targets}    user_facing_endpoints_status_rhods_dashboard
+    List Should Contain Value    ${targets}    user_facing_endpoints_status_workbenches
+
+
 *** Keywords ***
 Begin Metrics Web Test
     [Documentation]    Test Setup