diff --git a/ods_ci/tests/Resources/CLI/TrustyAI/trustyai_service.resource b/ods_ci/tests/Resources/CLI/TrustyAI/trustyai_service.resource index e3591f7ef..311168fa0 100644 --- a/ods_ci/tests/Resources/CLI/TrustyAI/trustyai_service.resource +++ b/ods_ci/tests/Resources/CLI/TrustyAI/trustyai_service.resource @@ -25,9 +25,9 @@ Verify Model Is Registered With TrustyAI Service ... value is the TrustyAI service route [Arguments] ${namespace}=${PRJ_TITLE} IF "${PRODUCT}" == "ODH" - ${MM_PAYLOAD_PROCESSORS_Expected}= Set Variable http://trustyai-service.${namespace}.svc.cluster.local/consumer/kserve/v2 + ${MM_PAYLOAD_PROCESSORS_Expected}= Set Variable https://trustyai-service.${namespace}.svc.cluster.local/consumer/kserve/v2 ELSE - ${MM_PAYLOAD_PROCESSORS_Expected}= Set Variable http://trustyai-service.${namespace}.svc/consumer/kserve/v2 + ${MM_PAYLOAD_PROCESSORS_Expected}= Set Variable https://trustyai-service.${namespace}.svc/consumer/kserve/v2 END Wait Until Keyword Succeeds 3 min 20 sec Verify Model Serving Pod Exists ${namespace} ... label_selector=modelmesh-service=modelmesh-serving diff --git a/ods_ci/tests/Resources/Common.robot b/ods_ci/tests/Resources/Common.robot index b625a9694..fe83f35a3 100644 --- a/ods_ci/tests/Resources/Common.robot +++ b/ods_ci/tests/Resources/Common.robot @@ -523,3 +523,13 @@ Clear Element And Input Text Clear Element Text ${element_xpath} Sleep 0.5s Input Text ${element_xpath} ${new_text} + +Clone Git Repository + [Documentation] Clone Git repository in local framework + [Arguments] ${REPO_URL} ${REPO_BRANCH} ${DIR} + ${result} = Run Process git clone -b ${REPO_BRANCH} ${REPO_URL} ${DIR} + ... shell=true stderr=STDOUT + Log To Console ${result.stdout} + IF ${result.rc} != 0 + FAIL Unable to clone DW repo ${REPO_URL}:${REPO_BRANCH}:${DIR} + END diff --git a/ods_ci/tests/Tests/1200__ai_explainability/1201__model_serving_bias_metrics.robot b/ods_ci/tests/Tests/1200__ai_explainability/1201__model_serving_bias_metrics.robot index 4d3ff49b9..111fa6b7b 100644 --- a/ods_ci/tests/Tests/1200__ai_explainability/1201__model_serving_bias_metrics.robot +++ b/ods_ci/tests/Tests/1200__ai_explainability/1201__model_serving_bias_metrics.robot @@ -62,7 +62,7 @@ Verify DIR Bias Metrics Available In CLI For Models Deployed Prior To Enabling T Verify SPD Metrics Available In CLI For Models Deployed After Enabling Trusty Service For Basic User [Documentation] Verifies that the Bias metrics are available in Metrics Console for a model ... deployed after enabling the TrustyAI service - [Tags] ODS-2482 ODS-2476 + [Tags] ODS-2482 ODS-2476 OpenDataHub ExcludeOnRHOAI Launch Data Science Project Main Page Create Data Science Project title=${PRJ_TITLE1} description=${PRJ_DESCRIPTION1} Append To List ${PROJECTS_TO_DELETE} ${PRJ_TITLE1} diff --git a/ods_ci/tests/Tests/1200__ai_explainability/1203__trustyai_drift_metrics.robot b/ods_ci/tests/Tests/1200__ai_explainability/1203__trustyai_drift_metrics.robot new file mode 100644 index 000000000..9afb4e52b --- /dev/null +++ b/ods_ci/tests/Tests/1200__ai_explainability/1203__trustyai_drift_metrics.robot @@ -0,0 +1,68 @@ +*** Settings *** +Documentation This is a resource file for running Drift Metrics. +Suite Setup Prepare TrustyAi-tests Test Suite +Suite Teardown Teardown TrustyAi-tests Test Suite +Library OperatingSystem +Library Process +Resource ../../Resources/Common.robot +Resource ../../Resources/RHOSi.resource + + +*** Variables *** +${TRUSTYAI-TESTS_URL} %{TRUSTYAI-TESTS-URL=https://github.com/trustyai-explainability/trustyai-tests} +${TRUSTYAI-TESTS_DIR} trustyai-tests +${GIT_BRANCH} main +${VIRTUAL_ENV_NAME} drift_venv +${TESTS_FOLDER} drift +${LOG_OUTPUT_FILE} drift_test.txt + + +*** Test Cases *** +Run Drift Metrics Tests + [Documentation] Verifies that the Drift metrics are available for a deployed model + [Tags] RHOAIENG-8163 Smoke + Run Drift Pytest Framework + + +*** Keywords *** +Prepare TrustyAi-tests Test Suite + [Documentation] Prepare trustyai-tests E2E Test Suite + Log To Console "Prepare Test Suite" + Drift Setup + RHOSi Setup + +Teardown TrustyAi-tests Test Suite + [Documentation] TrustyAi-tests Test Suite + Cleanup TrustyAI-Tests Setup + RHOSi Teardown + +Drift Setup + [Documentation] Setup for Drift tests + Log To Console "Cloning ${TRUSTYAI-TESTS_DIR} repo" + Common.Clone Git Repository ${TRUSTYAI-TESTS_URL} ${GIT_BRANCH} ${TRUSTYAI-TESTS_DIR} + ${return_code} ${output} Run And Return Rc And Output cd ${TRUSTYAI-TESTS_DIR} + Should Be Equal As Integers ${return_code} 0 msg=Error detected while cloning git repo + ${result} = Run Process virtualenv -p python3 ${VIRTUAL_ENV_NAME} + ... shell=true stderr=STDOUT + Log To Console ${result.stdout} + IF ${result.rc} != 0 + FAIL Unable to setup Python virtual environment + END + +Cleanup TrustyAI-Tests Setup + [Documentation] cleanup trustyai-tests repository cloned + Log To Console "Removing Python virtual environment ${VIRTUAL_ENV_NAME}" + ${result} = Run Process rm -rf ${VIRTUAL_ENV_NAME} + ... shell=true stderr=STDOUT + Log To Console ${result.stdout} + IF ${result.rc} != 0 FAIL Unable to cleanup Python virtual environment + Log To Console "Removing directory ${TRUSTYAI-TESTS_DIR}" + Remove Directory ${TRUSTYAI-TESTS_DIR} recursive=True + +Run Drift Pytest Framework + [Documentation] Run Drift tests using poetry + Log To Console Running Drift tests using pytest + ${result} = Run Process source ${VIRTUAL_ENV_NAME}/bin/activate && cd ${TRUSTYAI-TESTS_DIR} && poetry install && poetry run pytest -v -s ./trustyai_tests/tests/${TESTS_FOLDER}/ && deactivate # robocop: disable:line-too-long + ... shell=true stderr=STDOUT stdout=${LOG_OUTPUT_FILE} + Log To Console ${result.stdout} + IF ${result.rc} != 0 FAIL Tests failed due to ${result.stdout} failed