diff --git a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot index ce4c34bf1..4157669a9 100644 --- a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot +++ b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot @@ -167,10 +167,9 @@ Clean Up Server Should Match "${ls_server}" "${EMPTY}" Get User Notebook Pod Name - [Documentation] Returns notebook pod name for given username (e.g. for user ldap-admin1 it will be jupyterhub-nb-ldap-2dadmin1) + [Documentation] Returns notebook pod name for given username (e.g. for user ldap-admin1 it will be jupyter-nb-ldap-2dadmin1-0) [Arguments] ${username} ${safe_username}= Get Safe Username ${username} - #${notebook_pod_name}= Set Variable jupyterhub-nb-${safe_username} ${notebook_pod_name}= Set Variable jupyter-nb-${safe_username}-0 RETURN ${notebook_pod_name} diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot index 37c454b72..501977e84 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot @@ -1,34 +1,40 @@ *** Settings *** -Documentation Test Suite for Upgrade testing, to be run before the upgrade -Library OpenShiftLibrary -Resource ../../Resources/RHOSi.resource -Resource ../../Resources/ODS.robot -Resource ../../Resources/Page/ODH/ODHDashboard/ODHDashboard.resource -Resource ../../Resources/Page/ODH/ODHDashboard/ODHDashboardResources.resource -Resource ../../Resources/Page/ODH/ODHDashboard/ODHModelServing.resource -Resource ../../Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/DataConnections.resource -Resource ../../Resources/Page/ODH/JupyterHub/HighAvailability.robot -Resource ../../Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource -Resource ../../Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource -Resource ../../Resources/Page/ODH/AiApps/Anaconda.resource -Resource ../../Resources/Page/LoginPage.robot -Resource ../../Resources/Page/OCPLogin/OCPLogin.robot -Resource ../../Resources/Common.robot -Resource ../../Resources/Page/OCPDashboard/Pods/Pods.robot -Resource ../../Resources/Page/OCPDashboard/Builds/Builds.robot -Resource ../../Resources/Page/HybridCloudConsole/OCM.robot -Resource ../../Resources/CLI/ModelServing/modelmesh.resource -Resource ../../Resources/Page/DistributedWorkloads/DistributedWorkloads.resource -Resource ../../Resources/Page/DistributedWorkloads/WorkloadMetricsUI.resource -Suite Setup Dashboard Suite Setup -Suite Teardown RHOSi Teardown -Test Tags PreUpgrade +Documentation Test Suite for Upgrade testing, to be run before the upgrade + +Library OpenShiftLibrary +Resource ../../Resources/RHOSi.resource +Resource ../../Resources/ODS.robot +Resource ../../Resources/Page/ODH/ODHDashboard/ODHDashboard.resource +Resource ../../Resources/Page/ODH/ODHDashboard/ODHDashboardResources.resource +Resource ../../Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +Resource ../../Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/DataConnections.resource +Resource ../../Resources/Page/ODH/JupyterHub/HighAvailability.robot +Resource ../../Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource +Resource ../../Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +Resource ../../Resources/Page/ODH/AiApps/Anaconda.resource +Resource ../../Resources/Page/LoginPage.robot +Resource ../../Resources/Page/OCPLogin/OCPLogin.robot +Resource ../../Resources/Common.robot +Resource ../../Resources/Page/OCPDashboard/Pods/Pods.robot +Resource ../../Resources/Page/OCPDashboard/Builds/Builds.robot +Resource ../../Resources/Page/HybridCloudConsole/OCM.robot +Resource ../../Resources/CLI/ModelServing/modelmesh.resource +Resource ../../Resources/Page/DistributedWorkloads/DistributedWorkloads.resource +Resource ../../Resources/Page/DistributedWorkloads/WorkloadMetricsUI.resource + +Suite Setup Upgrade Suite Setup +Suite Teardown RHOSi Teardown + +Test Tags PreUpgrade *** Variables *** ${CUSTOM_CULLER_TIMEOUT} 60000 ${S_SIZE} 25 ${DW_PROJECT_CREATED} False +${CODE} while True: import time ; time.sleep(10); print ("Hello") +${UPGRADE_NS} upgrade +${UPGRADE_CONFIG_MAP} upgrade-config-map *** Test Cases *** @@ -204,12 +210,54 @@ Run Training Operator ODH Setup Sleep PyTorchJob Test Use Case Run Training Operator ODH Upgrade Test TestSetupSleepPytorchjob [Teardown] Teardown Training Operator E2E Upgrade Test Suite +Long Running Jupyter Notebook + [Documentation] Launch a long running notebook before the upgrade + [Tags] Upgrade + Launch Notebook + Add And Run JupyterLab Code Cell In Active Notebook ${CODE} + + # Get the notebook pod creation timestamp + ${notebook_pod_name}= Get User Notebook Pod Name ${TEST_USER2.USERNAME} + ${return_code} ${ntb_creation_timestamp} = Run And Return Rc And Output + ... oc get pod -n ${NOTEBOOKS_NAMESPACE} ${notebook_pod_name} --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${ntb_creation_timestamp} + + # Save the timestamp to the OpenShift ConfigMap so it can be used in test in the next phase + ${return_code} ${cmd_output} = Run And Return Rc And Output + ... oc create configmap ${UPGRADE_CONFIG_MAP} -n ${UPGRADE_NS} --from-literal=ntb_creation_timestamp=${ntb_creation_timestamp} # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + + Close Browser + *** Keywords *** -Dashboard Suite Setup - [Documentation] Basic suite setup +Launch Notebook + [Documentation] Launch notebook for the suite + [Arguments] ${notebook_image}=minimal-notebook + ... ${username}=${TEST_USER2.USERNAME} + ... ${password}=${TEST_USER2.PASSWORD} + ... ${auth_type}=${TEST_USER2.AUTH_TYPE} + Begin Web Test username=${username} password=${password} auth_type=${auth_type} + Launch Jupyter From RHODS Dashboard Link + Spawn Notebook With Arguments + ... image=${notebook_image} + ... username=${username} + ... password=${password} + ... auth_type=${auth_type} + +Upgrade Suite Setup + [Documentation] Basic suite setup Set Library Search Order SeleniumLibrary RHOSi Setup + # Prepare a namespace for storing values that should be shared between different upgrade test phases + # 1. if the namespace exists already, let's remove it + ${return_code} ${cmd_output} = Run And Return Rc And Output + ... oc delete namespace --wait --ignore-not-found ${UPGRADE_NS} + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + # 2. create the namespace now + ${return_code} ${cmd_output} = Run And Return Rc And Output + ... oc create namespace ${UPGRADE_NS} + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} Dashboard Test Teardown [Documentation] Basic suite teardown diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot index b34d1e897..9d1eb015c 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot @@ -12,21 +12,7 @@ Library JupyterLibrary Test Tags DuringUpgrade -*** Variables *** -${CODE} while True: import time ; time.sleep(10); print ("Hello") - - *** Test Cases *** -Long Running Jupyter Notebook - [Documentation] Launch a long running notebook before the upgrade - [Tags] Upgrade - Launch Notebook - Add And Run JupyterLab Code Cell In Active Notebook ${CODE} - ${return_code} ${timestamp} Run And Return Rc And Output oc get pod -n ${NOTEBOOKS_NAMESPACE} jupyter-nb-ldap-2dadmin2-0 --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' #robocop:disable - Should Be Equal As Integers ${return_code} 0 - Set Global Variable ${timestamp} #robocop: disable - Close Browser - Upgrade RHODS [Documentation] Approve the install plan for the upgrade and make sure that upgrade has completed [Tags] ODS-1766 @@ -58,17 +44,18 @@ PyTorch Image Workload Test *** Keywords *** Launch Notebook - [Documentation] Launch notebook for the suite - [Arguments] ${notbook_image}=minimal-notebook ${username}=${TEST_USER2.USERNAME} ${password}=${TEST_USER2.PASSWORD} ${auth_type}=${TEST_USER2.AUTH_TYPE} #robocop: disable - Begin Web Test username=${username} password=${password} auth_type=${auth_type} - Login To RHODS Dashboard ${username} ${password} ${auth_type} - Wait For RHODS Dashboard To Load + [Documentation] Launch notebook for the suite + [Arguments] ${notebook_image}=minimal-notebook + ... ${username}=${TEST_USER2.USERNAME} + ... ${password}=${TEST_USER2.PASSWORD} + ... ${auth_type}=${TEST_USER2.AUTH_TYPE} + Begin Web Test username=${username} password=${password} auth_type=${auth_type} Launch Jupyter From RHODS Dashboard Link - Login To Jupyterhub ${username} ${password} ${auth_type} - ${authorization_required} Is Service Account Authorization Required - IF ${authorization_required} Authorize Jupyterhub Service Account - Fix Spawner Status - Spawn Notebook With Arguments image=${notbook_image} username=${username} password=${password} auth_type=${auth_type} #robocop: disable + Spawn Notebook With Arguments + ... image=${notebook_image} + ... username=${username} + ... password=${password} + ... auth_type=${auth_type} Upgrade Test Teardown End Web Test diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot index 2f3bdd004..d06653a56 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot @@ -29,6 +29,8 @@ Test Tags PostUpgrade *** Variables *** ${S_SIZE} 25 ${DW_PROJECT_CREATED} False +${UPGRADE_NS} upgrade +${UPGRADE_CONFIG_MAP} upgrade-config-map *** Test Cases *** @@ -63,11 +65,26 @@ Verify Culler is Enabled END Verify Notebook Has Not Restarted - [Documentation] Verify Notbook pod has not restarted after the upgrade - [Tags] Upgrade - ${return_code} ${new_timestamp} Run And Return Rc And Output oc get pod -n ${NOTEBOOKS_NAMESPACE} jupyter-nb-ldap-2dadmin2-0 --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' #robocop:disable - Should Be Equal As Integers ${return_code} 0 - Should Be Equal ${timestamp} ${new_timestamp} msg=Running notebook pod has restarted + [Documentation] Verify Notebook pod has not restarted after the upgrade + [Tags] Upgrade + ${notebook_name}= Get User CR Notebook Name ${TEST_USER2.USERNAME} + ${notebook_pod_name}= Get User Notebook Pod Name ${TEST_USER2.USERNAME} + + # Get the running notebook creation timestamp + ${return_code} ${new_timestamp} Run And Return Rc And Output + ... oc get pod -n ${NOTEBOOKS_NAMESPACE} ${notebook_pod_name} --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${new_timestamp} + + # Get the running notebook creation timestamp from the upgrade ConfigMap safed in the previous + # phase (before the actual RHOAI upgrade) + ${return_code} ${ntb_creation_timestamp} Run And Return Rc And Output + ... oc get configmap ${UPGRADE_CONFIG_MAP} -n ${UPGRADE_NS} -o jsonpath='{.data.ntb_creation_timestamp}' + Should Be Equal As Integers ${return_code} 0 msg=${ntb_creation_timestamp} + + # The timestamps should be equal + Should Be Equal ${ntb_creation_timestamp} ${new_timestamp} msg=Running notebook pod has restarted + + [Teardown] Terminate Running Notebook ${notebook_name} Verify Custom Image Is Present [Tags] Upgrade @@ -276,6 +293,13 @@ Delete OOTB Image IF not ${status} Fail Notebook image is deleted after the upgrade IF not ${IS_SELF_MANAGED} Managed RHOAI Upgrade Test Teardown +Terminate Running Notebook + [Documentation] Terminates the running notebook instance + [Arguments] ${notebook_name} + ${return_code} ${cmd_output} Run And Return Rc And Output + ... oc delete Notebook.kubeflow.org -n ${NOTEBOOKS_NAMESPACE} ${notebook_name} + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + Managed RHOAI Upgrade Test Teardown [Documentation] Check rhods_aggregate_availability metric when RHOAI is installed as managed ${expression} = Set Variable rhods_aggregate_availability&step=1