Fix relative paths under ods_ci (red-hat-data-services#1516)

* Fix relative paths under ods_ci

* Updated README to run tests inside ods_ci folder

.github/workflows/dry_run.yml

@@ -21,8 +21,9 @@ jobs:
 
       - name: Execute ods-ci dryrun
         run: |
-          mv ods_ci/test-variables.yml.example ods_ci/test-variables.yml
-          ./ods_ci/run_robot_test.sh --extra-robot-args '--dryrun' --skip-oclogin true --no-output-subfolder true
+          cd ods_ci
+          mv test-variables.yml.example test-variables.yml
+          ./run_robot_test.sh --extra-robot-args '--dryrun' --skip-oclogin true --no-output-subfolder true
 
       - name: Save PR number and artifacts
         run: |

README.md

@@ -10,11 +10,16 @@ and its upstream project, [Open Data Hub](https://opendatahub.io/).
 
   2. [Poetry](https://python-poetry.org/docs/#installation) tool installed and added to your `$PATH`.
 
+
 # Quick Start
+  1. Move to the ods_ci folder inside ods-ci repo
+     ```bash
+     cd ods_ci
+     ```
   1. Create a variables file for all of the global test values
      ```bash
      # Create the initial test variables from the example template variables file
-     cp ods_ci/test-variables.yml.example ods_ci/test-variables.yml
+     cp test-variables.yml.example test-variables.yml
      ```
   2. Edit the test variables file to include information required for this test run.
      You will need to add info required for test execution:
@@ -28,26 +33,26 @@ and its upstream project, [Open Data Hub](https://opendatahub.io/).
   3. Run this script that will create the virtual environment, install the required packages and kickoff the Robot test suite.
   ```bash
      # Running all the tests
-     sh ods_ci/run_robot_test.sh
+     sh run_robot_test.sh
 
      # Running Smoke test suite via tag
-     sh ods_ci/run_robot_test.sh --include Smoke
+     sh run_robot_test.sh --include Smoke
 
      # Running a specific test via tag
-     sh ods_ci/run_robot_test.sh --include ODS-XYZ
+     sh run_robot_test.sh --include ODS-XYZ
 
      # Running tests in Open Data Hub:
      # You need to set accordingly the PRODUCT, APPLICATIONS_NAMESPACE, MONITORING_NAMESPACE,
      # OPERATOR_NAMESPACE and NOTEBOOKS_NAMESPACE in test-variables.yaml (or pass them as parameters
      # when launching the tests) and overwrite some local variables used in the test suites
      # adding --variablefile ./ods_ci/test-variables-odh-overwrite.yml
-     sh ods_ci/run_robot_test.sh \
+     sh run_robot_test.sh \
       --test-variable PRODUCT:ODH \
       --test-variable APPLICATIONS_NAMESPACE:opendatahub \
       --test-variable MONITORING_NAMESPACE:opendatahub \
       --test-variable OPERATOR_NAMESPACE:openshift-operators \
       --test-variable NOTEBOOKS_NAMESPACE:opendatahub \
-      --extra-robot-args '--variablefile ./ods_ci/test-variables-odh-overwrite.yml' \
+      --extra-robot-args '--variablefile test-variables-odh-overwrite.yml' \
       --include OpenDataHub
    ```
 

ods_ci/build/clean_idp.sh

@@ -74,7 +74,7 @@ uninstall_identity_provider(){
   # delete htpasswd and LDAp idp
   # delete ldap deployment and idp
   # oc wait --for=delete $(oc get namespace openldap)
-  oc delete -f ods_ci/configs/templates/ldap/ldap.yaml
+  oc delete -f configs/templates/ldap/ldap.yaml
   if [ "${USE_OCM_IDP}" -eq 1 ]
     then
       ocm delete idp -c "${CLUSTER_NAME}" ldap-provider-qe
@@ -105,14 +105,14 @@ function check_installation(){
             echo $ocm_clusterid
             while read -r line; do
               if [[ $line == *"ldap"* ]] || [[ $line == *"htpasswd"* ]] ; then
-                  found=1                 
+                  found=1
               fi
             done < <(ocm get /api/clusters_mgmt/v1/clusters/$ocm_clusterid/identity_providers)
             if  [ -z $found ]
               then
                 echo -e "\033[0;33m LDAP and/or htpasswd Identity not found. Skipping removal of IDPs \033[0m"
                 exit 0
-              else 
+              else
                 echo -e "\033[0;33m LDAP and/or htpasswd Identity providers found. Going to remove IDPs \033[0m"
             fi
       else
@@ -138,14 +138,14 @@ function check_uninstallation(){
             while read -r line; do
               if [[ $line == *"ldap"* ]] || [[ $line == *"htpasswd"* ]] ; then
                   echo -e "\033[0;33m LDAP and/or htpasswd Identity providers are still installed. Please check the cluster \033[0m"
-                  found=1 
+                  found=1
                   exit 0
               fi
             done < <(ocm get /api/clusters_mgmt/v1/clusters/$ocm_clusterid/identity_providers)
             if  [ -z $found ]
               then
                 echo -e "\033[0;33m LDAP and/or htpasswd Identity providers have been deleted from the cluster \033[0m"
-              else 
+              else
                 echo -e "\033[0;33m LDAP and/or htpasswd Identity providers are still installed. Please check the cluster \033[0m"
             fi
       else
@@ -167,7 +167,7 @@ while [ "$#" -gt 0 ]; do
       OC_HOST=$1
       shift
       ;;
-    
+
     --use-ocm)
       shift
       USE_OCM_IDP=$1
@@ -214,4 +214,4 @@ if [ "${USE_OCM_IDP}" -eq 1 ]
 fi
 check_installation
 uninstall_identity_provider
-check_uninstallation
+check_uninstallation

ods_ci/build/install_idp.sh

@@ -30,7 +30,7 @@ function generate_rand_string(){
 
 function generate_rand_user_base_suffix(){
   array=()
-  for i in {a..z} {A..Z}; 
+  for i in {a..z} {A..Z};
     do
     array[$RANDOM]=$i
   done
@@ -63,14 +63,14 @@ function generate_custom_suffixes(){
         quotes_flag=$(echo $suffix | egrep -o '".+"')
         if [[ -n $quotes_flag ]]; then
           complete_name=$(echo $complete_name | tr -d '"')
-        fi        
+        fi
         USERS_ARR+=($complete_name)
         PWS_ARR+=($pw)
     done
 }
 
 function extract_testvariables_users_mapping(){
-  test_user_mapping=$(jq -r --arg idpname $1 --arg test_user $2 '.[][$idpname][$test_user]' ods_ci/configs/templates/user_config.json)
+  test_user_mapping=$(jq -r --arg idpname $1 --arg test_user $2 '.[][$idpname][$test_user]' configs/templates/user_config.json)
   users_string=$3
   if [[ "$test_user_mapping" = *"<RAND_BASE>"* ]]; then
       test_user_regex=$(echo "${test_user_mapping/<RAND_BASE>/"[a-zA-Z]+"}")
@@ -82,7 +82,7 @@ function extract_testvariables_users_mapping(){
 }
 function generate_users_creds(){
   echo "--> Generating users based on requested configuration"
-  idp=$(jq --arg idpname $1 '.[][$idpname]' ods_ci/configs/templates/user_config.json)
+  idp=$(jq --arg idpname $1 '.[][$idpname]' configs/templates/user_config.json)
   USERS_ARR=()
   PWS_ARR=()
   pw=$(echo $idp | jq -r '.pw')
@@ -144,12 +144,12 @@ function set_htpasswd_users_and_login(){
     else
         htp_string=$(htpasswd -b -B -n $cluster_adm_user $htp_pw)
         oc create secret generic htpasswd-secret --from-literal=htpasswd="$htp_string" -n openshift-config
-        OAUTH_HTPASSWD_JSON="$(cat ods_ci/configs/resources/oauth_htp_idp.json)"
+        OAUTH_HTPASSWD_JSON="$(cat configs/resources/oauth_htp_idp.json)"
         oc patch oauth cluster --type json -p '[{"op": "add", "path": "/spec/identityProviders/-", "value": '"$OAUTH_HTPASSWD_JSON"'}]'
-        cp ods_ci/configs/templates/ca-rolebinding.yaml ods_ci/configs/ca-rolebinding.yaml
-        sed -i "s/<rolebinding_name>/ods-ci-htp-admin/g" ods_ci/configs/ca-rolebinding.yaml
-        sed -i "s/<username>/$cluster_adm_user/g" ods_ci/configs/ca-rolebinding.yaml
-        oc apply -f ods_ci/configs/ca-rolebinding.yaml
+        cp configs/templates/ca-rolebinding.yaml configs/ca-rolebinding.yaml
+        sed -i'' -e "s/<rolebinding_name>/ods-ci-htp-admin/g" configs/ca-rolebinding.yaml
+        sed -i'' -e "s/<username>/$cluster_adm_user/g" configs/ca-rolebinding.yaml
+        oc apply -f configs/ca-rolebinding.yaml
   fi
   # login using htpasswd
   echo "----> Performing log in with newly created HTP user"
@@ -158,34 +158,34 @@ function set_htpasswd_users_and_login(){
   # add more htpasswd users, if present
   echo "---> Adding additional HTP users, if needed per requested configuration"
   secret_name=$(oc get oauth cluster -o json | jq -r '.spec.identityProviders[] | select(.htpasswd!=null) | .htpasswd.fileData.name')
-  oc get secret $secret_name -ojsonpath={.data.htpasswd} -n openshift-config | base64 --decode > ods_ci/configs/users.htpasswd
+  oc get secret $secret_name -ojsonpath={.data.htpasswd} -n openshift-config | base64 --decode > configs/users.htpasswd
   update_secret=0
   for htp_user in "${HTP_USERS[@]}"; do
     if [[ ! "$htp_user" =  "$cluster_adm_user" ]]
       then
-        htpasswd -bB ods_ci/configs/users.htpasswd $htp_user $htp_pw
+        htpasswd -bB configs/users.htpasswd $htp_user $htp_pw
         update_secret=1
     fi
   done
   if [[ $update_secret -eq 1 ]]; then
-    oc create secret generic $secret_name --from-file=htpasswd=ods_ci/configs/users.htpasswd --dry-run=client -o yaml -n openshift-config | oc replace -f -
+    oc create secret generic $secret_name --from-file=htpasswd=configs/users.htpasswd --dry-run=client -o yaml -n openshift-config | oc replace -f -
   else
     echo "----> SKIP"
   fi
 
   # update test-variables.yml with admin creds
   export adm_user=$cluster_adm_user
   export adm_p=$htp_pw
-  yq --inplace '.OCP_ADMIN_USER.AUTH_TYPE="htpasswd"' ods_ci/test-variables.yml
-  yq --inplace '.OCP_ADMIN_USER.USERNAME=env(adm_user)' ods_ci/test-variables.yml
-  yq --inplace '.OCP_ADMIN_USER.PASSWORD=env(adm_p)' ods_ci/test-variables.yml
+  yq --inplace '.OCP_ADMIN_USER.AUTH_TYPE="htpasswd"' test-variables.yml
+  yq --inplace '.OCP_ADMIN_USER.USERNAME=env(adm_user)' test-variables.yml
+  yq --inplace '.OCP_ADMIN_USER.PASSWORD=env(adm_p)' test-variables.yml
 
 }
 
 function set_ldap_users(){
   generate_users_creds  ldap
   LDAP_USERS=("${USERS_ARR[@]}")
-  LDAP_PWS=("${PWS_ARR[@]}")  
+  LDAP_PWS=("${PWS_ARR[@]}")
   ldap_users_str=$(printf ,%s ${LDAP_USERS[@]})
   ldap_pws_str=$(printf ,%s ${LDAP_PWS[@]})
   ldap_users_str=${ldap_users_str:1}
@@ -195,16 +195,16 @@ function set_ldap_users(){
   rand_base64=$(echo -n $ldap_pws_str | base64 -w 0)
   # update ldap.yaml with creds
   echo "--> configuring LDAP server and users"
-  cp ods_ci/configs/templates/ldap/ldap.yaml  ods_ci/configs/ldap.yaml
-  sed -i "s/<users_string>/$users_base64/g" ods_ci/configs/ldap.yaml
-  sed -i "s/<passwords_string>/$rand_base64/g" ods_ci/configs/ldap.yaml
+  cp configs/templates/ldap/ldap.yaml  configs/ldap.yaml
+  sed -i'' -e "s/<users_string>/$users_base64/g" configs/ldap.yaml
+  sed -i'' -e "s/<passwords_string>/$rand_base64/g" configs/ldap.yaml
   rand=$(generate_rand_string)
   export RAND_ADMIN=$rand
   rand_base64=$(echo -n $rand | base64 -w 0)
-  sed -i "s/<adminpassword>/$rand_base64/g" ods_ci/configs/ldap.yaml
+  sed -i'' -e "s/<adminpassword>/$rand_base64/g" configs/ldap.yaml
 
   # create ldap deployment
-  oc apply -f ods_ci/configs/ldap.yaml
+  oc apply -f configs/ldap.yaml
   if [ "${USE_OCM_IDP}" -eq 1 ]
       then
           ocm_clusterid=$(ocm list clusters  --no-headers --parameter search="api.url = '${OC_HOST}'" | awk '{print $1}')
@@ -213,14 +213,14 @@ function set_ldap_users(){
             echo "Cluster $OC_HOST not found. Please fix it and try again..."
             exit 1
           fi
-          cp  ods_ci/utils/scripts/ocm/templates/create_ldap_idp.jinja  ods_ci/configs/create_ldap_idp.jinja
-          sed -i "s/{{ LDAP_BIND_PASSWORD }}/$RAND_ADMIN/g" ods_ci/configs/create_ldap_idp.jinja
-          sed -i "s/{{ LDAP_BIND_DN }}/cn=admin,dc=example,dc=org/g" ods_ci/configs/create_ldap_idp.jinja
-          sed -i 's/{{ LDAP_URL }}/ldap:\/\/openldap.openldap.svc.cluster.local:1389\/dc=example,dc=org?uid/g' ods_ci/configs/create_ldap_idp.jinja
-          ocm post /api/clusters_mgmt/v1/clusters/${ocm_clusterid}/identity_providers --body=ods_ci/configs/create_ldap_idp.jinja
+          cp  utils/scripts/ocm/templates/create_ldap_idp.jinja  configs/create_ldap_idp.jinja
+          sed -i'' -e "s/{{ LDAP_BIND_PASSWORD }}/$RAND_ADMIN/g" configs/create_ldap_idp.jinja
+          sed -i'' -e "s/{{ LDAP_BIND_DN }}/cn=admin,dc=example,dc=org/g" configs/create_ldap_idp.jinja
+          sed -i'' -e 's/{{ LDAP_URL }}/ldap:\/\/openldap.openldap.svc.cluster.local:1389\/dc=example,dc=org?uid/g' configs/create_ldap_idp.jinja
+          ocm post /api/clusters_mgmt/v1/clusters/${ocm_clusterid}/identity_providers --body=configs/create_ldap_idp.jinja
       else
           oc create secret generic ldap-bind-password --from-literal=bindPassword="$RAND_ADMIN" -n openshift-config
-          OAUTH_LDAP_JSON="$(cat ods_ci/configs/resources/oauth_ldap_idp.json)"
+          OAUTH_LDAP_JSON="$(cat configs/resources/oauth_ldap_idp.json)"
           oc patch oauth cluster --type json -p '[{"op": "add", "path": "/spec/identityProviders/-", "value": '"$OAUTH_LDAP_JSON"'}]'
   fi
 
@@ -230,21 +230,21 @@ function set_ldap_users(){
   test_user_4=$(extract_testvariables_users_mapping  ldap TEST_USER_4 $ldap_users_str)
   export ldap_pw=$ldap_pw
   export username=$test_user
-  yq --inplace '.TEST_USER.AUTH_TYPE="ldap-provider-qe"' ods_ci/test-variables.yml
-  yq --inplace '.TEST_USER.USERNAME=env(username)' ods_ci/test-variables.yml
-  yq --inplace '.TEST_USER.PASSWORD=env(ldap_pw)' ods_ci/test-variables.yml
+  yq --inplace '.TEST_USER.AUTH_TYPE="ldap-provider-qe"' test-variables.yml
+  yq --inplace '.TEST_USER.USERNAME=env(username)' test-variables.yml
+  yq --inplace '.TEST_USER.PASSWORD=env(ldap_pw)' test-variables.yml
   export username=$test_user_2
-  yq --inplace '.TEST_USER_2.AUTH_TYPE="ldap-provider-qe"' ods_ci/test-variables.yml
-  yq --inplace '.TEST_USER_2.USERNAME=env(username)' ods_ci/test-variables.yml
-  yq --inplace '.TEST_USER_2.PASSWORD=env(ldap_pw)' ods_ci/test-variables.yml
+  yq --inplace '.TEST_USER_2.AUTH_TYPE="ldap-provider-qe"' test-variables.yml
+  yq --inplace '.TEST_USER_2.USERNAME=env(username)' test-variables.yml
+  yq --inplace '.TEST_USER_2.PASSWORD=env(ldap_pw)' test-variables.yml
   export username=$test_user_3
-  yq --inplace '.TEST_USER_3.AUTH_TYPE="ldap-provider-qe"' ods_ci/test-variables.yml
-  yq --inplace '.TEST_USER_3.USERNAME=env(username)' ods_ci/test-variables.yml
-  yq --inplace '.TEST_USER_3.PASSWORD=env(ldap_pw)' ods_ci/test-variables.yml
+  yq --inplace '.TEST_USER_3.AUTH_TYPE="ldap-provider-qe"' test-variables.yml
+  yq --inplace '.TEST_USER_3.USERNAME=env(username)' test-variables.yml
+  yq --inplace '.TEST_USER_3.PASSWORD=env(ldap_pw)' test-variables.yml
   export username=$test_user_4
-  yq --inplace '.TEST_USER_4.AUTH_TYPE="ldap-provider-qe"' ods_ci/test-variables.yml
-  yq --inplace '.TEST_USER_4.USERNAME=env(username)' ods_ci/test-variables.yml
-  yq --inplace '.TEST_USER_4.PASSWORD=env(ldap_pw)' ods_ci/test-variables.yml
+  yq --inplace '.TEST_USER_4.AUTH_TYPE="ldap-provider-qe"' test-variables.yml
+  yq --inplace '.TEST_USER_4.USERNAME=env(username)' test-variables.yml
+  yq --inplace '.TEST_USER_4.PASSWORD=env(ldap_pw)' test-variables.yml
 
 }
 
@@ -254,7 +254,7 @@ function create_groups_and_assign_users(){
   oc adm groups new rhods-noaccess
   oc adm groups new dedicated-admins
   for prefix in "${prefixes[@]}"; do
-    groups=$(jq -r --arg idpname ldap --arg pref $prefix '.[][$idpname].groups_map[$pref][]' ods_ci/configs/templates/user_config.json)
+    groups=$(jq -r --arg idpname ldap --arg pref $prefix '.[][$idpname].groups_map[$pref][]' configs/templates/user_config.json)
     echo $groups
     groups=($groups)
     for group in "${groups[@]}"; do
@@ -294,7 +294,7 @@ function install_identity_provider(){
   echo "Stage) Setting LDAP Identity provider"
   set_ldap_users
   echo "Stage) Configure RHODS test user groups"
-  create_groups_and_assign_users  
+  create_groups_and_assign_users
   echo "Stage) Sleeping 180sec to wait for IDPs to become available"
   sleep 180
 }
@@ -335,7 +335,7 @@ function check_installation(){
 
 function validate_user_config_fields_and_values(){
   echo "--> Going through requested users configuration"
-  idp=$(jq --arg idpname $1 '.[][$idpname]' ods_ci/configs/templates/user_config.json)
+  idp=$(jq --arg idpname $1 '.[][$idpname]' configs/templates/user_config.json)
   pw=$(echo $idp | jq -r '.pw')
   if  [[ $pw = "null" || -z "${pw// }" ]]; then
     echo ".pw must be set with a custom value (i.e., no empty string or whitespacesonly) or <GEN_RAMDOM_PW>"
@@ -424,7 +424,7 @@ function validate_user_config_fields_and_values(){
             echo "The given suffix type $suffix_type is not supported..try again"
       esac
       if [[ $1 = "ldap" ]]; then
-        groups=$(jq -r --arg idpname ldap --arg pref $prefix '.[][$idpname].groups_map[$pref][]' ods_ci/configs/templates/user_config.json)
+        groups=$(jq -r --arg idpname ldap --arg pref $prefix '.[][$idpname].groups_map[$pref][]' configs/templates/user_config.json)
         groups=($groups)
         if  [[ $groups = "null" || -z "${groups// }" || ! ${#groups[@]} -gt 0 ]]; then
               echo ".groups_map must be set and have more than 0 element!"
@@ -436,12 +436,12 @@ function validate_user_config_fields_and_values(){
 
 function validate_user_config_file(){
   echo "Stage) validating user_config.json"
-  if [ ! -f "ods_ci/configs/templates/user_config.json" ]; then
-    echo user_config.json is not present in ods_ci/configs/templates! Fix it and try again...
+  if [ ! -f "configs/templates/user_config.json" ]; then
+    echo user_config.json is not present in configs/templates! Fix it and try again...
     exit 1
   else
     echo user_config.json found! Starting json validation..
-  fi  
+  fi
   validate_user_config_fields_and_values htpasswd
   validate_user_config_fields_and_values ldap
 }

ods_ci/build/run.sh

@@ -21,10 +21,10 @@ if [ "${SET_ENVIRONMENT}" -eq 1 ]; then \
             fi
         fi
         echo "-----| SET_ENVIRONMENT option is enabled. ODS-CI is going to configure the test environment for you..|-----"
-        ./ods_ci/build/install_idp.sh
+        ./build/install_idp.sh
   fi
 fi
 echo "-----| ODS-CI is starting the tests run...|-----"
-./ods_ci/run_robot_test.sh --skip-install ${RUN_SCRIPT_ARGS} --extra-robot-args "${ROBOT_EXTRA_ARGS}"
+./run_robot_test.sh --skip-install ${RUN_SCRIPT_ARGS} --extra-robot-args "${ROBOT_EXTRA_ARGS}"
 
 

ods_ci/run_robot_test.sh

@@ -4,10 +4,10 @@ SKIP_OC_LOGIN=false
 SERVICE_ACCOUNT=""
 SA_NAMESPACE="default"
 SET_RHODS_URLS=false
-TEST_CASE_FILE=ods_ci/tests/Tests
-TEST_VARIABLES_FILE=ods_ci/test-variables.yml
+TEST_CASE_FILE=tests/Tests
+TEST_VARIABLES_FILE=test-variables.yml
 TEST_VARIABLES=""
-TEST_ARTIFACT_DIR="ods_ci/test-output"
+TEST_ARTIFACT_DIR="test-output"
 EXTRA_ROBOT_ARGS=""
 SKIP_INSTALL=0
 TEST_INCLUDE_TAG=""
@@ -388,7 +388,7 @@ if ${EMAIL_REPORT}
             rm rf_results.tar.gz
             tar cvzf rf_results.tar.gz $(find "${TEST_ARTIFACT_DIR}" -regex  '.*\(xml\|html\)$') &> /dev/null
      fi
-     python3 ods_ci/utils/scripts/Sender/send_report.py send_email_report -s "${EMAIL_FROM}" -r "${EMAIL_TO}" -b "ODS-CI: Run Results" \
+     python3 utils/scripts/Sender/send_report.py send_email_report -s "${EMAIL_FROM}" -r "${EMAIL_TO}" -b "ODS-CI: Run Results" \
                         -v "${EMAIL_SERVER}" -a "rf_results.tar.gz" -u "${EMAIL_SERVER_USER}" -p "${EMAIL_SERVER_PW}" \
                         -l "${EMAIL_SERVER_SSL}" -d "${EMAIL_SERVER_UNSECURE}"
 fi

ods_ci/tests/Resources/CLI/AcceleratorProfiles/AcceleratorProfiles.resource

@@ -3,7 +3,7 @@ Documentation    Collection of keywords to interact with the administration of A
 
 
 *** Variables ***
-${ACCELERATOR_RESOURCES_DIRPATH}=    ods_ci/tests/Resources/Files/accelerator
+${ACCELERATOR_RESOURCES_DIRPATH}=    tests/Resources/Files/accelerator
 
 
 *** Keywords ***