Adding run_rename_benchmark bash script to be used for kokoro periodi…

…c tests (#2449)

* added check directory functions and its unit tests

* adding function to parse config file and generate dir in bucket

* exit_code set to 1

* correct format

* refactors

* test formatting

* function to avoid code repitition

* added check directory functions and its unit tests

* moving mount functions to utils

* compute metrics from time of operation

* uploading metrics to gsheet

* nits

* added logic to accumulate nested rename folder metrics

* getting VM metrics relevant to rename operation

* added unit test and function description

* adding bash script for kokoro periodic tests

* correcting list dir function

perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh

@@ -87,6 +87,10 @@ LIST_CONFIG_FILE="config-hns.json"
 run_load_test_and_fetch_metrics "$GCSFUSE_FIO_FLAGS" "$BUCKET_NAME" "$SPREADSHEET_ID"
 run_ls_benchmark "$GCSFUSE_LS_FLAGS" "$SPREADSHEET_ID" "$LIST_CONFIG_FILE"
 
+#Running the rename benchmark script.
+cd "./hns_rename_folders_metrics"
+./run_rename_benchmark.sh $UPLOAD_FLAGS
+
 
 # TODO: Testing for hns bucket with client protocol set to grpc. To be done when
 #  includeFolderAsPrefixes is supported in grpc.

perfmetrics/scripts/hns_rename_folders_metrics/generate_folders_and_files.py

@@ -99,6 +99,10 @@ def _list_directory(path) -> list:
     contents = subprocess.check_output(
         'gcloud storage ls {}'.format(path), shell=True)
     contents_url = contents.decode('utf-8').split('\n')[:-1]
+    # gcloud storage ls returns current directory with "/" suffix sometimes which can lead to
+    # false inconsistency when listing.
+    if contents_url and contents_url[0] == "{}/".format(path):
+      contents_url = contents_url[1:]
     return contents_url
   except subprocess.CalledProcessError as e:
     _logmessage(e.output.decode('utf-8'),LOG_ERROR)

perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark_test.py

@@ -432,5 +432,17 @@ def test_get_upload_value_for_vm_metrics(self):
     self.assertEqual(upload_values,expected_values)
 
 
+  def test_get_upload_value_for_vm_metrics(self):
+    vm_metrics = {
+        'test_folder1': [1,2,3],
+        'test_folder2': [1,2,3]
+    }
+    expected_values= [['test_folder1',1,2,3],['test_folder2',1,2,3]]
+
+    upload_values = renaming_benchmark._get_upload_value_for_vm_metrics(vm_metrics)
+
+    self.assertEqual(upload_values,expected_values)
+
+
 if __name__ == '__main__':
   unittest.main()

perfmetrics/scripts/hns_rename_folders_metrics/requirements.in

@@ -16,3 +16,4 @@ statistics
 numpy
 google-api-python-client
 google-auth
+google-cloud-monitoring