evaluation/1_QueryData.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running in non-CI mode\n",
      "False\n",
      "experiment-advanced-mongo-with-latency-12-test-orchestrator\n"
     ]
    }
   ],
   "source": [
    "try: \n",
    "    print(ci)\n",
    "except Exception:  \n",
    "    ci = False\n",
    "# Following variables can be set here or via papermill\n",
    "\n",
    "if ci == False:\n",
    "    print(\"Running in non-CI mode\")\n",
    "    experiment = \"experiment-advanced-mongo-with-latency-12-test-orchestrator\"\n",
    "\n",
    "print(ci)\n",
    "print(experiment)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.oauth2 import service_account\n",
    "from google.cloud import monitoring_v3\n",
    "from google.cloud import bigquery\n",
    "from google.cloud import storage\n",
    "from google.oauth2 import service_account\n",
    "from google.cloud.bigquery.job import ExtractJobConfig\n",
    "import time\n",
    "import matplotlib.dates as mdates\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "import os\n",
    "from pathlib import Path\n",
    "import datetime\n",
    "import shutil\n",
    "import subprocess\n",
    "from subprocess import PIPE\n",
    "import sharedVariables\n",
    "from sharedVariables import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Network Logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "credentials = service_account.Credentials.from_service_account_file(\n",
    "    '../infrastructure/credentials.json')\n",
    "\n",
    "client = bigquery.Client(credentials=credentials)\n",
    "today = datetime.datetime.today()\n",
    "table_id = f\"compute_googleapis_com_vpc_flows_{today.strftime('%Y%m%d')}\"\n",
    "# table_id = 'compute_googleapis_com_vpc_flows_20210720'\n",
    "blob_name = \"export.log\"\n",
    "\n",
    "\n",
    "\n",
    "extract_conf = ExtractJobConfig()\n",
    "extract_conf.compression = 'NONE'\n",
    "extract_conf.destination_format = 'NEWLINE_DELIMITED_JSON'\n",
    "\n",
    "def getDataset(experiment, outDir = datetime.datetime.today().strftime('%Y%m%d-%H')): \n",
    "    ensureDirectory(outDir)\n",
    "    bucket_name = experiment + \"-log-bucket\"\n",
    "\n",
    "    destination_uri = \"gs://{}/{}\".format(bucket_name, blob_name)\n",
    "    dataset_ref = bigquery.DatasetReference(project, experiment.replace(\"-\", \"_\"))\n",
    "    table_ref = dataset_ref.table(table_id)\n",
    "\n",
    "    extract_job = client.extract_table(\n",
    "        table_ref,\n",
    "        destination_uri,\n",
    "        # Location must match that of the source table.\n",
    "        location=\"US\",\n",
    "        job_config=extract_conf\n",
    "    )  # API request\n",
    "    extract_job.result()  # Waits for job to complete.\n",
    "\n",
    "    print(\n",
    "        \"Exported {}:{}.{} to {}\".format(project, experiment, table_id, destination_uri)\n",
    "    )\n",
    "\n",
    "\n",
    "    storage_client = storage.Client(credentials=credentials)\n",
    "\n",
    "    bucket = storage_client.bucket(bucket_name)\n",
    "    blob = bucket.blob(blob_name)\n",
    "    blob.download_to_filename(f\"{outDir}/gcp-flow-network.log\")\n",
    "\n",
    "    print(\n",
    "        \"Blob downloaded successfully.\"\n",
    "    )\n",
    "\n",
    "if 'experiment' not in locals():\n",
    "    for experiment in sharedVariables.experiments: \n",
    "        try: \n",
    "            getDataset(experiment)\n",
    "        except Exception as e: \n",
    "            print(e)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PCAPs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_pcap(ip, outDir = datetime.datetime.today().strftime('%Y%m%d-%H')):\n",
    "    ensureDirectory(outDir)\n",
    "    print(\"Dowloading File\")\n",
    "    print(os.path.join(Path.cwd(), \"..\", \"cert\"))\n",
    "    result = subprocess.run(f\"scp -i ../infrastructure/orchestrator.pem -o StrictHostKeyChecking=no orchestrator@{ip}:/captures.zip ./captures.zip\", shell=True, stdout=PIPE, stderr=PIPE, universal_newlines=True)\n",
    "    if(result.returncode == 0):\n",
    "        print(result.stdout)\n",
    "    else:\n",
    "        raise Exception(result.stderr)\n",
    "    shutil.unpack_archive(\"./captures.zip\", outDir)\n",
    "\n",
    "if 'experiment' not in locals():\n",
    "    download_pcap(\"35.224.133.98\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Monitoring"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Timeframe: 3.0 hours\n"
     ]
    }
   ],
   "source": [
    "credentials = service_account.Credentials.from_service_account_file(\n",
    "    '../infrastructure/credentials.json')\n",
    "\n",
    "# Our project ID\n",
    "\n",
    "monitoring_client = monitoring_v3.MetricServiceClient(credentials=credentials)\n",
    "project_name = f\"projects/{project}\"\n",
    "now = time.time()\n",
    "seconds = int(now)\n",
    "nanos = int((now - seconds) * 10 ** 9)\n",
    "\n",
    "# 3600 = Get the last hour of metrics\n",
    "timeframe = 3600 * 3\n",
    "if 'experiment' not in locals():\n",
    "    print(\"set timeframe to one hour\")\n",
    "    timeframe = 3600 * 1\n",
    "\n",
    "print(\"Timeframe:\", timeframe / 3600, \"hours\")\n",
    "interval = monitoring_v3.TimeInterval(\n",
    "    {\n",
    "        \"end_time\": {\"seconds\": seconds, \"nanos\": nanos},\n",
    "        \"start_time\": {\"seconds\": (seconds - timeframe), \"nanos\": nanos},\n",
    "    }\n",
    ")\n",
    "\n",
    "\n",
    "# Add Filter? metric.label.instance_name = \"your-instance-id\"\n",
    "results_cpu = monitoring_client.list_time_series(\n",
    "    request={\n",
    "        \"name\": project_name,\n",
    "        \"filter\": 'metric.type =  \"compute.googleapis.com/instance/cpu/utilization\"',\n",
    "        \"interval\": interval,\n",
    "        \"view\": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,\n",
    "    }\n",
    ")\n",
    "results_io_read = monitoring_client.list_time_series(\n",
    "    request={\n",
    "        \"name\": project_name,\n",
    "        \"filter\": 'metric.type = \"compute.googleapis.com/instance/disk/read_bytes_count\"',\n",
    "        \"interval\": interval,\n",
    "        \"view\": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,\n",
    "    }\n",
    ")\n",
    "results_io_write = monitoring_client.list_time_series(\n",
    "    request={\n",
    "        \"name\": project_name,\n",
    "        \"filter\": 'metric.type = \"compute.googleapis.com/instance/disk/write_bytes_count\"',\n",
    "        \"interval\": interval,\n",
    "        \"view\": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,\n",
    "    }\n",
    ")\n",
    "results_iops_read = monitoring_client.list_time_series(\n",
    "    request={\n",
    "        \"name\": project_name,\n",
    "        \"filter\": 'metric.type = \"compute.googleapis.com/instance/disk/read_ops_count\"',\n",
    "        \"interval\": interval,\n",
    "        \"view\": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,\n",
    "    }\n",
    ")\n",
    "results_iops_write = monitoring_client.list_time_series(\n",
    "    request={\n",
    "        \"name\": project_name,\n",
    "        \"filter\": 'metric.type = \"compute.googleapis.com/instance/disk/write_ops_count\"',\n",
    "        \"interval\": interval,\n",
    "        \"view\": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,\n",
    "    }\n",
    ")\n",
    "\n",
    "mutli_results_memory = {}\n",
    "mem_states = [\"buffered\", \"cached\", \"free\", \"slab\", \"used\"]\n",
    "for state in mem_states:\n",
    "    mutli_results_memory[state] = monitoring_client.list_time_series(\n",
    "        request={\n",
    "            \"name\": project_name,\n",
    "            \"filter\": f'metric.type = \"agent.googleapis.com/memory/bytes_used\" AND metric.labels.state = \"{state}\"',\n",
    "            \"interval\": interval,\n",
    "            \"view\": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,\n",
    "        }\n",
    "    )\n",
    "\n",
    "mutli_results_memory_percentage = {}\n",
    "for state in mem_states:\n",
    "    mutli_results_memory_percentage[state] = monitoring_client.list_time_series(\n",
    "        request={\n",
    "            \"name\": project_name,\n",
    "            \"filter\": f'metric.type = \"agent.googleapis.com/memory/percent_used\" AND metric.labels.state = \"{state}\"',\n",
    "            \"interval\": interval,\n",
    "            \"view\": monitoring_v3.ListTimeSeriesRequest.TimeSeriesView.FULL,\n",
    "        }\n",
    "    )\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cpu_util_', 'io_read_', 'io_write_', 'iops_read_', 'iops_write_', 'mem_buffered_', 'mem_cached_', 'mem_free_', 'mem_slab_', 'mem_used_', 'mem_perc_buffered_', 'mem_perc_cached_', 'mem_perc_free_', 'mem_perc_slab_', 'mem_perc_used_']\n",
      "experiment-advanced-mongo-with-latency-12-test-orchestrator 34\n",
      "Overall min len: 33\n",
      "SET min len: 33\n",
      "Gathered all Monitoring data for experiment-advanced-mongo-with-latency-12-test-orchestrator\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cpu_util_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>io_read_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>io_write_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>iops_read_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>iops_write_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>mem_buffered_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>mem_cached_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>mem_free_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>mem_slab_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>mem_used_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>mem_perc_buffered_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>mem_perc_cached_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>mem_perc_free_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>mem_perc_slab_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "      <th>mem_perc_used_experiment-advanced-mongo-with-latency-12-test-orchestrator</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>timestamp</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:33:00</th>\n",
       "      <td>0.001318</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106295296.0</td>\n",
       "      <td>1.928245e+09</td>\n",
       "      <td>1.503527e+09</td>\n",
       "      <td>188825600.0</td>\n",
       "      <td>123408384.0</td>\n",
       "      <td>2.760701</td>\n",
       "      <td>50.080371</td>\n",
       "      <td>39.049590</td>\n",
       "      <td>4.904177</td>\n",
       "      <td>3.205162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:32:00</th>\n",
       "      <td>0.023193</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106295296.0</td>\n",
       "      <td>1.927901e+09</td>\n",
       "      <td>1.465037e+09</td>\n",
       "      <td>190091264.0</td>\n",
       "      <td>160976896.0</td>\n",
       "      <td>2.760701</td>\n",
       "      <td>50.071435</td>\n",
       "      <td>38.049925</td>\n",
       "      <td>4.937049</td>\n",
       "      <td>4.180891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:31:00</th>\n",
       "      <td>0.033103</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106291200.0</td>\n",
       "      <td>1.927889e+09</td>\n",
       "      <td>1.467040e+09</td>\n",
       "      <td>190111744.0</td>\n",
       "      <td>158969856.0</td>\n",
       "      <td>2.760594</td>\n",
       "      <td>50.071116</td>\n",
       "      <td>38.101945</td>\n",
       "      <td>4.937581</td>\n",
       "      <td>4.128764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:30:00</th>\n",
       "      <td>0.006065</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106283008.0</td>\n",
       "      <td>1.927983e+09</td>\n",
       "      <td>1.387250e+09</td>\n",
       "      <td>190488576.0</td>\n",
       "      <td>238297088.0</td>\n",
       "      <td>2.760381</td>\n",
       "      <td>50.073563</td>\n",
       "      <td>36.029638</td>\n",
       "      <td>4.947368</td>\n",
       "      <td>6.189050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:29:00</th>\n",
       "      <td>0.003152</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106283008.0</td>\n",
       "      <td>1.927983e+09</td>\n",
       "      <td>1.387377e+09</td>\n",
       "      <td>190513152.0</td>\n",
       "      <td>238145536.0</td>\n",
       "      <td>2.760381</td>\n",
       "      <td>50.073563</td>\n",
       "      <td>36.032936</td>\n",
       "      <td>4.948006</td>\n",
       "      <td>6.185114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:28:00</th>\n",
       "      <td>0.003260</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106270720.0</td>\n",
       "      <td>1.927983e+09</td>\n",
       "      <td>1.387213e+09</td>\n",
       "      <td>190525440.0</td>\n",
       "      <td>238309376.0</td>\n",
       "      <td>2.760062</td>\n",
       "      <td>50.073563</td>\n",
       "      <td>36.028680</td>\n",
       "      <td>4.948325</td>\n",
       "      <td>6.189369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:27:00</th>\n",
       "      <td>0.007198</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106270720.0</td>\n",
       "      <td>1.927983e+09</td>\n",
       "      <td>1.387532e+09</td>\n",
       "      <td>190500864.0</td>\n",
       "      <td>238014464.0</td>\n",
       "      <td>2.760062</td>\n",
       "      <td>50.073563</td>\n",
       "      <td>36.036978</td>\n",
       "      <td>4.947687</td>\n",
       "      <td>6.181710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:26:00</th>\n",
       "      <td>0.009411</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106266624.0</td>\n",
       "      <td>1.927987e+09</td>\n",
       "      <td>1.385083e+09</td>\n",
       "      <td>190570496.0</td>\n",
       "      <td>240394240.0</td>\n",
       "      <td>2.759956</td>\n",
       "      <td>50.073669</td>\n",
       "      <td>35.973362</td>\n",
       "      <td>4.949495</td>\n",
       "      <td>6.243517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:25:00</th>\n",
       "      <td>0.004963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106266624.0</td>\n",
       "      <td>1.927987e+09</td>\n",
       "      <td>1.385083e+09</td>\n",
       "      <td>190578688.0</td>\n",
       "      <td>240386048.0</td>\n",
       "      <td>2.759956</td>\n",
       "      <td>50.073669</td>\n",
       "      <td>35.973362</td>\n",
       "      <td>4.949708</td>\n",
       "      <td>6.243305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:24:00</th>\n",
       "      <td>0.022527</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106262528.0</td>\n",
       "      <td>1.927987e+09</td>\n",
       "      <td>1.386648e+09</td>\n",
       "      <td>190484480.0</td>\n",
       "      <td>238919680.0</td>\n",
       "      <td>2.759850</td>\n",
       "      <td>50.073669</td>\n",
       "      <td>36.014000</td>\n",
       "      <td>4.947261</td>\n",
       "      <td>6.205220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:23:00</th>\n",
       "      <td>0.028903</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106254336.0</td>\n",
       "      <td>1.927844e+09</td>\n",
       "      <td>1.464975e+09</td>\n",
       "      <td>190128128.0</td>\n",
       "      <td>161099776.0</td>\n",
       "      <td>2.759637</td>\n",
       "      <td>50.069946</td>\n",
       "      <td>38.048329</td>\n",
       "      <td>4.938006</td>\n",
       "      <td>4.184082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:22:00</th>\n",
       "      <td>0.023226</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106254336.0</td>\n",
       "      <td>1.927844e+09</td>\n",
       "      <td>1.465135e+09</td>\n",
       "      <td>190128128.0</td>\n",
       "      <td>160940032.0</td>\n",
       "      <td>2.759637</td>\n",
       "      <td>50.069946</td>\n",
       "      <td>38.052478</td>\n",
       "      <td>4.938006</td>\n",
       "      <td>4.179933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:21:00</th>\n",
       "      <td>0.030824</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106250240.0</td>\n",
       "      <td>1.927836e+09</td>\n",
       "      <td>1.467163e+09</td>\n",
       "      <td>190156800.0</td>\n",
       "      <td>158896128.0</td>\n",
       "      <td>2.759530</td>\n",
       "      <td>50.069733</td>\n",
       "      <td>38.105137</td>\n",
       "      <td>4.938751</td>\n",
       "      <td>4.126849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:20:00</th>\n",
       "      <td>0.006851</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106250240.0</td>\n",
       "      <td>1.927836e+09</td>\n",
       "      <td>1.467130e+09</td>\n",
       "      <td>190156800.0</td>\n",
       "      <td>158928896.0</td>\n",
       "      <td>2.759530</td>\n",
       "      <td>50.069733</td>\n",
       "      <td>38.104286</td>\n",
       "      <td>4.938751</td>\n",
       "      <td>4.127700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:19:00</th>\n",
       "      <td>0.008141</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106250240.0</td>\n",
       "      <td>1.927836e+09</td>\n",
       "      <td>1.467290e+09</td>\n",
       "      <td>190156800.0</td>\n",
       "      <td>158769152.0</td>\n",
       "      <td>2.759530</td>\n",
       "      <td>50.069733</td>\n",
       "      <td>38.108434</td>\n",
       "      <td>4.938751</td>\n",
       "      <td>4.123551</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:18:00</th>\n",
       "      <td>0.003790</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106250240.0</td>\n",
       "      <td>1.927836e+09</td>\n",
       "      <td>1.467257e+09</td>\n",
       "      <td>190156800.0</td>\n",
       "      <td>158801920.0</td>\n",
       "      <td>2.759530</td>\n",
       "      <td>50.069733</td>\n",
       "      <td>38.107583</td>\n",
       "      <td>4.938751</td>\n",
       "      <td>4.124402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:17:00</th>\n",
       "      <td>0.004336</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106246144.0</td>\n",
       "      <td>1.927582e+09</td>\n",
       "      <td>1.457136e+09</td>\n",
       "      <td>190562304.0</td>\n",
       "      <td>168775680.0</td>\n",
       "      <td>2.759424</td>\n",
       "      <td>50.063137</td>\n",
       "      <td>37.844715</td>\n",
       "      <td>4.949283</td>\n",
       "      <td>4.383441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:16:00</th>\n",
       "      <td>0.006806</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106237952.0</td>\n",
       "      <td>1.927586e+09</td>\n",
       "      <td>1.454969e+09</td>\n",
       "      <td>190738432.0</td>\n",
       "      <td>170770432.0</td>\n",
       "      <td>2.759211</td>\n",
       "      <td>50.063244</td>\n",
       "      <td>37.788440</td>\n",
       "      <td>4.953857</td>\n",
       "      <td>4.435248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:15:00</th>\n",
       "      <td>0.010376</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106233856.0</td>\n",
       "      <td>1.927582e+09</td>\n",
       "      <td>1.454744e+09</td>\n",
       "      <td>190787584.0</td>\n",
       "      <td>170954752.0</td>\n",
       "      <td>2.759105</td>\n",
       "      <td>50.063137</td>\n",
       "      <td>37.782589</td>\n",
       "      <td>4.955134</td>\n",
       "      <td>4.440036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:14:00</th>\n",
       "      <td>0.006176</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106229760.0</td>\n",
       "      <td>1.927598e+09</td>\n",
       "      <td>1.445835e+09</td>\n",
       "      <td>191070208.0</td>\n",
       "      <td>179568640.0</td>\n",
       "      <td>2.758999</td>\n",
       "      <td>50.063563</td>\n",
       "      <td>37.551209</td>\n",
       "      <td>4.962474</td>\n",
       "      <td>4.663755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:13:00</th>\n",
       "      <td>0.003975</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106229760.0</td>\n",
       "      <td>1.927545e+09</td>\n",
       "      <td>1.445835e+09</td>\n",
       "      <td>191115264.0</td>\n",
       "      <td>179576832.0</td>\n",
       "      <td>2.758999</td>\n",
       "      <td>50.062180</td>\n",
       "      <td>37.551209</td>\n",
       "      <td>4.963644</td>\n",
       "      <td>4.663968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:12:00</th>\n",
       "      <td>0.023854</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106229760.0</td>\n",
       "      <td>1.927545e+09</td>\n",
       "      <td>1.446081e+09</td>\n",
       "      <td>191004672.0</td>\n",
       "      <td>179441664.0</td>\n",
       "      <td>2.758999</td>\n",
       "      <td>50.062180</td>\n",
       "      <td>37.557592</td>\n",
       "      <td>4.960772</td>\n",
       "      <td>4.660458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:11:00</th>\n",
       "      <td>0.033929</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106213376.0</td>\n",
       "      <td>1.927537e+09</td>\n",
       "      <td>1.448403e+09</td>\n",
       "      <td>190976000.0</td>\n",
       "      <td>177172480.0</td>\n",
       "      <td>2.758573</td>\n",
       "      <td>50.061967</td>\n",
       "      <td>37.617910</td>\n",
       "      <td>4.960027</td>\n",
       "      <td>4.601522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:10:00</th>\n",
       "      <td>0.006624</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106180608.0</td>\n",
       "      <td>1.927537e+09</td>\n",
       "      <td>1.380766e+09</td>\n",
       "      <td>191086592.0</td>\n",
       "      <td>244731904.0</td>\n",
       "      <td>2.757722</td>\n",
       "      <td>50.061967</td>\n",
       "      <td>35.861236</td>\n",
       "      <td>4.962900</td>\n",
       "      <td>6.356175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:09:00</th>\n",
       "      <td>0.003769</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106180608.0</td>\n",
       "      <td>1.927537e+09</td>\n",
       "      <td>1.380897e+09</td>\n",
       "      <td>191098880.0</td>\n",
       "      <td>244588544.0</td>\n",
       "      <td>2.757722</td>\n",
       "      <td>50.061967</td>\n",
       "      <td>35.864640</td>\n",
       "      <td>4.963219</td>\n",
       "      <td>6.352452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:08:00</th>\n",
       "      <td>0.003069</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106180608.0</td>\n",
       "      <td>1.927537e+09</td>\n",
       "      <td>1.381126e+09</td>\n",
       "      <td>191119360.0</td>\n",
       "      <td>244338688.0</td>\n",
       "      <td>2.757722</td>\n",
       "      <td>50.061967</td>\n",
       "      <td>35.870598</td>\n",
       "      <td>4.963751</td>\n",
       "      <td>6.345963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:07:00</th>\n",
       "      <td>0.006217</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106180608.0</td>\n",
       "      <td>1.927537e+09</td>\n",
       "      <td>1.380893e+09</td>\n",
       "      <td>191098880.0</td>\n",
       "      <td>244592640.0</td>\n",
       "      <td>2.757722</td>\n",
       "      <td>50.061967</td>\n",
       "      <td>35.864534</td>\n",
       "      <td>4.963219</td>\n",
       "      <td>6.352558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:06:00</th>\n",
       "      <td>0.007981</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106180608.0</td>\n",
       "      <td>1.927541e+09</td>\n",
       "      <td>1.378222e+09</td>\n",
       "      <td>191279104.0</td>\n",
       "      <td>247078912.0</td>\n",
       "      <td>2.757722</td>\n",
       "      <td>50.062073</td>\n",
       "      <td>35.795173</td>\n",
       "      <td>4.967899</td>\n",
       "      <td>6.417132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:05:00</th>\n",
       "      <td>0.003317</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106180608.0</td>\n",
       "      <td>1.927541e+09</td>\n",
       "      <td>1.377993e+09</td>\n",
       "      <td>191311872.0</td>\n",
       "      <td>247275520.0</td>\n",
       "      <td>2.757722</td>\n",
       "      <td>50.062073</td>\n",
       "      <td>35.789216</td>\n",
       "      <td>4.968750</td>\n",
       "      <td>6.422238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:04:00</th>\n",
       "      <td>0.003309</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106180608.0</td>\n",
       "      <td>1.927541e+09</td>\n",
       "      <td>1.377534e+09</td>\n",
       "      <td>191307776.0</td>\n",
       "      <td>247738368.0</td>\n",
       "      <td>2.757722</td>\n",
       "      <td>50.062073</td>\n",
       "      <td>35.777301</td>\n",
       "      <td>4.968644</td>\n",
       "      <td>6.434259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:03:00</th>\n",
       "      <td>0.010118</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106180608.0</td>\n",
       "      <td>1.927541e+09</td>\n",
       "      <td>1.377980e+09</td>\n",
       "      <td>191193088.0</td>\n",
       "      <td>247406592.0</td>\n",
       "      <td>2.757722</td>\n",
       "      <td>50.062073</td>\n",
       "      <td>35.788897</td>\n",
       "      <td>4.965665</td>\n",
       "      <td>6.425642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:02:00</th>\n",
       "      <td>0.049694</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106172416.0</td>\n",
       "      <td>1.927541e+09</td>\n",
       "      <td>1.378173e+09</td>\n",
       "      <td>191127552.0</td>\n",
       "      <td>247287808.0</td>\n",
       "      <td>2.757509</td>\n",
       "      <td>50.062073</td>\n",
       "      <td>35.793897</td>\n",
       "      <td>4.963963</td>\n",
       "      <td>6.422557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2022-02-05 16:01:00</th>\n",
       "      <td>0.235243</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>106135552.0</td>\n",
       "      <td>1.926136e+09</td>\n",
       "      <td>1.388532e+09</td>\n",
       "      <td>190664704.0</td>\n",
       "      <td>238833664.0</td>\n",
       "      <td>2.756552</td>\n",
       "      <td>50.025585</td>\n",
       "      <td>36.062935</td>\n",
       "      <td>4.951942</td>\n",
       "      <td>6.202986</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     cpu_util_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                   \n",
       "2022-02-05 16:33:00                                           0.001318                      \n",
       "2022-02-05 16:32:00                                           0.023193                      \n",
       "2022-02-05 16:31:00                                           0.033103                      \n",
       "2022-02-05 16:30:00                                           0.006065                      \n",
       "2022-02-05 16:29:00                                           0.003152                      \n",
       "2022-02-05 16:28:00                                           0.003260                      \n",
       "2022-02-05 16:27:00                                           0.007198                      \n",
       "2022-02-05 16:26:00                                           0.009411                      \n",
       "2022-02-05 16:25:00                                           0.004963                      \n",
       "2022-02-05 16:24:00                                           0.022527                      \n",
       "2022-02-05 16:23:00                                           0.028903                      \n",
       "2022-02-05 16:22:00                                           0.023226                      \n",
       "2022-02-05 16:21:00                                           0.030824                      \n",
       "2022-02-05 16:20:00                                           0.006851                      \n",
       "2022-02-05 16:19:00                                           0.008141                      \n",
       "2022-02-05 16:18:00                                           0.003790                      \n",
       "2022-02-05 16:17:00                                           0.004336                      \n",
       "2022-02-05 16:16:00                                           0.006806                      \n",
       "2022-02-05 16:15:00                                           0.010376                      \n",
       "2022-02-05 16:14:00                                           0.006176                      \n",
       "2022-02-05 16:13:00                                           0.003975                      \n",
       "2022-02-05 16:12:00                                           0.023854                      \n",
       "2022-02-05 16:11:00                                           0.033929                      \n",
       "2022-02-05 16:10:00                                           0.006624                      \n",
       "2022-02-05 16:09:00                                           0.003769                      \n",
       "2022-02-05 16:08:00                                           0.003069                      \n",
       "2022-02-05 16:07:00                                           0.006217                      \n",
       "2022-02-05 16:06:00                                           0.007981                      \n",
       "2022-02-05 16:05:00                                           0.003317                      \n",
       "2022-02-05 16:04:00                                           0.003309                      \n",
       "2022-02-05 16:03:00                                           0.010118                      \n",
       "2022-02-05 16:02:00                                           0.049694                      \n",
       "2022-02-05 16:01:00                                           0.235243                      \n",
       "\n",
       "                     io_read_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                  \n",
       "2022-02-05 16:33:00                                                0.0                     \n",
       "2022-02-05 16:32:00                                                0.0                     \n",
       "2022-02-05 16:31:00                                                0.0                     \n",
       "2022-02-05 16:30:00                                                0.0                     \n",
       "2022-02-05 16:29:00                                                0.0                     \n",
       "2022-02-05 16:28:00                                                0.0                     \n",
       "2022-02-05 16:27:00                                                0.0                     \n",
       "2022-02-05 16:26:00                                                0.0                     \n",
       "2022-02-05 16:25:00                                                0.0                     \n",
       "2022-02-05 16:24:00                                                0.0                     \n",
       "2022-02-05 16:23:00                                                0.0                     \n",
       "2022-02-05 16:22:00                                                0.0                     \n",
       "2022-02-05 16:21:00                                                0.0                     \n",
       "2022-02-05 16:20:00                                                0.0                     \n",
       "2022-02-05 16:19:00                                                0.0                     \n",
       "2022-02-05 16:18:00                                                0.0                     \n",
       "2022-02-05 16:17:00                                                0.0                     \n",
       "2022-02-05 16:16:00                                                0.0                     \n",
       "2022-02-05 16:15:00                                                0.0                     \n",
       "2022-02-05 16:14:00                                                0.0                     \n",
       "2022-02-05 16:13:00                                                0.0                     \n",
       "2022-02-05 16:12:00                                                0.0                     \n",
       "2022-02-05 16:11:00                                                0.0                     \n",
       "2022-02-05 16:10:00                                                0.0                     \n",
       "2022-02-05 16:09:00                                                0.0                     \n",
       "2022-02-05 16:08:00                                                0.0                     \n",
       "2022-02-05 16:07:00                                                0.0                     \n",
       "2022-02-05 16:06:00                                                0.0                     \n",
       "2022-02-05 16:05:00                                                0.0                     \n",
       "2022-02-05 16:04:00                                                0.0                     \n",
       "2022-02-05 16:03:00                                                0.0                     \n",
       "2022-02-05 16:02:00                                                0.0                     \n",
       "2022-02-05 16:01:00                                                0.0                     \n",
       "\n",
       "                     io_write_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                   \n",
       "2022-02-05 16:33:00                                                0.0                      \n",
       "2022-02-05 16:32:00                                                0.0                      \n",
       "2022-02-05 16:31:00                                                0.0                      \n",
       "2022-02-05 16:30:00                                                0.0                      \n",
       "2022-02-05 16:29:00                                                0.0                      \n",
       "2022-02-05 16:28:00                                                0.0                      \n",
       "2022-02-05 16:27:00                                                0.0                      \n",
       "2022-02-05 16:26:00                                                0.0                      \n",
       "2022-02-05 16:25:00                                                0.0                      \n",
       "2022-02-05 16:24:00                                                0.0                      \n",
       "2022-02-05 16:23:00                                                0.0                      \n",
       "2022-02-05 16:22:00                                                0.0                      \n",
       "2022-02-05 16:21:00                                                0.0                      \n",
       "2022-02-05 16:20:00                                                0.0                      \n",
       "2022-02-05 16:19:00                                                0.0                      \n",
       "2022-02-05 16:18:00                                                0.0                      \n",
       "2022-02-05 16:17:00                                                0.0                      \n",
       "2022-02-05 16:16:00                                                0.0                      \n",
       "2022-02-05 16:15:00                                                0.0                      \n",
       "2022-02-05 16:14:00                                                0.0                      \n",
       "2022-02-05 16:13:00                                                0.0                      \n",
       "2022-02-05 16:12:00                                                0.0                      \n",
       "2022-02-05 16:11:00                                                0.0                      \n",
       "2022-02-05 16:10:00                                                0.0                      \n",
       "2022-02-05 16:09:00                                                0.0                      \n",
       "2022-02-05 16:08:00                                                0.0                      \n",
       "2022-02-05 16:07:00                                                0.0                      \n",
       "2022-02-05 16:06:00                                                0.0                      \n",
       "2022-02-05 16:05:00                                                0.0                      \n",
       "2022-02-05 16:04:00                                                0.0                      \n",
       "2022-02-05 16:03:00                                                0.0                      \n",
       "2022-02-05 16:02:00                                                0.0                      \n",
       "2022-02-05 16:01:00                                                0.0                      \n",
       "\n",
       "                     iops_read_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                    \n",
       "2022-02-05 16:33:00                                                0.0                       \n",
       "2022-02-05 16:32:00                                                0.0                       \n",
       "2022-02-05 16:31:00                                                0.0                       \n",
       "2022-02-05 16:30:00                                                0.0                       \n",
       "2022-02-05 16:29:00                                                0.0                       \n",
       "2022-02-05 16:28:00                                                0.0                       \n",
       "2022-02-05 16:27:00                                                0.0                       \n",
       "2022-02-05 16:26:00                                                0.0                       \n",
       "2022-02-05 16:25:00                                                0.0                       \n",
       "2022-02-05 16:24:00                                                0.0                       \n",
       "2022-02-05 16:23:00                                                0.0                       \n",
       "2022-02-05 16:22:00                                                0.0                       \n",
       "2022-02-05 16:21:00                                                0.0                       \n",
       "2022-02-05 16:20:00                                                0.0                       \n",
       "2022-02-05 16:19:00                                                0.0                       \n",
       "2022-02-05 16:18:00                                                0.0                       \n",
       "2022-02-05 16:17:00                                                0.0                       \n",
       "2022-02-05 16:16:00                                                0.0                       \n",
       "2022-02-05 16:15:00                                                0.0                       \n",
       "2022-02-05 16:14:00                                                0.0                       \n",
       "2022-02-05 16:13:00                                                0.0                       \n",
       "2022-02-05 16:12:00                                                0.0                       \n",
       "2022-02-05 16:11:00                                                0.0                       \n",
       "2022-02-05 16:10:00                                                0.0                       \n",
       "2022-02-05 16:09:00                                                0.0                       \n",
       "2022-02-05 16:08:00                                                0.0                       \n",
       "2022-02-05 16:07:00                                                0.0                       \n",
       "2022-02-05 16:06:00                                                0.0                       \n",
       "2022-02-05 16:05:00                                                0.0                       \n",
       "2022-02-05 16:04:00                                                0.0                       \n",
       "2022-02-05 16:03:00                                                0.0                       \n",
       "2022-02-05 16:02:00                                                0.0                       \n",
       "2022-02-05 16:01:00                                                0.0                       \n",
       "\n",
       "                     iops_write_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                     \n",
       "2022-02-05 16:33:00                                                0.0                        \n",
       "2022-02-05 16:32:00                                                0.0                        \n",
       "2022-02-05 16:31:00                                                0.0                        \n",
       "2022-02-05 16:30:00                                                0.0                        \n",
       "2022-02-05 16:29:00                                                0.0                        \n",
       "2022-02-05 16:28:00                                                0.0                        \n",
       "2022-02-05 16:27:00                                                0.0                        \n",
       "2022-02-05 16:26:00                                                0.0                        \n",
       "2022-02-05 16:25:00                                                0.0                        \n",
       "2022-02-05 16:24:00                                                0.0                        \n",
       "2022-02-05 16:23:00                                                0.0                        \n",
       "2022-02-05 16:22:00                                                0.0                        \n",
       "2022-02-05 16:21:00                                                0.0                        \n",
       "2022-02-05 16:20:00                                                0.0                        \n",
       "2022-02-05 16:19:00                                                0.0                        \n",
       "2022-02-05 16:18:00                                                0.0                        \n",
       "2022-02-05 16:17:00                                                0.0                        \n",
       "2022-02-05 16:16:00                                                0.0                        \n",
       "2022-02-05 16:15:00                                                0.0                        \n",
       "2022-02-05 16:14:00                                                0.0                        \n",
       "2022-02-05 16:13:00                                                0.0                        \n",
       "2022-02-05 16:12:00                                                0.0                        \n",
       "2022-02-05 16:11:00                                                0.0                        \n",
       "2022-02-05 16:10:00                                                0.0                        \n",
       "2022-02-05 16:09:00                                                0.0                        \n",
       "2022-02-05 16:08:00                                                0.0                        \n",
       "2022-02-05 16:07:00                                                0.0                        \n",
       "2022-02-05 16:06:00                                                0.0                        \n",
       "2022-02-05 16:05:00                                                0.0                        \n",
       "2022-02-05 16:04:00                                                0.0                        \n",
       "2022-02-05 16:03:00                                                0.0                        \n",
       "2022-02-05 16:02:00                                                0.0                        \n",
       "2022-02-05 16:01:00                                                0.0                        \n",
       "\n",
       "                     mem_buffered_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                       \n",
       "2022-02-05 16:33:00                                        106295296.0                          \n",
       "2022-02-05 16:32:00                                        106295296.0                          \n",
       "2022-02-05 16:31:00                                        106291200.0                          \n",
       "2022-02-05 16:30:00                                        106283008.0                          \n",
       "2022-02-05 16:29:00                                        106283008.0                          \n",
       "2022-02-05 16:28:00                                        106270720.0                          \n",
       "2022-02-05 16:27:00                                        106270720.0                          \n",
       "2022-02-05 16:26:00                                        106266624.0                          \n",
       "2022-02-05 16:25:00                                        106266624.0                          \n",
       "2022-02-05 16:24:00                                        106262528.0                          \n",
       "2022-02-05 16:23:00                                        106254336.0                          \n",
       "2022-02-05 16:22:00                                        106254336.0                          \n",
       "2022-02-05 16:21:00                                        106250240.0                          \n",
       "2022-02-05 16:20:00                                        106250240.0                          \n",
       "2022-02-05 16:19:00                                        106250240.0                          \n",
       "2022-02-05 16:18:00                                        106250240.0                          \n",
       "2022-02-05 16:17:00                                        106246144.0                          \n",
       "2022-02-05 16:16:00                                        106237952.0                          \n",
       "2022-02-05 16:15:00                                        106233856.0                          \n",
       "2022-02-05 16:14:00                                        106229760.0                          \n",
       "2022-02-05 16:13:00                                        106229760.0                          \n",
       "2022-02-05 16:12:00                                        106229760.0                          \n",
       "2022-02-05 16:11:00                                        106213376.0                          \n",
       "2022-02-05 16:10:00                                        106180608.0                          \n",
       "2022-02-05 16:09:00                                        106180608.0                          \n",
       "2022-02-05 16:08:00                                        106180608.0                          \n",
       "2022-02-05 16:07:00                                        106180608.0                          \n",
       "2022-02-05 16:06:00                                        106180608.0                          \n",
       "2022-02-05 16:05:00                                        106180608.0                          \n",
       "2022-02-05 16:04:00                                        106180608.0                          \n",
       "2022-02-05 16:03:00                                        106180608.0                          \n",
       "2022-02-05 16:02:00                                        106172416.0                          \n",
       "2022-02-05 16:01:00                                        106135552.0                          \n",
       "\n",
       "                     mem_cached_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                     \n",
       "2022-02-05 16:33:00                                       1.928245e+09                        \n",
       "2022-02-05 16:32:00                                       1.927901e+09                        \n",
       "2022-02-05 16:31:00                                       1.927889e+09                        \n",
       "2022-02-05 16:30:00                                       1.927983e+09                        \n",
       "2022-02-05 16:29:00                                       1.927983e+09                        \n",
       "2022-02-05 16:28:00                                       1.927983e+09                        \n",
       "2022-02-05 16:27:00                                       1.927983e+09                        \n",
       "2022-02-05 16:26:00                                       1.927987e+09                        \n",
       "2022-02-05 16:25:00                                       1.927987e+09                        \n",
       "2022-02-05 16:24:00                                       1.927987e+09                        \n",
       "2022-02-05 16:23:00                                       1.927844e+09                        \n",
       "2022-02-05 16:22:00                                       1.927844e+09                        \n",
       "2022-02-05 16:21:00                                       1.927836e+09                        \n",
       "2022-02-05 16:20:00                                       1.927836e+09                        \n",
       "2022-02-05 16:19:00                                       1.927836e+09                        \n",
       "2022-02-05 16:18:00                                       1.927836e+09                        \n",
       "2022-02-05 16:17:00                                       1.927582e+09                        \n",
       "2022-02-05 16:16:00                                       1.927586e+09                        \n",
       "2022-02-05 16:15:00                                       1.927582e+09                        \n",
       "2022-02-05 16:14:00                                       1.927598e+09                        \n",
       "2022-02-05 16:13:00                                       1.927545e+09                        \n",
       "2022-02-05 16:12:00                                       1.927545e+09                        \n",
       "2022-02-05 16:11:00                                       1.927537e+09                        \n",
       "2022-02-05 16:10:00                                       1.927537e+09                        \n",
       "2022-02-05 16:09:00                                       1.927537e+09                        \n",
       "2022-02-05 16:08:00                                       1.927537e+09                        \n",
       "2022-02-05 16:07:00                                       1.927537e+09                        \n",
       "2022-02-05 16:06:00                                       1.927541e+09                        \n",
       "2022-02-05 16:05:00                                       1.927541e+09                        \n",
       "2022-02-05 16:04:00                                       1.927541e+09                        \n",
       "2022-02-05 16:03:00                                       1.927541e+09                        \n",
       "2022-02-05 16:02:00                                       1.927541e+09                        \n",
       "2022-02-05 16:01:00                                       1.926136e+09                        \n",
       "\n",
       "                     mem_free_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                   \n",
       "2022-02-05 16:33:00                                       1.503527e+09                      \n",
       "2022-02-05 16:32:00                                       1.465037e+09                      \n",
       "2022-02-05 16:31:00                                       1.467040e+09                      \n",
       "2022-02-05 16:30:00                                       1.387250e+09                      \n",
       "2022-02-05 16:29:00                                       1.387377e+09                      \n",
       "2022-02-05 16:28:00                                       1.387213e+09                      \n",
       "2022-02-05 16:27:00                                       1.387532e+09                      \n",
       "2022-02-05 16:26:00                                       1.385083e+09                      \n",
       "2022-02-05 16:25:00                                       1.385083e+09                      \n",
       "2022-02-05 16:24:00                                       1.386648e+09                      \n",
       "2022-02-05 16:23:00                                       1.464975e+09                      \n",
       "2022-02-05 16:22:00                                       1.465135e+09                      \n",
       "2022-02-05 16:21:00                                       1.467163e+09                      \n",
       "2022-02-05 16:20:00                                       1.467130e+09                      \n",
       "2022-02-05 16:19:00                                       1.467290e+09                      \n",
       "2022-02-05 16:18:00                                       1.467257e+09                      \n",
       "2022-02-05 16:17:00                                       1.457136e+09                      \n",
       "2022-02-05 16:16:00                                       1.454969e+09                      \n",
       "2022-02-05 16:15:00                                       1.454744e+09                      \n",
       "2022-02-05 16:14:00                                       1.445835e+09                      \n",
       "2022-02-05 16:13:00                                       1.445835e+09                      \n",
       "2022-02-05 16:12:00                                       1.446081e+09                      \n",
       "2022-02-05 16:11:00                                       1.448403e+09                      \n",
       "2022-02-05 16:10:00                                       1.380766e+09                      \n",
       "2022-02-05 16:09:00                                       1.380897e+09                      \n",
       "2022-02-05 16:08:00                                       1.381126e+09                      \n",
       "2022-02-05 16:07:00                                       1.380893e+09                      \n",
       "2022-02-05 16:06:00                                       1.378222e+09                      \n",
       "2022-02-05 16:05:00                                       1.377993e+09                      \n",
       "2022-02-05 16:04:00                                       1.377534e+09                      \n",
       "2022-02-05 16:03:00                                       1.377980e+09                      \n",
       "2022-02-05 16:02:00                                       1.378173e+09                      \n",
       "2022-02-05 16:01:00                                       1.388532e+09                      \n",
       "\n",
       "                     mem_slab_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                   \n",
       "2022-02-05 16:33:00                                        188825600.0                      \n",
       "2022-02-05 16:32:00                                        190091264.0                      \n",
       "2022-02-05 16:31:00                                        190111744.0                      \n",
       "2022-02-05 16:30:00                                        190488576.0                      \n",
       "2022-02-05 16:29:00                                        190513152.0                      \n",
       "2022-02-05 16:28:00                                        190525440.0                      \n",
       "2022-02-05 16:27:00                                        190500864.0                      \n",
       "2022-02-05 16:26:00                                        190570496.0                      \n",
       "2022-02-05 16:25:00                                        190578688.0                      \n",
       "2022-02-05 16:24:00                                        190484480.0                      \n",
       "2022-02-05 16:23:00                                        190128128.0                      \n",
       "2022-02-05 16:22:00                                        190128128.0                      \n",
       "2022-02-05 16:21:00                                        190156800.0                      \n",
       "2022-02-05 16:20:00                                        190156800.0                      \n",
       "2022-02-05 16:19:00                                        190156800.0                      \n",
       "2022-02-05 16:18:00                                        190156800.0                      \n",
       "2022-02-05 16:17:00                                        190562304.0                      \n",
       "2022-02-05 16:16:00                                        190738432.0                      \n",
       "2022-02-05 16:15:00                                        190787584.0                      \n",
       "2022-02-05 16:14:00                                        191070208.0                      \n",
       "2022-02-05 16:13:00                                        191115264.0                      \n",
       "2022-02-05 16:12:00                                        191004672.0                      \n",
       "2022-02-05 16:11:00                                        190976000.0                      \n",
       "2022-02-05 16:10:00                                        191086592.0                      \n",
       "2022-02-05 16:09:00                                        191098880.0                      \n",
       "2022-02-05 16:08:00                                        191119360.0                      \n",
       "2022-02-05 16:07:00                                        191098880.0                      \n",
       "2022-02-05 16:06:00                                        191279104.0                      \n",
       "2022-02-05 16:05:00                                        191311872.0                      \n",
       "2022-02-05 16:04:00                                        191307776.0                      \n",
       "2022-02-05 16:03:00                                        191193088.0                      \n",
       "2022-02-05 16:02:00                                        191127552.0                      \n",
       "2022-02-05 16:01:00                                        190664704.0                      \n",
       "\n",
       "                     mem_used_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                   \n",
       "2022-02-05 16:33:00                                        123408384.0                      \n",
       "2022-02-05 16:32:00                                        160976896.0                      \n",
       "2022-02-05 16:31:00                                        158969856.0                      \n",
       "2022-02-05 16:30:00                                        238297088.0                      \n",
       "2022-02-05 16:29:00                                        238145536.0                      \n",
       "2022-02-05 16:28:00                                        238309376.0                      \n",
       "2022-02-05 16:27:00                                        238014464.0                      \n",
       "2022-02-05 16:26:00                                        240394240.0                      \n",
       "2022-02-05 16:25:00                                        240386048.0                      \n",
       "2022-02-05 16:24:00                                        238919680.0                      \n",
       "2022-02-05 16:23:00                                        161099776.0                      \n",
       "2022-02-05 16:22:00                                        160940032.0                      \n",
       "2022-02-05 16:21:00                                        158896128.0                      \n",
       "2022-02-05 16:20:00                                        158928896.0                      \n",
       "2022-02-05 16:19:00                                        158769152.0                      \n",
       "2022-02-05 16:18:00                                        158801920.0                      \n",
       "2022-02-05 16:17:00                                        168775680.0                      \n",
       "2022-02-05 16:16:00                                        170770432.0                      \n",
       "2022-02-05 16:15:00                                        170954752.0                      \n",
       "2022-02-05 16:14:00                                        179568640.0                      \n",
       "2022-02-05 16:13:00                                        179576832.0                      \n",
       "2022-02-05 16:12:00                                        179441664.0                      \n",
       "2022-02-05 16:11:00                                        177172480.0                      \n",
       "2022-02-05 16:10:00                                        244731904.0                      \n",
       "2022-02-05 16:09:00                                        244588544.0                      \n",
       "2022-02-05 16:08:00                                        244338688.0                      \n",
       "2022-02-05 16:07:00                                        244592640.0                      \n",
       "2022-02-05 16:06:00                                        247078912.0                      \n",
       "2022-02-05 16:05:00                                        247275520.0                      \n",
       "2022-02-05 16:04:00                                        247738368.0                      \n",
       "2022-02-05 16:03:00                                        247406592.0                      \n",
       "2022-02-05 16:02:00                                        247287808.0                      \n",
       "2022-02-05 16:01:00                                        238833664.0                      \n",
       "\n",
       "                     mem_perc_buffered_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                            \n",
       "2022-02-05 16:33:00                                           2.760701                               \n",
       "2022-02-05 16:32:00                                           2.760701                               \n",
       "2022-02-05 16:31:00                                           2.760594                               \n",
       "2022-02-05 16:30:00                                           2.760381                               \n",
       "2022-02-05 16:29:00                                           2.760381                               \n",
       "2022-02-05 16:28:00                                           2.760062                               \n",
       "2022-02-05 16:27:00                                           2.760062                               \n",
       "2022-02-05 16:26:00                                           2.759956                               \n",
       "2022-02-05 16:25:00                                           2.759956                               \n",
       "2022-02-05 16:24:00                                           2.759850                               \n",
       "2022-02-05 16:23:00                                           2.759637                               \n",
       "2022-02-05 16:22:00                                           2.759637                               \n",
       "2022-02-05 16:21:00                                           2.759530                               \n",
       "2022-02-05 16:20:00                                           2.759530                               \n",
       "2022-02-05 16:19:00                                           2.759530                               \n",
       "2022-02-05 16:18:00                                           2.759530                               \n",
       "2022-02-05 16:17:00                                           2.759424                               \n",
       "2022-02-05 16:16:00                                           2.759211                               \n",
       "2022-02-05 16:15:00                                           2.759105                               \n",
       "2022-02-05 16:14:00                                           2.758999                               \n",
       "2022-02-05 16:13:00                                           2.758999                               \n",
       "2022-02-05 16:12:00                                           2.758999                               \n",
       "2022-02-05 16:11:00                                           2.758573                               \n",
       "2022-02-05 16:10:00                                           2.757722                               \n",
       "2022-02-05 16:09:00                                           2.757722                               \n",
       "2022-02-05 16:08:00                                           2.757722                               \n",
       "2022-02-05 16:07:00                                           2.757722                               \n",
       "2022-02-05 16:06:00                                           2.757722                               \n",
       "2022-02-05 16:05:00                                           2.757722                               \n",
       "2022-02-05 16:04:00                                           2.757722                               \n",
       "2022-02-05 16:03:00                                           2.757722                               \n",
       "2022-02-05 16:02:00                                           2.757509                               \n",
       "2022-02-05 16:01:00                                           2.756552                               \n",
       "\n",
       "                     mem_perc_cached_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                          \n",
       "2022-02-05 16:33:00                                          50.080371                             \n",
       "2022-02-05 16:32:00                                          50.071435                             \n",
       "2022-02-05 16:31:00                                          50.071116                             \n",
       "2022-02-05 16:30:00                                          50.073563                             \n",
       "2022-02-05 16:29:00                                          50.073563                             \n",
       "2022-02-05 16:28:00                                          50.073563                             \n",
       "2022-02-05 16:27:00                                          50.073563                             \n",
       "2022-02-05 16:26:00                                          50.073669                             \n",
       "2022-02-05 16:25:00                                          50.073669                             \n",
       "2022-02-05 16:24:00                                          50.073669                             \n",
       "2022-02-05 16:23:00                                          50.069946                             \n",
       "2022-02-05 16:22:00                                          50.069946                             \n",
       "2022-02-05 16:21:00                                          50.069733                             \n",
       "2022-02-05 16:20:00                                          50.069733                             \n",
       "2022-02-05 16:19:00                                          50.069733                             \n",
       "2022-02-05 16:18:00                                          50.069733                             \n",
       "2022-02-05 16:17:00                                          50.063137                             \n",
       "2022-02-05 16:16:00                                          50.063244                             \n",
       "2022-02-05 16:15:00                                          50.063137                             \n",
       "2022-02-05 16:14:00                                          50.063563                             \n",
       "2022-02-05 16:13:00                                          50.062180                             \n",
       "2022-02-05 16:12:00                                          50.062180                             \n",
       "2022-02-05 16:11:00                                          50.061967                             \n",
       "2022-02-05 16:10:00                                          50.061967                             \n",
       "2022-02-05 16:09:00                                          50.061967                             \n",
       "2022-02-05 16:08:00                                          50.061967                             \n",
       "2022-02-05 16:07:00                                          50.061967                             \n",
       "2022-02-05 16:06:00                                          50.062073                             \n",
       "2022-02-05 16:05:00                                          50.062073                             \n",
       "2022-02-05 16:04:00                                          50.062073                             \n",
       "2022-02-05 16:03:00                                          50.062073                             \n",
       "2022-02-05 16:02:00                                          50.062073                             \n",
       "2022-02-05 16:01:00                                          50.025585                             \n",
       "\n",
       "                     mem_perc_free_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                        \n",
       "2022-02-05 16:33:00                                          39.049590                           \n",
       "2022-02-05 16:32:00                                          38.049925                           \n",
       "2022-02-05 16:31:00                                          38.101945                           \n",
       "2022-02-05 16:30:00                                          36.029638                           \n",
       "2022-02-05 16:29:00                                          36.032936                           \n",
       "2022-02-05 16:28:00                                          36.028680                           \n",
       "2022-02-05 16:27:00                                          36.036978                           \n",
       "2022-02-05 16:26:00                                          35.973362                           \n",
       "2022-02-05 16:25:00                                          35.973362                           \n",
       "2022-02-05 16:24:00                                          36.014000                           \n",
       "2022-02-05 16:23:00                                          38.048329                           \n",
       "2022-02-05 16:22:00                                          38.052478                           \n",
       "2022-02-05 16:21:00                                          38.105137                           \n",
       "2022-02-05 16:20:00                                          38.104286                           \n",
       "2022-02-05 16:19:00                                          38.108434                           \n",
       "2022-02-05 16:18:00                                          38.107583                           \n",
       "2022-02-05 16:17:00                                          37.844715                           \n",
       "2022-02-05 16:16:00                                          37.788440                           \n",
       "2022-02-05 16:15:00                                          37.782589                           \n",
       "2022-02-05 16:14:00                                          37.551209                           \n",
       "2022-02-05 16:13:00                                          37.551209                           \n",
       "2022-02-05 16:12:00                                          37.557592                           \n",
       "2022-02-05 16:11:00                                          37.617910                           \n",
       "2022-02-05 16:10:00                                          35.861236                           \n",
       "2022-02-05 16:09:00                                          35.864640                           \n",
       "2022-02-05 16:08:00                                          35.870598                           \n",
       "2022-02-05 16:07:00                                          35.864534                           \n",
       "2022-02-05 16:06:00                                          35.795173                           \n",
       "2022-02-05 16:05:00                                          35.789216                           \n",
       "2022-02-05 16:04:00                                          35.777301                           \n",
       "2022-02-05 16:03:00                                          35.788897                           \n",
       "2022-02-05 16:02:00                                          35.793897                           \n",
       "2022-02-05 16:01:00                                          36.062935                           \n",
       "\n",
       "                     mem_perc_slab_experiment-advanced-mongo-with-latency-12-test-orchestrator  \\\n",
       "timestamp                                                                                        \n",
       "2022-02-05 16:33:00                                           4.904177                           \n",
       "2022-02-05 16:32:00                                           4.937049                           \n",
       "2022-02-05 16:31:00                                           4.937581                           \n",
       "2022-02-05 16:30:00                                           4.947368                           \n",
       "2022-02-05 16:29:00                                           4.948006                           \n",
       "2022-02-05 16:28:00                                           4.948325                           \n",
       "2022-02-05 16:27:00                                           4.947687                           \n",
       "2022-02-05 16:26:00                                           4.949495                           \n",
       "2022-02-05 16:25:00                                           4.949708                           \n",
       "2022-02-05 16:24:00                                           4.947261                           \n",
       "2022-02-05 16:23:00                                           4.938006                           \n",
       "2022-02-05 16:22:00                                           4.938006                           \n",
       "2022-02-05 16:21:00                                           4.938751                           \n",
       "2022-02-05 16:20:00                                           4.938751                           \n",
       "2022-02-05 16:19:00                                           4.938751                           \n",
       "2022-02-05 16:18:00                                           4.938751                           \n",
       "2022-02-05 16:17:00                                           4.949283                           \n",
       "2022-02-05 16:16:00                                           4.953857                           \n",
       "2022-02-05 16:15:00                                           4.955134                           \n",
       "2022-02-05 16:14:00                                           4.962474                           \n",
       "2022-02-05 16:13:00                                           4.963644                           \n",
       "2022-02-05 16:12:00                                           4.960772                           \n",
       "2022-02-05 16:11:00                                           4.960027                           \n",
       "2022-02-05 16:10:00                                           4.962900                           \n",
       "2022-02-05 16:09:00                                           4.963219                           \n",
       "2022-02-05 16:08:00                                           4.963751                           \n",
       "2022-02-05 16:07:00                                           4.963219                           \n",
       "2022-02-05 16:06:00                                           4.967899                           \n",
       "2022-02-05 16:05:00                                           4.968750                           \n",
       "2022-02-05 16:04:00                                           4.968644                           \n",
       "2022-02-05 16:03:00                                           4.965665                           \n",
       "2022-02-05 16:02:00                                           4.963963                           \n",
       "2022-02-05 16:01:00                                           4.951942                           \n",
       "\n",
       "                     mem_perc_used_experiment-advanced-mongo-with-latency-12-test-orchestrator  \n",
       "timestamp                                                                                       \n",
       "2022-02-05 16:33:00                                           3.205162                          \n",
       "2022-02-05 16:32:00                                           4.180891                          \n",
       "2022-02-05 16:31:00                                           4.128764                          \n",
       "2022-02-05 16:30:00                                           6.189050                          \n",
       "2022-02-05 16:29:00                                           6.185114                          \n",
       "2022-02-05 16:28:00                                           6.189369                          \n",
       "2022-02-05 16:27:00                                           6.181710                          \n",
       "2022-02-05 16:26:00                                           6.243517                          \n",
       "2022-02-05 16:25:00                                           6.243305                          \n",
       "2022-02-05 16:24:00                                           6.205220                          \n",
       "2022-02-05 16:23:00                                           4.184082                          \n",
       "2022-02-05 16:22:00                                           4.179933                          \n",
       "2022-02-05 16:21:00                                           4.126849                          \n",
       "2022-02-05 16:20:00                                           4.127700                          \n",
       "2022-02-05 16:19:00                                           4.123551                          \n",
       "2022-02-05 16:18:00                                           4.124402                          \n",
       "2022-02-05 16:17:00                                           4.383441                          \n",
       "2022-02-05 16:16:00                                           4.435248                          \n",
       "2022-02-05 16:15:00                                           4.440036                          \n",
       "2022-02-05 16:14:00                                           4.663755                          \n",
       "2022-02-05 16:13:00                                           4.663968                          \n",
       "2022-02-05 16:12:00                                           4.660458                          \n",
       "2022-02-05 16:11:00                                           4.601522                          \n",
       "2022-02-05 16:10:00                                           6.356175                          \n",
       "2022-02-05 16:09:00                                           6.352452                          \n",
       "2022-02-05 16:08:00                                           6.345963                          \n",
       "2022-02-05 16:07:00                                           6.352558                          \n",
       "2022-02-05 16:06:00                                           6.417132                          \n",
       "2022-02-05 16:05:00                                           6.422238                          \n",
       "2022-02-05 16:04:00                                           6.434259                          \n",
       "2022-02-05 16:03:00                                           6.425642                          \n",
       "2022-02-05 16:02:00                                           6.422557                          \n",
       "2022-02-05 16:01:00                                           6.202986                          "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Prepare Data so that as the monitoring time_series_list is not aligned\n",
    "time_series_list_list_labels = [\"cpu_util_\", \"io_read_\", \"io_write_\", \"iops_read_\", \"iops_write_\"]\n",
    "time_series_list_list = [results_cpu, results_io_read, results_io_write, results_iops_read, results_iops_write]\n",
    "# Add Memory\n",
    "for state in mem_states:\n",
    "    time_series_list_list.append(mutli_results_memory[state])\n",
    "    time_series_list_list_labels.append(f\"mem_{state}_\")\n",
    "\n",
    "for state in mem_states:\n",
    "    time_series_list_list.append(mutli_results_memory_percentage[state])\n",
    "    time_series_list_list_labels.append(f\"mem_perc_{state}_\")\n",
    "\n",
    "print(time_series_list_list_labels)\n",
    "\n",
    "instance_logs = {}\n",
    "\n",
    "for index, time_series_list in enumerate(time_series_list_list):\n",
    "    for time_series in time_series_list:\n",
    "        if time_series.metric.labels[\"instance_name\"]:\n",
    "            label = time_series.metric.labels[\"instance_name\"]\n",
    "        else:\n",
    "            # Find out label by comapring the instance ID\n",
    "            for compare_item in time_series_list_list[0]:\n",
    "                if compare_item.resource.labels[\"instance_id\"] == time_series.resource.labels[\"instance_id\"]:\n",
    "                    label = compare_item.metric.labels[\"instance_name\"]\n",
    "        if label not in instance_logs:\n",
    "            instance_logs[label] = np.empty(len(time_series_list_list), dtype=object)\n",
    "        instance_logs[label][index] = time_series\n",
    "\n",
    "def loadMonitoringData(experiment, outDir = datetime.datetime.today().strftime('%Y%m%d-%H')): \n",
    "    ensureDirectory(outDir)\n",
    "    df = pd.DataFrame()\n",
    "    set_timestamp_column = True\n",
    "    first_len = 0\n",
    "    # Filter instances for our experiment\n",
    "    filteredInstances = {}\n",
    "    for key, instance in instance_logs.items():\n",
    "        if key.startswith(experiment):\n",
    "            filteredInstances[key] = instance\n",
    "\n",
    "    # Find out which point array is smallest\n",
    "    first_len = min([len(item.points) for instances in filteredInstances.values() for item in instances])\n",
    "    for instances in filteredInstances.values():\n",
    "        print(instances[0].metric.labels[\"instance_name\"], len(instances[0].points))\n",
    "\n",
    "\n",
    "\n",
    "    print(\"Overall min len:\", first_len)\n",
    "\n",
    "    for key, instances in filteredInstances.items():\n",
    "        for index in range(len(instances)):\n",
    "            label = key\n",
    "            # if \"orchestrator\" in label:\n",
    "            #     # Skip Orchesrtator\n",
    "            #     continue\n",
    "            if set_timestamp_column:\n",
    "                # first_len = min([len(item.points) - 1 for item in instances])\n",
    "                # print(first_len)\n",
    "                print(\"SET min len:\", first_len)\n",
    "\n",
    "                df['timestamp'] = pd.to_datetime([p.interval.start_time.ToDatetime() for p in instances[0].points[:first_len]])\n",
    "                set_timestamp_column = False\n",
    "            # print(ts.points[0])\n",
    "            # When deploying the vm they might take different amount of time leading to some values beeing available a minute early this leading to different length\n",
    "            # We can trim off the last values as they are orderer from most recent to last\n",
    "            df[time_series_list_list_labels[index] + label] = [p.value.double_value for p in instances[index].points[:first_len]]\n",
    "\n",
    "            # df['cpu_util_' + label] = [p.value.double_value for p in ts_cpu.points[:first_len]]\n",
    "            # df['io_read_' + label] = [p.value.int64_value for p in ts_io_read.points[:first_len]]\n",
    "            # df['io_write_' + label] = [p.value.int64_value for p in ts_io_write.points[:first_len]]\n",
    "            # df['iops_read_' + label] = [p.value.int64_value for p in ts_iops_read.points[:first_len]]\n",
    "            # df['iops_write_' + label] = [p.value.int64_value for p in ts_iops_write.points[:first_len]]\n",
    "\n",
    "    df.set_index('timestamp', inplace=True)\n",
    "    df.index = pd.to_datetime(df.index)\n",
    "    columns = df.columns\n",
    "    df.to_csv(f\"{outDir}/monitoring.csv\")\n",
    "    print(f\"Gathered all Monitoring data for {experiment}\")\n",
    "    return df\n",
    "\n",
    "\n",
    "# Make sure your experiment name is included in the experiments list, otherwise only errors will occur.\n",
    "if 'experiment' not in locals():\n",
    "    for experiment in sharedVariables.experiments: \n",
    "        try: \n",
    "            loadMonitoringData(experiment)\n",
    "        except Exception as e: \n",
    "            print(e)\n",
    "# loadMonitoringData(experiment)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get TimeStamps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Workaround for now\n",
    "# # df_sync3 = pd.read_csv(f'{outdir}/experiment-syncmesh-with-latency-3.csv')\n",
    "# # df_base3 = pd.read_csv(f'{outdir}/experiment-baseline-with-latency-3.csv')\n",
    "# df_sync3 = loadData(f'{outdir}/experiment-syncmesh-with-latency-3.log')\n",
    "# df_base3 = loadData(f'{outdir}/experiment-baseline-with-latency-3.log')\n",
    "# # df_base3.set_index('timestamp', inplace=True)\n",
    "# # df_sync3.set_index('timestamp', inplace=True)\n",
    "\n",
    "# # df_sync6 = loadData(f'{outdir}/experiment-syncmesh-with-latency-6.log')\n",
    "# # df_base6 = loadData(f'{outdir}/experiment-baseline-with-latency-6.log')\n",
    "\n",
    "# def filterDataForSeperator(df): \n",
    "#     df = df[df[\"jsonPayload.connection.src_ip\"].isin([ip_seperator, ip_orchestrator])]\n",
    "#     df = df[df[\"jsonPayload.connection.dest_ip\"].isin([ip_seperator, ip_orchestrator])]\n",
    "#     df = df[df[\"jsonPayload.connection.dest_port\"] == 443]\n",
    "#     return df\n",
    "\n",
    "# seperator_base3 = filterDataForSeperator(df_base3)\n",
    "# seperator_sync3 = filterDataForSeperator(df_sync3)\n",
    "\n",
    "# # df = df[df[\"jsonPayload.connection.dest_ip\"]]\n",
    "# print(seperator_sync3.index)\n",
    "# print(seperator_base3.index)\n",
    "# # df_base3.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dowloading File\n",
      "c:\\Develop\\GitHub\\DSPJ2021\\syncmesh\\evaluation\\..\\cert\n",
      "\n",
      "experiment-advanced-mongo-with-latency-12-test-orchestrator 34\n",
      "Overall min len: 33\n",
      "SET min len: 33\n",
      "Gathered all Monitoring data for experiment-advanced-mongo-with-latency-12-test-orchestrator\n"
     ]
    }
   ],
   "source": [
    "# Script for automatic data retrieval via papermill\n",
    "if 'experiment' in locals():\n",
    "    todaystring = datetime.datetime.today().strftime('%Y%m%d-%H')\n",
    "    outdir = f\"data/{todaystring}-{experiment}\"\n",
    "\n",
    "    f = open(os.path.join(Path.cwd(), \"..\", \"infrastructure\", \"orchestrator.txt\"), \"r\")\n",
    "    ip = f.read()\n",
    "    download_pcap(ip, outDir=outdir)\n",
    "    loadMonitoringData(experiment, outDir=outdir)\n",
    "    #getDataset(experiment, outDir=outdir)\n",
    "\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}