Omics-20201123.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Omics Data QA/QC Analysis: \n",
    "* **Authors: Nurgul Kaplan, Christopher Petzold, Yan Chen, Jennifer Gin**\n",
    "* **Authors_EDD Utils:Zak Costello, William Morrell, Mark Forrer, Tijana Radivojevic**\n",
    "* **Author_Kernel-Jupyter server: Mark Kulawik**\n",
    "* **Version: 1.07**\n",
    "* **Date:20201123**\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load necessary python libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import re\n",
    "from datetime import datetime\n",
    "import seaborn as sns; sns.set(color_codes=True)\n",
    "import ipywidgets as widgets\n",
    "from IPython.display import display\n",
    "from ipywidgets import HBox, Label, Layout ,Button,AppLayout, jslink, IntText, IntSlider\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Import EDD utils to import study as pandas dataframe\n",
    "from edd_utils import login, export_study\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Create a function which is for entering the URL of EDD study\n",
    "def edd_study_url():\n",
    "     \n",
    "#     colors:\n",
    "    class bcolors:\n",
    "        HEADER = '\\033[95m'\n",
    "        FAIL = '\\033[91m'\n",
    "        BOLD = '\\033[1m'\n",
    "        \n",
    "    while True:\n",
    "        try:\n",
    "            user_edd_study_url = input(\"Please enter EDD STUDY URL: \").lower()\n",
    "            user_edd_study_url = user_edd_study_url.replace(\" \",\"\")\n",
    "            if \"/s/\" not in user_edd_study_url:\n",
    "                raise ValueError()\n",
    "            else:\n",
    "                break\n",
    "        except ValueError:\n",
    "            print(f\"{bcolors.HEADER}{bcolors.BOLD}PLEASE TRY AGAIN\")\n",
    "            \n",
    "            \n",
    "    url_parts = user_edd_study_url.split(\"/\")    \n",
    "    s_index = url_parts.index(\"s\")\n",
    "    edd_server = url_parts[s_index-1]\n",
    "    study_slug = url_parts[s_index+1]\n",
    "    final_url_parts = [edd_server, study_slug]\n",
    "\n",
    "    return final_url_parts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***\n",
    " **Example of EDD study URL link:**\n",
    "\n",
    " https://<font color=blue>edd_server</font>/s/<font color=red>This-is-my-Study-Slug</font>/\n",
    "\n",
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font color=blue>**EDD** study url:</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " https://<font color=blue>public-edd.agilebiofoundry.org</font>/s/<font color=red>example-data-quality-study</font>/\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Enter <font color=green>**EDD study URL link**</font>, Please see the example shown above\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Please enter EDD STUDY URL:  https://public-edd.agilebiofoundry.org/s/example-data-quality-study/\n"
     ]
    }
   ],
   "source": [
    "#User input requested/\n",
    "final_url_parts = edd_study_url()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Parse EDD STUDY URL for edd_utils\n",
    "edd_server = final_url_parts[0]\n",
    "study_slug = final_url_parts[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Enter <font color=green>**LDAP password**</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " * session = login(edd_server=edd_server)\n",
    " * Password for <font color=red>YourLDAPusername</font>:     <font color=green>**LDAP password**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Password for nkaplan:  ················\n"
     ]
    }
   ],
   "source": [
    "# Create EDD session\n",
    "session = login(edd_server=edd_server)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0758a3bbbc64297a63f079e4a74c624",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=54.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "df = export_study(session, study_slug, edd_server=edd_server)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Study ID</th>\n",
       "      <th>Study Name</th>\n",
       "      <th>Line ID</th>\n",
       "      <th>Line Name</th>\n",
       "      <th>Line Description</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay ID</th>\n",
       "      <th>Assay Name</th>\n",
       "      <th>Formal Type</th>\n",
       "      <th>Measurement Type</th>\n",
       "      <th>Compartment</th>\n",
       "      <th>Units</th>\n",
       "      <th>Value</th>\n",
       "      <th>Hours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7342</td>\n",
       "      <td>Example Data Quality Study</td>\n",
       "      <td>7343</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>This is Line-1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>7352</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>0</td>\n",
       "      <td>counts</td>\n",
       "      <td>7209.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7342</td>\n",
       "      <td>Example Data Quality Study</td>\n",
       "      <td>7343</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>This is Line-1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>7352</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>0</td>\n",
       "      <td>counts</td>\n",
       "      <td>2506.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7342</td>\n",
       "      <td>Example Data Quality Study</td>\n",
       "      <td>7343</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>This is Line-1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>7352</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00698|LYSC_CHICK Lysozyme C</td>\n",
       "      <td>Lysozyme C</td>\n",
       "      <td>0</td>\n",
       "      <td>counts</td>\n",
       "      <td>3154.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7342</td>\n",
       "      <td>Example Data Quality Study</td>\n",
       "      <td>7344</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>This is Line-1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>7353</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>0</td>\n",
       "      <td>counts</td>\n",
       "      <td>12513.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7342</td>\n",
       "      <td>Example Data Quality Study</td>\n",
       "      <td>7344</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>This is Line-1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>7353</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>0</td>\n",
       "      <td>counts</td>\n",
       "      <td>3244.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Study ID                  Study Name  Line ID  Line Name Line Description  \\\n",
       "0      7342  Example Data Quality Study     7343  Line-1-R1   This is Line-1   \n",
       "1      7342  Example Data Quality Study     7343  Line-1-R1   This is Line-1   \n",
       "2      7342  Example Data Quality Study     7343  Line-1-R1   This is Line-1   \n",
       "3      7342  Example Data Quality Study     7344  Line-1-R2   This is Line-1   \n",
       "4      7342  Example Data Quality Study     7344  Line-1-R2   This is Line-1   \n",
       "\n",
       "              Protocol  Assay ID Assay Name  \\\n",
       "0  Targeted Proteomics      7352  Line-1-R1   \n",
       "1  Targeted Proteomics      7352  Line-1-R1   \n",
       "2  Targeted Proteomics      7352  Line-1-R1   \n",
       "3  Targeted Proteomics      7353  Line-1-R2   \n",
       "4  Targeted Proteomics      7353  Line-1-R2   \n",
       "\n",
       "                          Formal Type Measurement Type  Compartment   Units  \\\n",
       "0          sp|P00761|TRYP_PIG Trypsin          Trypsin            0  counts   \n",
       "1  sp|P02769|ALBU_BOVIN Serum albumin    Serum albumin            0  counts   \n",
       "2     sp|P00698|LYSC_CHICK Lysozyme C       Lysozyme C            0  counts   \n",
       "3          sp|P00761|TRYP_PIG Trypsin          Trypsin            0  counts   \n",
       "4  sp|P02769|ALBU_BOVIN Serum albumin    Serum albumin            0  counts   \n",
       "\n",
       "     Value  Hours  \n",
       "0   7209.0   24.0  \n",
       "1   2506.0   24.0  \n",
       "2   3154.0   24.0  \n",
       "3  12513.0   24.0  \n",
       "4   3244.0   24.0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract study name from EDD data and replace special characters/spaces with \"_\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Example_Data_Quality_Study'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "study_name = df[\"Study Name\"][0]\n",
    "study_name = re.sub('\\W+','_', study_name )\n",
    "study_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create function for cleaning raw data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Including 'Protocol' and 'Units'\n",
    "excluded_columns = ['Study ID', 'Study Name', 'Study Description', 'Study Contact',\n",
    "       'Line ID', 'Line Name', 'Line Description', 'Control', 'Strain(s)',\n",
    "       'Carbon Source(s)', 'Line Experimenter', 'Line Contact', 'Media',\n",
    "       'Protocol ID', 'Protocol Name', 'Assay ID', \n",
    "       'Measurement Updated', 'X Units', 'Y Units','Compartment']\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "def parse_edd_study(df):\n",
    "    df = df.loc[:,~df.columns.isin(excluded_columns)]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "df = parse_edd_study(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay Name</th>\n",
       "      <th>Formal Type</th>\n",
       "      <th>Measurement Type</th>\n",
       "      <th>Units</th>\n",
       "      <th>Value</th>\n",
       "      <th>Hours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>7209.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>2506.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00698|LYSC_CHICK Lysozyme C</td>\n",
       "      <td>Lysozyme C</td>\n",
       "      <td>counts</td>\n",
       "      <td>3154.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>12513.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>3244.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay Name                         Formal Type  \\\n",
       "0  Targeted Proteomics  Line-1-R1          sp|P00761|TRYP_PIG Trypsin   \n",
       "1  Targeted Proteomics  Line-1-R1  sp|P02769|ALBU_BOVIN Serum albumin   \n",
       "2  Targeted Proteomics  Line-1-R1     sp|P00698|LYSC_CHICK Lysozyme C   \n",
       "3  Targeted Proteomics  Line-1-R2          sp|P00761|TRYP_PIG Trypsin   \n",
       "4  Targeted Proteomics  Line-1-R2  sp|P02769|ALBU_BOVIN Serum albumin   \n",
       "\n",
       "  Measurement Type   Units    Value  Hours  \n",
       "0          Trypsin  counts   7209.0   24.0  \n",
       "1    Serum albumin  counts   2506.0   24.0  \n",
       "2       Lysozyme C  counts   3154.0   24.0  \n",
       "3          Trypsin  counts  12513.0   24.0  \n",
       "4    Serum albumin  counts   3244.0   24.0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Targeted Proteomics']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dropdown options from unique protocol names\n",
    "ALL = 'ALL'\n",
    "def unique_values(array):\n",
    "    unique = array.unique().tolist()\n",
    "#     unique.sort()\n",
    "#     unique.insert(0, ALL)\n",
    "    return unique\n",
    "unique_values(df[\"Protocol\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Create a class for colorful printing\n",
    "class bcolors:\n",
    "    HEADER = '\\033[95m'\n",
    "    OKBLUE = '\\033[94m'\n",
    "    OKGREEN = '\\033[92m'\n",
    "    WARNING = '\\033[93m'\n",
    "    FAIL = '\\033[91m'\n",
    "    ENDC = '\\033[0m'\n",
    "    BOLD = '\\033[1m'\n",
    "    UNDERLINE = '\\033[4m'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OPTIONAL: SPECIFY PROTOCOL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Create dropdown options for different protocols. This will happen when there are multiple protocols in dataset.\n",
    "if len(unique_values(df[\"Protocol\"])) > 1:\n",
    "    dropdown_protocol = widgets.Dropdown(options = unique_values(df[\"Protocol\"]), description = \"Protocol:\" )\n",
    "    # dropdown_type_abb\n",
    "    print(f\"{bcolors.HEADER}{bcolors.BOLD}{bcolors.UNDERLINE}OPTIONAL:SPECIFY THE PROTOCOL FROM DROPDOWN MENU\")\n",
    "    print(\"If protocol is NOT specified, ONLY the FIRST protocol will be ANALYZED \")\n",
    "\n",
    "    def final_df():\n",
    "        user_change = dropdown_protocol.value\n",
    "        updated_df = df[df[\"Protocol\"] == user_change].reset_index(drop=True)\n",
    "        return updated_df\n",
    "\n",
    "\n",
    "    def df_update(protocol):\n",
    "        updated_df = df[df[\"Protocol\"] == protocol].reset_index(drop=True)\n",
    "        display(updated_df.head())\n",
    "\n",
    "    widgets.interact(df_update, protocol=dropdown_protocol)\n",
    "    print(f\"{bcolors.HEADER}{bcolors.BOLD}{bcolors.UNDERLINE}AFTER PROTOCOL SELECTION, YOU MUST GO TO THE NEXT CELL\")\n",
    "    print(f\"{bcolors.HEADER}{bcolors.BOLD}{bcolors.UNDERLINE}Run --> Run Selected Cell and All Below\")\n",
    "else:\n",
    "    df_protocol = df.copy()\n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run this cell and all below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# After sepecifying protocol, run this cell and all below\n",
    "if len(unique_values(df[\"Protocol\"])) > 1:\n",
    "    df_protocol = final_df()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay Name</th>\n",
       "      <th>Formal Type</th>\n",
       "      <th>Measurement Type</th>\n",
       "      <th>Units</th>\n",
       "      <th>Value</th>\n",
       "      <th>Hours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>7209.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>2506.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00698|LYSC_CHICK Lysozyme C</td>\n",
       "      <td>Lysozyme C</td>\n",
       "      <td>counts</td>\n",
       "      <td>3154.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>12513.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>3244.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay Name                         Formal Type  \\\n",
       "0  Targeted Proteomics  Line-1-R1          sp|P00761|TRYP_PIG Trypsin   \n",
       "1  Targeted Proteomics  Line-1-R1  sp|P02769|ALBU_BOVIN Serum albumin   \n",
       "2  Targeted Proteomics  Line-1-R1     sp|P00698|LYSC_CHICK Lysozyme C   \n",
       "3  Targeted Proteomics  Line-1-R2          sp|P00761|TRYP_PIG Trypsin   \n",
       "4  Targeted Proteomics  Line-1-R2  sp|P02769|ALBU_BOVIN Serum albumin   \n",
       "\n",
       "  Measurement Type   Units    Value  Hours  \n",
       "0          Trypsin  counts   7209.0   24.0  \n",
       "1    Serum albumin  counts   2506.0   24.0  \n",
       "2       Lysozyme C  counts   3154.0   24.0  \n",
       "3          Trypsin  counts  12513.0   24.0  \n",
       "4    Serum albumin  counts   3244.0   24.0  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_protocol.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Checkpoint\n",
    "# df_protocol.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "# Extract only Replicate numbers from Assay Name\n",
    "df0 = df_protocol['Assay Name'].str.extract(r'(?P<Assay_Name>[a-zA-Z0-9\\_\\- ]+)-(?P<Replicate_Num>[Rr]\\d+)',expand=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Replicate_Num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Assay_Name Replicate_Num\n",
       "0     Line-1            R1\n",
       "1     Line-1            R1\n",
       "2     Line-1            R1\n",
       "3     Line-1            R2\n",
       "4     Line-1            R2"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df0.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***If there are no replicate numbers in column Replicete_Num, Assay Name will be kept same with raw data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "if df0[\"Replicate_Num\"].isnull().values.all(axis=0)== False:\n",
    "    df1 = df0.merge(df_protocol, left_index=True,right_index=True)\n",
    "elif df0[\"Replicate_Num\"].isnull().values.all(axis=0)== True:\n",
    "    df1=df_protocol.copy()\n",
    "    df1 = df1.rename(columns={'Assay Name':'Assay_Name'})\n",
    "else:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Replicate_Num</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay Name</th>\n",
       "      <th>Formal Type</th>\n",
       "      <th>Measurement Type</th>\n",
       "      <th>Units</th>\n",
       "      <th>Value</th>\n",
       "      <th>Hours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>7209.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>2506.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00698|LYSC_CHICK Lysozyme C</td>\n",
       "      <td>Lysozyme C</td>\n",
       "      <td>counts</td>\n",
       "      <td>3154.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>12513.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>3244.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Assay_Name Replicate_Num             Protocol Assay Name  \\\n",
       "0     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "1     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "2     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "3     Line-1            R2  Targeted Proteomics  Line-1-R2   \n",
       "4     Line-1            R2  Targeted Proteomics  Line-1-R2   \n",
       "\n",
       "                          Formal Type Measurement Type   Units    Value  Hours  \n",
       "0          sp|P00761|TRYP_PIG Trypsin          Trypsin  counts   7209.0   24.0  \n",
       "1  sp|P02769|ALBU_BOVIN Serum albumin    Serum albumin  counts   2506.0   24.0  \n",
       "2     sp|P00698|LYSC_CHICK Lysozyme C       Lysozyme C  counts   3154.0   24.0  \n",
       "3          sp|P00761|TRYP_PIG Trypsin          Trypsin  counts  12513.0   24.0  \n",
       "4  sp|P02769|ALBU_BOVIN Serum albumin    Serum albumin  counts   3244.0   24.0  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***If Formal Type is empty, do not try to extract anything. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "if df1[\"Formal Type\"].isnull().values.all(axis=0)== True and df1[\"Measurement Type\"].isnull().values.all(axis=0)== True:\n",
    "    df1[\"Formal Type\"] = \"-\"\n",
    "    df1[\"Measurement Type\"] = \"-\"\n",
    "elif df1[\"Formal Type\"].isnull().values.all(axis=0)== True and df1[\"Measurement Type\"].isnull().values.all(axis=0)== False:\n",
    "    df1[\"Formal Type\"] = df1[\"Measurement Type\"]\n",
    "elif df1[\"Formal Type\"].isnull().values.all(axis=0)== False and df1[\"Measurement Type\"].isnull().values.all(axis=0)== True:\n",
    "    df1[\"Measurement Type\"] = df1[\"Formal Type\"]\n",
    "else:\n",
    "    pass\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Replicate_Num</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay Name</th>\n",
       "      <th>Formal Type</th>\n",
       "      <th>Measurement Type</th>\n",
       "      <th>Units</th>\n",
       "      <th>Value</th>\n",
       "      <th>Hours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>7209.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>2506.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00698|LYSC_CHICK Lysozyme C</td>\n",
       "      <td>Lysozyme C</td>\n",
       "      <td>counts</td>\n",
       "      <td>3154.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>12513.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>3244.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Assay_Name Replicate_Num             Protocol Assay Name  \\\n",
       "0     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "1     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "2     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "3     Line-1            R2  Targeted Proteomics  Line-1-R2   \n",
       "4     Line-1            R2  Targeted Proteomics  Line-1-R2   \n",
       "\n",
       "                          Formal Type Measurement Type   Units    Value  Hours  \n",
       "0          sp|P00761|TRYP_PIG Trypsin          Trypsin  counts   7209.0   24.0  \n",
       "1  sp|P02769|ALBU_BOVIN Serum albumin    Serum albumin  counts   2506.0   24.0  \n",
       "2     sp|P00698|LYSC_CHICK Lysozyme C       Lysozyme C  counts   3154.0   24.0  \n",
       "3          sp|P00761|TRYP_PIG Trypsin          Trypsin  counts  12513.0   24.0  \n",
       "4  sp|P02769|ALBU_BOVIN Serum albumin    Serum albumin  counts   3244.0   24.0  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    df2 = df1['Formal Type'].str.extract(r'\\|(?P<Type_ID>[a-zA-Z0-9]+)\\|(?P<Type_Abb>[a-zA-Z0-9\\_]+)?', expand=True)\n",
    "    if df2[\"Type_ID\"].isnull().values.all(axis=0)== True and df2[\"Type_Abb\"].isnull().values.all(axis=0)== True:\n",
    "        if df1['Formal Type'].isnull().values.all(axis=0)== False:\n",
    "            df2[\"Type_ID\"] = df1[\"Formal Type\"]\n",
    "            df2[\"Type_Abb\"] = df1[\"Measurement Type\"]\n",
    "except AttributeError:\n",
    "    df2=df1.copy()    \n",
    "\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Type_ID    Type_Abb\n",
       "0  P00761    TRYP_PIG\n",
       "1  P02769  ALBU_BOVIN\n",
       "2  P00698  LYSC_CHICK\n",
       "3  P00761    TRYP_PIG\n",
       "4  P02769  ALBU_BOVIN"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "df3 = df1.merge(df2, left_index=True,right_index=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Replicate_Num</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay Name</th>\n",
       "      <th>Formal Type</th>\n",
       "      <th>Measurement Type</th>\n",
       "      <th>Units</th>\n",
       "      <th>Value</th>\n",
       "      <th>Hours</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>7209.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>2506.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00698|LYSC_CHICK Lysozyme C</td>\n",
       "      <td>Lysozyme C</td>\n",
       "      <td>counts</td>\n",
       "      <td>3154.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>12513.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>3244.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Assay_Name Replicate_Num             Protocol Assay Name  \\\n",
       "0     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "1     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "2     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "3     Line-1            R2  Targeted Proteomics  Line-1-R2   \n",
       "4     Line-1            R2  Targeted Proteomics  Line-1-R2   \n",
       "\n",
       "                          Formal Type Measurement Type   Units    Value  \\\n",
       "0          sp|P00761|TRYP_PIG Trypsin          Trypsin  counts   7209.0   \n",
       "1  sp|P02769|ALBU_BOVIN Serum albumin    Serum albumin  counts   2506.0   \n",
       "2     sp|P00698|LYSC_CHICK Lysozyme C       Lysozyme C  counts   3154.0   \n",
       "3          sp|P00761|TRYP_PIG Trypsin          Trypsin  counts  12513.0   \n",
       "4  sp|P02769|ALBU_BOVIN Serum albumin    Serum albumin  counts   3244.0   \n",
       "\n",
       "   Hours Type_ID    Type_Abb  \n",
       "0   24.0  P00761    TRYP_PIG  \n",
       "1   24.0  P02769  ALBU_BOVIN  \n",
       "2   24.0  P00698  LYSC_CHICK  \n",
       "3   24.0  P00761    TRYP_PIG  \n",
       "4   24.0  P02769  ALBU_BOVIN  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(54, 11)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** If Type_Abb is NaN, replace NaN value with Measurement type value. \n",
    "** If Type_ID is NaN, replace NaN value with Formal type (This should work for Metabolomics data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "df3.loc[(pd.isnull(df3.Type_Abb)), \"Type_Abb\"] = df3[\"Measurement Type\"]\n",
    "df3.loc[(pd.isnull(df3.Type_ID)), \"Type_ID\"] = df3[\"Formal Type\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Replicate_Num</th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay Name</th>\n",
       "      <th>Formal Type</th>\n",
       "      <th>Measurement Type</th>\n",
       "      <th>Units</th>\n",
       "      <th>Value</th>\n",
       "      <th>Hours</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>7209.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>2506.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R1</td>\n",
       "      <td>sp|P00698|LYSC_CHICK Lysozyme C</td>\n",
       "      <td>Lysozyme C</td>\n",
       "      <td>counts</td>\n",
       "      <td>3154.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P00761|TRYP_PIG Trypsin</td>\n",
       "      <td>Trypsin</td>\n",
       "      <td>counts</td>\n",
       "      <td>12513.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Line-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>Targeted Proteomics</td>\n",
       "      <td>Line-1-R2</td>\n",
       "      <td>sp|P02769|ALBU_BOVIN Serum albumin</td>\n",
       "      <td>Serum albumin</td>\n",
       "      <td>counts</td>\n",
       "      <td>3244.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Assay_Name Replicate_Num             Protocol Assay Name  \\\n",
       "0     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "1     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "2     Line-1            R1  Targeted Proteomics  Line-1-R1   \n",
       "3     Line-1            R2  Targeted Proteomics  Line-1-R2   \n",
       "4     Line-1            R2  Targeted Proteomics  Line-1-R2   \n",
       "\n",
       "                          Formal Type Measurement Type   Units    Value  \\\n",
       "0          sp|P00761|TRYP_PIG Trypsin          Trypsin  counts   7209.0   \n",
       "1  sp|P02769|ALBU_BOVIN Serum albumin    Serum albumin  counts   2506.0   \n",
       "2     sp|P00698|LYSC_CHICK Lysozyme C       Lysozyme C  counts   3154.0   \n",
       "3          sp|P00761|TRYP_PIG Trypsin          Trypsin  counts  12513.0   \n",
       "4  sp|P02769|ALBU_BOVIN Serum albumin    Serum albumin  counts   3244.0   \n",
       "\n",
       "   Hours Type_ID    Type_Abb  \n",
       "0   24.0  P00761    TRYP_PIG  \n",
       "1   24.0  P02769  ALBU_BOVIN  \n",
       "2   24.0  P00698  LYSC_CHICK  \n",
       "3   24.0  P00761    TRYP_PIG  \n",
       "4   24.0  P02769  ALBU_BOVIN  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "if \"Replicate_Num\" in df3.columns:\n",
    "    df4 = df3[[\"Protocol\",\"Assay_Name\", \"Replicate_Num\", \"Type_ID\", \"Type_Abb\", \"Measurement Type\", \"Hours\",\"Units\", \"Value\"]]\n",
    "else:\n",
    "    df4 = df3[[\"Protocol\",\"Assay_Name\", \"Type_ID\", \"Type_Abb\", \"Measurement Type\", \"Hours\",\"Units\", \"Value\"]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "df4 = df4.applymap(lambda s:s.upper() if type(s) == str else s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Replicate_Num</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Measurement Type</th>\n",
       "      <th>Hours</th>\n",
       "      <th>Units</th>\n",
       "      <th>Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>TRYPSIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>7209.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "      <td>SERUM ALBUMIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>2506.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>LYSOZYME C</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>3154.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>TRYPSIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>12513.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "      <td>SERUM ALBUMIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>3244.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay_Name Replicate_Num Type_ID    Type_Abb  \\\n",
       "0  TARGETED PROTEOMICS     LINE-1            R1  P00761    TRYP_PIG   \n",
       "1  TARGETED PROTEOMICS     LINE-1            R1  P02769  ALBU_BOVIN   \n",
       "2  TARGETED PROTEOMICS     LINE-1            R1  P00698  LYSC_CHICK   \n",
       "3  TARGETED PROTEOMICS     LINE-1            R2  P00761    TRYP_PIG   \n",
       "4  TARGETED PROTEOMICS     LINE-1            R2  P02769  ALBU_BOVIN   \n",
       "\n",
       "  Measurement Type  Hours   Units    Value  \n",
       "0          TRYPSIN   24.0  COUNTS   7209.0  \n",
       "1    SERUM ALBUMIN   24.0  COUNTS   2506.0  \n",
       "2       LYSOZYME C   24.0  COUNTS   3154.0  \n",
       "3          TRYPSIN   24.0  COUNTS  12513.0  \n",
       "4    SERUM ALBUMIN   24.0  COUNTS   3244.0  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df4.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# df4[\"Type_ID\"].isnull().values.all(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Replicate_Num</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Measurement Type</th>\n",
       "      <th>Hours</th>\n",
       "      <th>Units</th>\n",
       "      <th>Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>TRYPSIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>7209.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "      <td>SERUM ALBUMIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>2506.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>LYSOZYME C</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>3154.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>TRYPSIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>12513.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "      <td>SERUM ALBUMIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>3244.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay_Name Replicate_Num Type_ID    Type_Abb  \\\n",
       "0  TARGETED PROTEOMICS     LINE-1            R1  P00761    TRYP_PIG   \n",
       "1  TARGETED PROTEOMICS     LINE-1            R1  P02769  ALBU_BOVIN   \n",
       "2  TARGETED PROTEOMICS     LINE-1            R1  P00698  LYSC_CHICK   \n",
       "3  TARGETED PROTEOMICS     LINE-1            R2  P00761    TRYP_PIG   \n",
       "4  TARGETED PROTEOMICS     LINE-1            R2  P02769  ALBU_BOVIN   \n",
       "\n",
       "  Measurement Type  Hours   Units    Value  \n",
       "0          TRYPSIN   24.0  COUNTS   7209.0  \n",
       "1    SERUM ALBUMIN   24.0  COUNTS   2506.0  \n",
       "2       LYSOZYME C   24.0  COUNTS   3154.0  \n",
       "3          TRYPSIN   24.0  COUNTS  12513.0  \n",
       "4    SERUM ALBUMIN   24.0  COUNTS   3244.0  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df4.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# fill all NaN values in Units with \"-\"\n",
    "df4[\"Units\"] = df4[\"Units\"].replace(np.nan, '-', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Replicate_Num</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Measurement Type</th>\n",
       "      <th>Hours</th>\n",
       "      <th>Units</th>\n",
       "      <th>Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>TRYPSIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>7209.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "      <td>SERUM ALBUMIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>2506.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>LYSOZYME C</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>3154.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>TRYPSIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>12513.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "      <td>SERUM ALBUMIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>3244.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay_Name Replicate_Num Type_ID    Type_Abb  \\\n",
       "0  TARGETED PROTEOMICS     LINE-1            R1  P00761    TRYP_PIG   \n",
       "1  TARGETED PROTEOMICS     LINE-1            R1  P02769  ALBU_BOVIN   \n",
       "2  TARGETED PROTEOMICS     LINE-1            R1  P00698  LYSC_CHICK   \n",
       "3  TARGETED PROTEOMICS     LINE-1            R2  P00761    TRYP_PIG   \n",
       "4  TARGETED PROTEOMICS     LINE-1            R2  P02769  ALBU_BOVIN   \n",
       "\n",
       "  Measurement Type  Hours   Units    Value  \n",
       "0          TRYPSIN   24.0  COUNTS   7209.0  \n",
       "1    SERUM ALBUMIN   24.0  COUNTS   2506.0  \n",
       "2       LYSOZYME C   24.0  COUNTS   3154.0  \n",
       "3          TRYPSIN   24.0  COUNTS  12513.0  \n",
       "4    SERUM ALBUMIN   24.0  COUNTS   3244.0  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df4.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['None', 'LINE-1', 'LINE-2', 'LINE-3']\n"
     ]
    }
   ],
   "source": [
    "# Selection options from unique assay names\n",
    "none_exclude = 'None'\n",
    "def unique_values_assay(array):\n",
    "    unique = array.unique().tolist()\n",
    "#     unique.sort()\n",
    "    unique.insert(0, none_exclude)\n",
    "    return unique\n",
    "print(unique_values_assay(df4[\"Assay_Name\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OPTIONAL: EXCLUDE ASSAY OR ASSAYS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[95m\u001b[1m\u001b[4mOPTIONAL: EXCLUDE ASSAY(S) \n",
      "None means ALL the ASSAYS will be ANALYZED \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d716bad2eda148e1ac80f555df25219a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(SelectMultiple(description='Exclude Assay(s): ', index=(0,), layout=Layout(height='150px…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[95m\u001b[1m\u001b[4mAFTER ASSAY(S) SELECTION, YOU MUST GO TO THE NEXT CELL\n",
      "\u001b[95m\u001b[1m\u001b[4mRun --> Run Selected Cell and All Below\n"
     ]
    }
   ],
   "source": [
    "# Create selection options to exclude assay/s....\n",
    "if len(unique_values_assay(df4[\"Assay_Name\"])) > 2:\n",
    "    select_multi_assays = widgets.SelectMultiple(\n",
    "                        options=unique_values_assay(df4[\"Assay_Name\"]),\n",
    "                        value=[none_exclude],\n",
    "                        rows=10,\n",
    "                        description=\"Exclude Assay(s): \",\n",
    "                        style = {\"description_width\":\"initial\"},\n",
    "                        disabled=False,\n",
    "                        layout=Layout(width='30%', height='150px')\n",
    "                    )   \n",
    "    print(f\"{bcolors.HEADER}{bcolors.BOLD}{bcolors.UNDERLINE}OPTIONAL: EXCLUDE ASSAY(S) \")\n",
    "    print(\"None means ALL the ASSAYS will be ANALYZED \")\n",
    "\n",
    "\n",
    "    \n",
    "    def final_df_assay():\n",
    "        exc_assay_list = select_multi_assays.value\n",
    "        if list(exc_assay_list) == [\"None\"]:\n",
    "            updated_df = df4.copy()\n",
    "            \n",
    "        else:\n",
    "            updated_df = df4[~df4[\"Assay_Name\"].isin(exc_assay_list)].reset_index(drop=True)\n",
    "        return updated_df    \n",
    "        \n",
    "\n",
    "    def df_update_assay(exc_assay_list):\n",
    "        if list(exc_assay_list) == [\"None\"]:\n",
    "            updated_df = df4.copy()\n",
    "            display(updated_df.head())\n",
    "        else:\n",
    "            updated_df = df4[~df4[\"Assay_Name\"].isin(exc_assay_list)].reset_index(drop=True)\n",
    "            display(updated_df.head())\n",
    "\n",
    "    widgets.interact(df_update_assay, exc_assay_list=select_multi_assays)\n",
    "    \n",
    "else:\n",
    "    df_protocol_assay = df4.copy()\n",
    "    pass\n",
    "print(f\"{bcolors.HEADER}{bcolors.BOLD}{bcolors.UNDERLINE}AFTER ASSAY(S) SELECTION, YOU MUST GO TO THE NEXT CELL\")\n",
    "print(f\"{bcolors.HEADER}{bcolors.BOLD}{bcolors.UNDERLINE}Run --> Run Selected Cell and All Below\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run this cell and all below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "if len(unique_values_assay(df4[\"Assay_Name\"])) > 2:\n",
    "    df_protocol_assay = final_df_assay()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Test!!!!!\n",
    "\n",
    "# df_protocol_assay[\"Units\"][0] = \"test\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Replicate_Num</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Measurement Type</th>\n",
       "      <th>Hours</th>\n",
       "      <th>Units</th>\n",
       "      <th>Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>TRYPSIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>7209.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "      <td>SERUM ALBUMIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>2506.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>LYSOZYME C</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>3154.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>TRYPSIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>12513.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>R2</td>\n",
       "      <td>P02769</td>\n",
       "      <td>ALBU_BOVIN</td>\n",
       "      <td>SERUM ALBUMIN</td>\n",
       "      <td>24.0</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>3244.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay_Name Replicate_Num Type_ID    Type_Abb  \\\n",
       "0  TARGETED PROTEOMICS     LINE-1            R1  P00761    TRYP_PIG   \n",
       "1  TARGETED PROTEOMICS     LINE-1            R1  P02769  ALBU_BOVIN   \n",
       "2  TARGETED PROTEOMICS     LINE-1            R1  P00698  LYSC_CHICK   \n",
       "3  TARGETED PROTEOMICS     LINE-1            R2  P00761    TRYP_PIG   \n",
       "4  TARGETED PROTEOMICS     LINE-1            R2  P02769  ALBU_BOVIN   \n",
       "\n",
       "  Measurement Type  Hours   Units    Value  \n",
       "0          TRYPSIN   24.0  COUNTS   7209.0  \n",
       "1    SERUM ALBUMIN   24.0  COUNTS   2506.0  \n",
       "2       LYSOZYME C   24.0  COUNTS   3154.0  \n",
       "3          TRYPSIN   24.0  COUNTS  12513.0  \n",
       "4    SERUM ALBUMIN   24.0  COUNTS   3244.0  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_protocol_assay.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Check Points:\n",
    "\n",
    "if df_protocol_assay[\"Units\"].isnull().values.all(axis=0)== True:\n",
    "    df_protocol_assay[\"Units\"] = \"-\"\n",
    "else:\n",
    "    pass\n",
    "\n",
    "if df_protocol_assay[\"Type_ID\"].isnull().values.all(axis=0)== True:\n",
    "    df_protocol_assay = df_protocol_assay.drop(columns=\"Type_ID\")\n",
    "    stat_df = df_protocol_assay.groupby([\"Protocol\",\"Assay_Name\", \"Type_Abb\",\"Units\",\"Hours\"]).mean()\n",
    "    stat_df.rename(columns={'Value': 'mean'}, inplace=True)\n",
    "    stat_df[\"std\"] = df_protocol_assay.groupby([\"Protocol\",\"Assay_Name\", \"Type_Abb\",\"Units\",\"Hours\"]).std()\n",
    "else:\n",
    "    stat_df = df_protocol_assay.groupby([\"Protocol\",\"Assay_Name\",\"Type_ID\", \"Type_Abb\",\"Units\",\"Hours\"]).mean()\n",
    "    stat_df.rename(columns={'Value': 'mean'}, inplace=True)\n",
    "    stat_df[\"std\"] = df_protocol_assay.groupby([\"Protocol\",\"Assay_Name\", \"Type_ID\", \"Type_Abb\",\"Units\",\"Hours\"]).std()\n",
    "\n",
    "stat_df[\"CV\"] = stat_df['std'] / stat_df['mean']\n",
    "stat_df = stat_df.round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>CV</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Units</th>\n",
       "      <th>Hours</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">TARGETED PROTEOMICS</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">LINE-1</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">P00698</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">LYSC_CHICK</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">COUNTS</th>\n",
       "      <th>24.0</th>\n",
       "      <td>3257.00</td>\n",
       "      <td>145.66</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48.0</th>\n",
       "      <td>4885.50</td>\n",
       "      <td>218.50</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72.0</th>\n",
       "      <td>6351.15</td>\n",
       "      <td>284.04</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">P00761</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">TRYP_PIG</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">COUNTS</th>\n",
       "      <th>24.0</th>\n",
       "      <td>9861.00</td>\n",
       "      <td>3750.49</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48.0</th>\n",
       "      <td>14791.50</td>\n",
       "      <td>5625.74</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    mean  \\\n",
       "Protocol            Assay_Name Type_ID Type_Abb   Units  Hours             \n",
       "TARGETED PROTEOMICS LINE-1     P00698  LYSC_CHICK COUNTS 24.0    3257.00   \n",
       "                                                         48.0    4885.50   \n",
       "                                                         72.0    6351.15   \n",
       "                               P00761  TRYP_PIG   COUNTS 24.0    9861.00   \n",
       "                                                         48.0   14791.50   \n",
       "\n",
       "                                                                    std    CV  \n",
       "Protocol            Assay_Name Type_ID Type_Abb   Units  Hours                 \n",
       "TARGETED PROTEOMICS LINE-1     P00698  LYSC_CHICK COUNTS 24.0    145.66  0.04  \n",
       "                                                         48.0    218.50  0.04  \n",
       "                                                         72.0    284.04  0.04  \n",
       "                               P00761  TRYP_PIG   COUNTS 24.0   3750.49  0.38  \n",
       "                                                         48.0   5625.74  0.38  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stat_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# sum_df = stat_df.unstack(3)\n",
    "sum_df = stat_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>CV</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Units</th>\n",
       "      <th>Hours</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">TARGETED PROTEOMICS</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">LINE-1</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">P00698</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">LYSC_CHICK</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">COUNTS</th>\n",
       "      <th>24.0</th>\n",
       "      <td>3257.00</td>\n",
       "      <td>145.66</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48.0</th>\n",
       "      <td>4885.50</td>\n",
       "      <td>218.50</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72.0</th>\n",
       "      <td>6351.15</td>\n",
       "      <td>284.04</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">P00761</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">TRYP_PIG</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">COUNTS</th>\n",
       "      <th>24.0</th>\n",
       "      <td>9861.00</td>\n",
       "      <td>3750.49</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48.0</th>\n",
       "      <td>14791.50</td>\n",
       "      <td>5625.74</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                    mean  \\\n",
       "Protocol            Assay_Name Type_ID Type_Abb   Units  Hours             \n",
       "TARGETED PROTEOMICS LINE-1     P00698  LYSC_CHICK COUNTS 24.0    3257.00   \n",
       "                                                         48.0    4885.50   \n",
       "                                                         72.0    6351.15   \n",
       "                               P00761  TRYP_PIG   COUNTS 24.0    9861.00   \n",
       "                                                         48.0   14791.50   \n",
       "\n",
       "                                                                    std    CV  \n",
       "Protocol            Assay_Name Type_ID Type_Abb   Units  Hours                 \n",
       "TARGETED PROTEOMICS LINE-1     P00698  LYSC_CHICK COUNTS 24.0    145.66  0.04  \n",
       "                                                         48.0    218.50  0.04  \n",
       "                                                         72.0    284.04  0.04  \n",
       "                               P00761  TRYP_PIG   COUNTS 24.0   3750.49  0.38  \n",
       "                                                         48.0   5625.74  0.38  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# len(unique_values(df[\"Protocol\"]))\n",
    "len(df_protocol[\"Protocol\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Targeted_Proteomics\n"
     ]
    }
   ],
   "source": [
    "# Protocol name\n",
    "\n",
    "user_identified_protocol= df_protocol[\"Protocol\"].unique()[0].replace(\" \",\"_\")\n",
    "print(user_identified_protocol)\n",
    "\n",
    "# current date and time\n",
    "now = datetime.now()\n",
    "analysis_time = now.strftime(\"%Y%m%d-%H%M%S\")\n",
    "# print(analysis_time)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Renaming report files respectively such as subset_sum_data, subset_full_data\n",
    "row_num_ori = df4.shape[0]\n",
    "row_num_custom = df_protocol_assay.shape[0]\n",
    "\n",
    "\n",
    "if row_num_ori == row_num_custom:\n",
    "    # write summary report to a csv with a timestamps\n",
    "    sum_df.to_csv(\"Sum_data_\"+study_name+\"_\"+user_identified_protocol.upper()+\"_\"+analysis_time+\".csv\")\n",
    "    # write full report to a csv with a timestamps\n",
    "    df_protocol_assay.to_csv(\"Full_data_\"+study_name+\"_\"+user_identified_protocol.upper()+\"_\"+analysis_time+\".csv\")\n",
    "else:\n",
    "     # write summary report to a csv with a timestamps\n",
    "    sum_df.to_csv(\"Subset_Sum_data_\"+study_name+\"_\"+user_identified_protocol.upper()+\"_\"+analysis_time+\".csv\")\n",
    "    # write full report to a csv with a timestamps\n",
    "    df_protocol_assay.to_csv(\"Subset_Full_data_\"+study_name+\"_\"+user_identified_protocol.upper()+\"_\"+analysis_time+\".csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.colors as mcolors\n",
    "import plotly.offline as pyo\n",
    "import plotly.graph_objs as go\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Units</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>CV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>3257.00</td>\n",
       "      <td>145.66</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>4885.50</td>\n",
       "      <td>218.50</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>72.0</td>\n",
       "      <td>6351.15</td>\n",
       "      <td>284.04</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>9861.00</td>\n",
       "      <td>3750.49</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>14791.50</td>\n",
       "      <td>5625.74</td>\n",
       "      <td>0.38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay_Name Type_ID    Type_Abb   Units  Time_Point  \\\n",
       "0  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        24.0   \n",
       "1  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        48.0   \n",
       "2  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        72.0   \n",
       "3  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        24.0   \n",
       "4  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        48.0   \n",
       "\n",
       "       mean      std    CV  \n",
       "0   3257.00   145.66  0.04  \n",
       "1   4885.50   218.50  0.04  \n",
       "2   6351.15   284.04  0.04  \n",
       "3   9861.00  3750.49  0.38  \n",
       "4  14791.50  5625.74  0.38  "
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tidy_data_v0 = stat_df.reset_index()\n",
    "tidy_data_v0.rename(columns={'Hours': 'Time_Point'}, inplace=True)\n",
    "\n",
    "tidy_data_v0.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "tidy_data = tidy_data_v0.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "tidy_data[\"CV%\"] = tidy_data[\"CV\"]*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Units</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>CV</th>\n",
       "      <th>CV%</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>3257.00</td>\n",
       "      <td>145.66</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>4885.50</td>\n",
       "      <td>218.50</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>72.0</td>\n",
       "      <td>6351.15</td>\n",
       "      <td>284.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>9861.00</td>\n",
       "      <td>3750.49</td>\n",
       "      <td>0.38</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>14791.50</td>\n",
       "      <td>5625.74</td>\n",
       "      <td>0.38</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay_Name Type_ID    Type_Abb   Units  Time_Point  \\\n",
       "0  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        24.0   \n",
       "1  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        48.0   \n",
       "2  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        72.0   \n",
       "3  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        24.0   \n",
       "4  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        48.0   \n",
       "\n",
       "       mean      std    CV   CV%  \n",
       "0   3257.00   145.66  0.04   4.0  \n",
       "1   4885.50   218.50  0.04   4.0  \n",
       "2   6351.15   284.04  0.04   4.0  \n",
       "3   9861.00  3750.49  0.38  38.0  \n",
       "4  14791.50  5625.74  0.38  38.0  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tidy_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Visualization data, labels\n",
    "l_type = list(tidy_data[\"Type_Abb\"])\n",
    "l_unit = list(tidy_data[\"Units\"].str.lower())\n",
    "tidy_data[\"type_units\"] = [f'{x}*{y}' for x,y in list(zip(l_type, l_unit))]\n",
    "\n",
    "# omics_types = tidy_data[\"Type_Abb\"].unique()\n",
    "omics_types_units = tidy_data[\"type_units\"].unique()\n",
    "type_units = tidy_data[\"type_units\"].unique()\n",
    "assay_types = tidy_data[\"Assay_Name\"].unique()\n",
    "time_points = [time for time in tidy_data.Time_Point.unique() if time in tidy_data.Time_Point.unique()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# sort time points in case there are assays with different time points\n",
    "time_points.sort()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# time_points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Units</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>CV</th>\n",
       "      <th>CV%</th>\n",
       "      <th>type_units</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>3257.00</td>\n",
       "      <td>145.66</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>4885.50</td>\n",
       "      <td>218.50</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>72.0</td>\n",
       "      <td>6351.15</td>\n",
       "      <td>284.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>9861.00</td>\n",
       "      <td>3750.49</td>\n",
       "      <td>0.38</td>\n",
       "      <td>38.0</td>\n",
       "      <td>TRYP_PIG*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>14791.50</td>\n",
       "      <td>5625.74</td>\n",
       "      <td>0.38</td>\n",
       "      <td>38.0</td>\n",
       "      <td>TRYP_PIG*counts</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay_Name Type_ID    Type_Abb   Units  Time_Point  \\\n",
       "0  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        24.0   \n",
       "1  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        48.0   \n",
       "2  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        72.0   \n",
       "3  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        24.0   \n",
       "4  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        48.0   \n",
       "\n",
       "       mean      std    CV   CV%         type_units  \n",
       "0   3257.00   145.66  0.04   4.0  LYSC_CHICK*counts  \n",
       "1   4885.50   218.50  0.04   4.0  LYSC_CHICK*counts  \n",
       "2   6351.15   284.04  0.04   4.0  LYSC_CHICK*counts  \n",
       "3   9861.00  3750.49  0.38  38.0    TRYP_PIG*counts  \n",
       "4  14791.50  5625.74  0.38  38.0    TRYP_PIG*counts  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tidy_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tidy_data[\"type_units\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tidy_data[\"Type_Abb\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# y axis label with units\n",
    "unit_list = [f'avg ('+unit.split(\"*\")[1]+')' for unit in tidy_data[\"type_units\"].unique() ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# unit_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# subplot titles\n",
    "titles_subplots = [unit.split(\"*\")[0] for unit in tidy_data[\"type_units\"].unique() ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# titles_subplots"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bar Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Bar_Example_Data_Quality_Study_TARGETED_PROTEOMICS_20201124-153115.html'"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "\n",
    "subplot_rows= len(tidy_data[\"type_units\"].unique())\n",
    "subplot_cols = 1\n",
    "tot_subplots = subplot_rows * subplot_cols\n",
    "\n",
    "fig= make_subplots(\n",
    "    rows=subplot_rows, cols=subplot_cols,\n",
    "    shared_xaxes=False, shared_yaxes=False,\n",
    "    horizontal_spacing=0.03, \n",
    "#     vertical_spacing=0.03,\n",
    "    subplot_titles=titles_subplots,\n",
    "#     x_title=\"Assay\",\n",
    "#     y_title=\"test_y_title\",\n",
    "#     changing row titles\n",
    "#     row_titles=[\"avg\"]*subplot_rows\n",
    ")\n",
    "\n",
    "\n",
    "for time_point in time_points:\n",
    "    for i in range(len(tidy_data[\"type_units\"].unique())):\n",
    "        omics_type = omics_types_units[i]\n",
    "        mean_vals = list()\n",
    "        std_vals = list()\n",
    "        for j in range(len(tidy_data[\"Assay_Name\"].unique())):\n",
    "            assay_type=assay_types[j]\n",
    "            filt_val = tidy_data[\n",
    "                (tidy_data[\"Time_Point\"]==time_point)\n",
    "                &\n",
    "                (tidy_data[\"type_units\"]==omics_type)\n",
    "                &\n",
    "                (tidy_data[\"Assay_Name\"]==assay_type)\n",
    "                    ]\n",
    "            \n",
    "            if len(filt_val)>0:\n",
    "                mean_val = filt_val[\"mean\"].values[0]\n",
    "            else:\n",
    "                mean_val = 0\n",
    "            mean_vals.append(mean_val)\n",
    "            \n",
    "            if len(filt_val)>0:\n",
    "                std_val = filt_val[\"std\"].values[0]\n",
    "            else:\n",
    "                std_val = 0\n",
    "            std_vals.append(std_val)\n",
    "        \n",
    "#         print(std_vals)\n",
    "        fig.add_trace(\n",
    "             go.Bar(\n",
    "                x=assay_types,\n",
    "                y=mean_vals,\n",
    "                name=time_point,\n",
    "                error_y=dict(\n",
    "                type=\"data\", \n",
    "                array=std_vals,\n",
    "                visible=True,\n",
    "                thickness=1,\n",
    "                width=2\n",
    "                ),\n",
    "#                 hovertext = time_point\n",
    "    #             mode=\"lines\"\n",
    "            ),\n",
    "            row=(i) + 1, col=(1),\n",
    "#             row=(i // subplot_cols) + 1, col=((i % subplot_cols) + 1),\n",
    "            )\n",
    "           \n",
    "            \n",
    "            \n",
    "fig.update_layout(\n",
    "    title_text=f'{user_identified_protocol.upper()} Data',\n",
    "    barmode='group', height=240 * (subplot_rows), width=230 * 8,\n",
    "    showlegend=False,\n",
    "#     xaxis_title=\"x Axis Title\",\n",
    "#     yaxis_title=\"y Axis Title\",\n",
    "#     plot_bgcolor=\"white\",\n",
    "    font=dict(\n",
    "        family=\"Arial, Tahoma, Helvetica\",\n",
    "        size=7,\n",
    "        color=\"#404040\",\n",
    "    ),\n",
    ")\n",
    "fig.update_annotations(patch=dict(font=dict(size=10, color='#404040')))\n",
    "\n",
    "\n",
    "\n",
    "for k in list(range(0, subplot_rows)):\n",
    "    fig.update_yaxes(title_text=unit_list[k], row=k+1, col=1)\n",
    "\n",
    "\n",
    "\n",
    "fig.update_yaxes( fixedrange=True)\n",
    "fig.update_xaxes(patch=(dict(type= 'category')),fixedrange=True)\n",
    "\n",
    "# fig.show(config={'displayModeBar': False})\n",
    "\n",
    "pyo.plot(fig, filename = \"Bar_\"+study_name+\"_\"+user_identified_protocol.upper()+\"_\"+analysis_time+\".html\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Line Plot "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Line plot will be generated if there are are more than 1 time points"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Create a function for line plot: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tidy_data.Time_Point.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "def line_plot(df_plot):\n",
    "    subplot_rows= len(df_plot[\"type_units\"].unique())\n",
    "    subplot_cols = 1\n",
    "\n",
    "    tot_subplots = subplot_rows * subplot_cols\n",
    "\n",
    "    fig= make_subplots(\n",
    "    rows=subplot_rows, cols=subplot_cols,\n",
    "    shared_xaxes=False, shared_yaxes=False,\n",
    "    horizontal_spacing=0.03, \n",
    "    #     vertical_spacing=0.03,\n",
    "    subplot_titles=titles_subplots,\n",
    "#         x_title=\"Time Point\",\n",
    "    #     y_title=\"test_y_title\",\n",
    "#     row_titles=[\"avg\"]*subplot_rows\n",
    "    )\n",
    "    \n",
    "    for j in range(len(df_plot[\"Assay_Name\"].unique())):\n",
    "        assay_type=assay_types[j]\n",
    "        for i in range(len(df_plot[\"type_units\"].unique())):\n",
    "            omics_type = omics_types_units[i]\n",
    "            mean_vals = list()\n",
    "            std_vals=list()\n",
    "            for time_point in time_points:\n",
    "                filt_val = df_plot[\n",
    "                    (df_plot[\"type_units\"]==omics_type)\n",
    "                    &\n",
    "                    (df_plot[\"Assay_Name\"]==assay_type)\n",
    "                    &\n",
    "                    (df_plot[\"Time_Point\"]==time_point)\n",
    "                    ]\n",
    "                if len(filt_val)>0:\n",
    "                    mean_val = filt_val[\"mean\"].values[0]\n",
    "                else:\n",
    "                    mean_val = 0\n",
    "                mean_vals.append(mean_val)\n",
    "    \n",
    "    \n",
    "    \n",
    "                if len(filt_val)>0:\n",
    "                    std_val = filt_val[\"std\"].values[0]\n",
    "                else:\n",
    "                    std_val = 0\n",
    "                std_vals.append(std_val)\n",
    "    \n",
    "            fig.add_trace(\n",
    "                go.Scatter(\n",
    "                x=time_points,\n",
    "                y=mean_vals,\n",
    "                name=assay_type,\n",
    "                error_y=dict(\n",
    "                    type=\"data\", \n",
    "                    array=std_vals,\n",
    "                    visible=True,\n",
    "                    thickness=1,\n",
    "                    width=2    \n",
    "                ),\n",
    "                mode=\"lines\"\n",
    "            ),\n",
    "            row=(i) + 1, col=(1),\n",
    "#             row=(i // subplot_cols) + 1, col=((i % subplot_cols) + 1),\n",
    "            )\n",
    "           \n",
    "    fig.update_layout(\n",
    "        title_text=f'{user_identified_protocol.upper()} Data',\n",
    "        barmode='group', height=240 * (subplot_rows), width=230 * 8,\n",
    "        showlegend=False,\n",
    "    #     plot_bgcolor=\"white\",\n",
    "        font=dict(\n",
    "            family=\"Arial, Tahoma, Helvetica\",\n",
    "            size=7,\n",
    "            color=\"#404040\",\n",
    "        ),\n",
    "    )\n",
    "    fig.update_annotations(patch=dict(font=dict(size=10, color='#404040')))\n",
    "\n",
    "    for k in list(range(0, subplot_rows)):\n",
    "        fig.update_yaxes(title_text=unit_list[k], row=k+1, col=1)\n",
    "        \n",
    "#     fig.update_yaxes( fixedrange=True)\n",
    "    fig.update_xaxes(fixedrange=True)\n",
    "#     fig.show(config={'displayModeBar': False})\n",
    "\n",
    "       \n",
    "    return pyo.plot(fig, filename = \"Line_\"+study_name+\"_\"+user_identified_protocol.upper()+\"_\"+analysis_time+\".html\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "if len(tidy_data.Time_Point.unique())>1:\n",
    "    line_plot(tidy_data)\n",
    "else:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create a function to generate scatter plots.**\n",
    "\n",
    "**Condition: if there is no replicates, do not generate scatter plots. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Quality visuals:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# checkpoint: if assay names are only numerical, \n",
    "# Work around to change numerical assay names to categorical name, added letter \"x\"\n",
    "# tidy_data[\"Assay_Name\"] = [f\"x{x}\" for x in tidy_data[\"Assay_Name\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Units</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>CV</th>\n",
       "      <th>CV%</th>\n",
       "      <th>type_units</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>3257.00</td>\n",
       "      <td>145.66</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>4885.50</td>\n",
       "      <td>218.50</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>72.0</td>\n",
       "      <td>6351.15</td>\n",
       "      <td>284.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>9861.00</td>\n",
       "      <td>3750.49</td>\n",
       "      <td>0.38</td>\n",
       "      <td>38.0</td>\n",
       "      <td>TRYP_PIG*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>14791.50</td>\n",
       "      <td>5625.74</td>\n",
       "      <td>0.38</td>\n",
       "      <td>38.0</td>\n",
       "      <td>TRYP_PIG*counts</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay_Name Type_ID    Type_Abb   Units  Time_Point  \\\n",
       "0  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        24.0   \n",
       "1  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        48.0   \n",
       "2  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        72.0   \n",
       "3  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        24.0   \n",
       "4  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        48.0   \n",
       "\n",
       "       mean      std    CV   CV%         type_units  \n",
       "0   3257.00   145.66  0.04   4.0  LYSC_CHICK*counts  \n",
       "1   4885.50   218.50  0.04   4.0  LYSC_CHICK*counts  \n",
       "2   6351.15   284.04  0.04   4.0  LYSC_CHICK*counts  \n",
       "3   9861.00  3750.49  0.38  38.0    TRYP_PIG*counts  \n",
       "4  14791.50  5625.74  0.38  38.0    TRYP_PIG*counts  "
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tidy_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "def is_int(val):\n",
    "    try:\n",
    "        num = int(val)\n",
    "    except ValueError:\n",
    "        return False\n",
    "    return True\n",
    "\n",
    "if  is_int(tidy_data[\"Assay_Name\"][0]):\n",
    "    tidy_data[\"Assay_Name\"] = [f\"x{x}\" for x in tidy_data[\"Assay_Name\"]]\n",
    "else:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Units</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>CV</th>\n",
       "      <th>CV%</th>\n",
       "      <th>type_units</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>3257.00</td>\n",
       "      <td>145.66</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>4885.50</td>\n",
       "      <td>218.50</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>72.0</td>\n",
       "      <td>6351.15</td>\n",
       "      <td>284.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>9861.00</td>\n",
       "      <td>3750.49</td>\n",
       "      <td>0.38</td>\n",
       "      <td>38.0</td>\n",
       "      <td>TRYP_PIG*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>14791.50</td>\n",
       "      <td>5625.74</td>\n",
       "      <td>0.38</td>\n",
       "      <td>38.0</td>\n",
       "      <td>TRYP_PIG*counts</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay_Name Type_ID    Type_Abb   Units  Time_Point  \\\n",
       "0  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        24.0   \n",
       "1  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        48.0   \n",
       "2  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        72.0   \n",
       "3  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        24.0   \n",
       "4  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        48.0   \n",
       "\n",
       "       mean      std    CV   CV%         type_units  \n",
       "0   3257.00   145.66  0.04   4.0  LYSC_CHICK*counts  \n",
       "1   4885.50   218.50  0.04   4.0  LYSC_CHICK*counts  \n",
       "2   6351.15   284.04  0.04   4.0  LYSC_CHICK*counts  \n",
       "3   9861.00  3750.49  0.38  38.0    TRYP_PIG*counts  \n",
       "4  14791.50  5625.74  0.38  38.0    TRYP_PIG*counts  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tidy_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "l_type = list(tidy_data[\"Type_Abb\"])\n",
    "l_unit = list(tidy_data[\"Units\"])\n",
    "h_text = [f'{x},{y}' for x,y in list(zip(l_type, l_unit))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# h_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OPTIONAL: SPECIFY CUTOFF PEAK AREA "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[95m\u001b[1m\u001b[4mOPTIONAL: SPECIFY CUTOFF PEAK AREA , Default is ZERO for cutoff peak area  \n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b411c0fdc5e04e528d7857745df97c65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(IntSlider(value=0, description='Cutoff Peak:', layout=Layout(height='100px', width='75%'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[95m\u001b[1m\u001b[4mAFTER CHANGING THE INTENSITY, YOU MUST GO TO THE NEXT CELL\n",
      "\u001b[95m\u001b[1m\u001b[4mRun --> Run Selected Cell and All Below\n"
     ]
    }
   ],
   "source": [
    "# Create a slider to specify cutoff peak area\n",
    "\n",
    "slider = widgets.IntSlider(min=0, max=max(tidy_data[\"mean\"]), value=0, description='Cutoff Peak:',step=100, layout=widgets.Layout(width='75%', height='100px'))\n",
    "\n",
    "\n",
    "print(f\"{bcolors.HEADER}{bcolors.BOLD}{bcolors.UNDERLINE}OPTIONAL: SPECIFY CUTOFF PEAK AREA , Default is ZERO for cutoff peak area  \")\n",
    "\n",
    "\n",
    "def return_tidy_data0():\n",
    "    user_cutoff = slider.value\n",
    "    if user_cutoff == 0:\n",
    "        tidy_data0 = tidy_data.copy()\n",
    "    else:\n",
    "        tidy_data0 = tidy_data[tidy_data[\"mean\"]>=user_cutoff].reset_index(drop=True)\n",
    "    return tidy_data0\n",
    "\n",
    "def user_cutoff_value():\n",
    "    return slider.value\n",
    "\n",
    "\n",
    " \n",
    "def calculation_extracted_data():\n",
    "    user_cutoff = slider.value\n",
    "    if user_cutoff == 0:\n",
    "        tidy_data0 = tidy_data.copy()\n",
    "        new_data_counts = 0\n",
    "    else:\n",
    "        tidy_data0 = tidy_data[tidy_data[\"mean\"]>=user_cutoff].reset_index(drop=True)\n",
    "        new_data_counts = tidy_data0.shape[0]\n",
    "    ori_data_counts = tidy_data.shape[0]\n",
    "    perc_ori_data_counts = (new_data_counts*100)/ori_data_counts\n",
    "    sum_cutoff_area = [new_data_counts,ori_data_counts,round(perc_ori_data_counts,2)]\n",
    "    return sum_cutoff_area\n",
    "\n",
    "def tidy_data_cutoff(user_cutoff):\n",
    "    user_cutoff = slider.value\n",
    "   \n",
    "    sum_cutoff_area_handler = calculation_extracted_data()\n",
    "    \n",
    "#    Buttons\n",
    "    b1 = widgets.Button(description=\"Subset Data\", layout=widgets.Layout(width=\"30%\"))\n",
    "    b2 = widgets.Button(description=\"Full Data\", layout=widgets.Layout(width=\"30%\"))\n",
    "    b3 = widgets.Button(description=\"Percentage %\", layout=widgets.Layout(width=\"30%\"))\n",
    "    b4 = widgets.Button(description=str(sum_cutoff_area_handler[0]), layout=widgets.Layout(width=\"30%\"))\n",
    "    b5 = widgets.Button(description=str(sum_cutoff_area_handler[1]), layout=widgets.Layout(width=\"30%\"))\n",
    "    b6 = widgets.Button(description=str(sum_cutoff_area_handler[2])+\"%\", layout=widgets.Layout(width=\"30%\"))    \n",
    "    h1 = widgets.HBox(children=[b1,b2,b3])\n",
    "    h2 = widgets.HBox(children=[b4,b5, b6])\n",
    "    data_buttons  = widgets.VBox(children=[h1,h2])\n",
    "    if user_cutoff == 0:\n",
    "        tidy_data0 = tidy_data.copy()  \n",
    "        display(data_buttons)\n",
    "        display(tidy_data0.head())\n",
    "    else:\n",
    "        tidy_data0 = tidy_data[tidy_data[\"mean\"]>=user_cutoff].reset_index(drop=True)\n",
    "        display(data_buttons)\n",
    "        display(tidy_data0.head())\n",
    "\n",
    "widgets.interact(tidy_data_cutoff, user_cutoff = slider)\n",
    "\n",
    "print(f\"{bcolors.HEADER}{bcolors.BOLD}{bcolors.UNDERLINE}AFTER CHANGING THE INTENSITY, YOU MUST GO TO THE NEXT CELL\")\n",
    "print(f\"{bcolors.HEADER}{bcolors.BOLD}{bcolors.UNDERLINE}Run --> Run Selected Cell and All Below\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run this cell and all below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "tidy_data_df = return_tidy_data0()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Protocol</th>\n",
       "      <th>Assay_Name</th>\n",
       "      <th>Type_ID</th>\n",
       "      <th>Type_Abb</th>\n",
       "      <th>Units</th>\n",
       "      <th>Time_Point</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>CV</th>\n",
       "      <th>CV%</th>\n",
       "      <th>type_units</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>3257.00</td>\n",
       "      <td>145.66</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>4885.50</td>\n",
       "      <td>218.50</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00698</td>\n",
       "      <td>LYSC_CHICK</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>72.0</td>\n",
       "      <td>6351.15</td>\n",
       "      <td>284.04</td>\n",
       "      <td>0.04</td>\n",
       "      <td>4.0</td>\n",
       "      <td>LYSC_CHICK*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>24.0</td>\n",
       "      <td>9861.00</td>\n",
       "      <td>3750.49</td>\n",
       "      <td>0.38</td>\n",
       "      <td>38.0</td>\n",
       "      <td>TRYP_PIG*counts</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TARGETED PROTEOMICS</td>\n",
       "      <td>LINE-1</td>\n",
       "      <td>P00761</td>\n",
       "      <td>TRYP_PIG</td>\n",
       "      <td>COUNTS</td>\n",
       "      <td>48.0</td>\n",
       "      <td>14791.50</td>\n",
       "      <td>5625.74</td>\n",
       "      <td>0.38</td>\n",
       "      <td>38.0</td>\n",
       "      <td>TRYP_PIG*counts</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Protocol Assay_Name Type_ID    Type_Abb   Units  Time_Point  \\\n",
       "0  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        24.0   \n",
       "1  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        48.0   \n",
       "2  TARGETED PROTEOMICS     LINE-1  P00698  LYSC_CHICK  COUNTS        72.0   \n",
       "3  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        24.0   \n",
       "4  TARGETED PROTEOMICS     LINE-1  P00761    TRYP_PIG  COUNTS        48.0   \n",
       "\n",
       "       mean      std    CV   CV%         type_units  \n",
       "0   3257.00   145.66  0.04   4.0  LYSC_CHICK*counts  \n",
       "1   4885.50   218.50  0.04   4.0  LYSC_CHICK*counts  \n",
       "2   6351.15   284.04  0.04   4.0  LYSC_CHICK*counts  \n",
       "3   9861.00  3750.49  0.38  38.0    TRYP_PIG*counts  \n",
       "4  14791.50  5625.74  0.38  38.0    TRYP_PIG*counts  "
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tidy_data_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# tidy_data_df.shape\n",
    "# tidy_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "cutoff_peak_area = user_cutoff_value()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cutoff_peak_area"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# current date and time\n",
    "now = datetime.now()\n",
    "cutoff_peak_time = now.strftime(\"%Y%m%d-%H%M%S\")\n",
    "\n",
    "# Exporting dataset after cutoff peak area gets updated:\n",
    "\n",
    "if user_cutoff_value() >0 :\n",
    "    print(f\"User changed cutoff peak area: {cutoff_peak_area} ,Please see the new csv file generated as trimmed_data_DATE.csv\")\n",
    "    tidy_data_df.to_csv(\"trimmed_data_\"+cutoff_peak_time+\".csv\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scatter, Violin plot visualization function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "def data_quality_visual(df_dq):\n",
    "\n",
    "# Creating labels for markers    \n",
    "    l_type = list(tidy_data_df[\"Type_Abb\"])\n",
    "    l_unit = list(tidy_data_df[\"Units\"].str.lower())\n",
    "    h_text = [f'{x}, {y}' for x,y in list(zip(l_type, l_unit))]\n",
    "    \n",
    "    \n",
    "#   Specifying colors in order to make assay colors same in scatter and violin \n",
    "    cols=list(mcolors.XKCD_COLORS.values())\n",
    "    \n",
    "        \n",
    "    # Calculating overall Data Quality\n",
    "    tidy_data_passed_20 = df_dq[df_dq[\"CV%\"]<20]\n",
    "    counts = df_dq.shape[0]\n",
    "    counts_dq = tidy_data_passed_20.shape[0]\n",
    "    data_quality = round((counts_dq/counts)*100,2)\n",
    "    \n",
    "    # Calculating Data Quality metric per assay\n",
    "\n",
    "    assay_name_list = list(df_dq[\"Assay_Name\"].unique())\n",
    "    time_point_list = list(df_dq[\"Time_Point\"].unique())\n",
    "    assay_dq = []\n",
    "    for assay in assay_name_list:\n",
    "        for time in time_point_list:\n",
    "            counts_at = df_dq[(df_dq[\"Assay_Name\"]==assay)&(df_dq[\"Time_Point\"]==time)].shape[0]\n",
    "            assay_passed_20 = df_dq.loc[(df_dq[\"Assay_Name\"]==assay)&(df_dq[\"Time_Point\"]==time)&(df_dq[\"CV%\"]<20)].shape[0]\n",
    "            if counts_at == 0 : \n",
    "                pass\n",
    "            elif assay_passed_20 == 0:\n",
    "                assay_dq.append(0)\n",
    "            else:\n",
    "                assay_dq.append(round((assay_passed_20/counts_at)*100,2))    \n",
    "    # Creating Data Quality metric dataframe (Assay Name, Time Point, DataPointCount, DataQuality% )\n",
    "\n",
    "    df_dq_assay = df_dq.groupby([\"Assay_Name\", \"Time_Point\"]).size().reset_index(name='DataPointCounts')\n",
    "    df_dq_assay[\"DataQuality%\"] = assay_dq\n",
    "    \n",
    "\n",
    "    fig_dq = make_subplots(\n",
    "        rows=3, cols=1,\n",
    "        shared_yaxes=False,\n",
    "        shared_xaxes=False,\n",
    "        vertical_spacing=0.08,\n",
    "#         y_title=\"Proteomics Data Quality\",\n",
    "        row_titles=[\"CV%\"],\n",
    "        specs=[[{\"type\": \"scatter\"}],\n",
    "               [{\"type\": \"table\"}],\n",
    "               [{\"type\": \"table\"}]]\n",
    "        )\n",
    "\n",
    "    # Add traces\n",
    "    for assay in assay_name_list:\n",
    "        fig_dq.add_trace(\n",
    "                go.Scatter(\n",
    "                        x=df_dq[\"mean\"][df_dq[\"Assay_Name\"]==assay], \n",
    "                        y=df_dq[\"CV%\"][df_dq[\"Assay_Name\"]==assay],\n",
    "                        mode='markers',\n",
    "                        hovertext=h_text,\n",
    "                        marker=dict(color=cols[assay_name_list.index(assay)]),\n",
    "                        name=assay,\n",
    "                        ),\n",
    "                row=1,col=1)\n",
    "\n",
    "    fig_dq.add_trace(\n",
    "        go.Table(\n",
    "            columnwidth = [1,1],\n",
    "            header=dict(\n",
    "                    values=list(df_dq_assay.columns),\n",
    "                    font=dict(size=15),\n",
    "                    align=\"left\"\n",
    "                    ),\n",
    "            cells=dict(\n",
    "                    values=[df_dq_assay[\"Assay_Name\"],\n",
    "                            df_dq_assay[\"Time_Point\"],\n",
    "                            df_dq_assay[\"DataPointCounts\"],\n",
    "                            df_dq_assay[\"DataQuality%\"]],\n",
    "                    align = \"left\",\n",
    "                    ) \n",
    "    \n",
    "                    ),\n",
    "            row=2, col=1\n",
    "            )\n",
    "\n",
    "    fig_dq.add_trace(\n",
    "        go.Table(\n",
    "            columnwidth = [80,100],\n",
    "            header=dict(\n",
    "                values=[\"Overall Data Quality%\"],\n",
    "                font=dict(size=20),\n",
    "                align=\"left\"\n",
    "            ),\n",
    "            cells=dict(\n",
    "                values=[data_quality],\n",
    "                align = \"left\") \n",
    "\n",
    "        ),\n",
    "        row=3, col=1\n",
    "    )\n",
    "    \n",
    "    \n",
    "    fig_dq.add_shape(\n",
    "        # Line Horizontal\n",
    "            type=\"line\",\n",
    "            x0=0,\n",
    "            y0=20,\n",
    "            x1=df_dq[\"mean\"].max(),\n",
    "            y1=20,\n",
    "            line=dict(\n",
    "                color=\"Red\",\n",
    "                width=2,\n",
    "                dash=\"dashdot\",\n",
    "            ),row=1,col=1\n",
    "        )\n",
    "    # make space for explanation / annotation\n",
    "#     fig_dq.update_layout(\n",
    "# #                         margin=dict(l=20, r=20, t=20, b=60),\n",
    "#                          paper_bgcolor=\"LightSteelBlue\")\n",
    "\n",
    "    # add annotation\n",
    "    fig_dq.add_annotation(dict(font=dict(family=\"Times New Roman\",color='black',size=20),\n",
    "                                        x=0,\n",
    "                                        y=0.69,\n",
    "                                        showarrow=False,\n",
    "                                        text=f\"DataQuality%: The percentage of the data is below 20% CV (coefficient variation) for each set of replicates. <br> The red line in the scatter plot corresponds to 20% CV, a commonly used cutoff for data quality. This may or may not be suitable for your application..\",\n",
    "                                        textangle=0,\n",
    "                                        align=\"left\",\n",
    "                                        xanchor='left',\n",
    "                                        xref=\"paper\",\n",
    "                                        yref=\"paper\"\n",
    "                           ))\n",
    "\n",
    "    \n",
    "    fig_dq.update_layout(\n",
    "            height=2000,\n",
    "            title_text=f'{user_identified_protocol.upper()} Data Quality'\n",
    "#             title_text= f\"DataQuality%: The percentage of the data is below 20% CV (coefficient variation) for each set of replicates. <br> The red line in the scatter plot corresponds to 20% CV, a commonly used cutoff for data quality. This may or may not be suitable for your application..\"\n",
    "            )\n",
    "\n",
    "    \n",
    "#     fig_dq.show(config={'displayModeBar': False})\n",
    "\n",
    "#     py.plot(fig_dq, filename=\"scatter_violin_table.html\")\n",
    "    return pyo.plot(fig_dq, filename = \"Scatter_\"+study_name+\"_\"+user_identified_protocol.upper()+\"_\"+cutoff_peak_time+\".html\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Scatter Visualization step Data Quality per Time point: \n",
    "\n",
    "# If there are assay replicates, provide an option to modify cutoff peak area\n",
    "if tidy_data[\"CV%\"].isnull().all():\n",
    "    pass\n",
    "else:\n",
    "    data_quality_visual(tidy_data_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Violin Plot per Time point:\n",
    "\n",
    "def violin_visual(df_dq):\n",
    "\n",
    "    subplot_rows= len(tidy_data[\"Time_Point\"].unique())\n",
    "    subplot_cols = 1\n",
    "    tot_subplots = subplot_rows * subplot_cols\n",
    "    \n",
    "\n",
    "\n",
    "# Creating labels for markers    \n",
    "    l_type = list(tidy_data_df[\"Type_Abb\"])\n",
    "    l_unit = list(tidy_data_df[\"Units\"].str.lower())\n",
    "    h_text = [f'{x}, {y}' for x,y in list(zip(l_type, l_unit))]\n",
    "    \n",
    "    \n",
    "#   Specifying colors in order to make assay colors same in scatter and violin \n",
    "    cols=list(mcolors.XKCD_COLORS.values())\n",
    "    \n",
    "        \n",
    "    # Calculating overall Data Quality\n",
    "    tidy_data_passed_20 = df_dq[df_dq[\"CV%\"]<20]\n",
    "    counts = df_dq.shape[0]\n",
    "    counts_dq = tidy_data_passed_20.shape[0]\n",
    "    data_quality = round((counts_dq/counts)*100,2)\n",
    "    \n",
    "    # Calculating Data Quality metric per assay\n",
    "    assay_dq = []\n",
    "    assay_name_list = list(df_dq[\"Assay_Name\"].unique())\n",
    "    time_point_list = list(df_dq[\"Time_Point\"].unique())\n",
    "#     Sort the time point list. \n",
    "    time_point_list.sort()\n",
    "    \n",
    "    \n",
    "    \n",
    "    for assay in assay_name_list:\n",
    "        for time in time_point_list:\n",
    "            counts_at = df_dq[(df_dq[\"Assay_Name\"]==assay)&(df_dq[\"Time_Point\"]==time)].shape[0]\n",
    "            assay_passed_20 = df_dq.loc[(df_dq[\"Assay_Name\"]==assay)&(df_dq[\"Time_Point\"]==time)&(df_dq[\"CV%\"]<20)].shape[0]\n",
    "            if counts_at == 0 : \n",
    "                pass\n",
    "            elif assay_passed_20 == 0:\n",
    "                assay_dq.append(0)\n",
    "            else:\n",
    "                assay_dq.append(round((assay_passed_20/counts_at)*100,2))\n",
    "                \n",
    "    \n",
    "    # Creating Data Quality metric dataframe (Assay Name, Time Point, DataPointCount, DataQuality% )\n",
    "\n",
    "    df_dq_assay = df_dq.groupby([\"Assay_Name\", \"Time_Point\"]).size().reset_index(name='DataPointCounts')\n",
    "    df_dq_assay[\"DataQuality%\"] = assay_dq\n",
    "    \n",
    "\n",
    "#     Title Subplots with Time Point\n",
    "    time_subplots = list(map(lambda x: \"CV% Per Time Point(h): \" + str(x) , time_point_list))\n",
    "\n",
    "#     Make subplots\n",
    "\n",
    "    fig_dq= make_subplots(\n",
    "        rows=subplot_rows, cols=subplot_cols,\n",
    "        shared_xaxes=False, shared_yaxes=False,\n",
    "        horizontal_spacing=0.03, \n",
    "    vertical_spacing=0.03,\n",
    "        subplot_titles=time_subplots,\n",
    "#     x_title=\"Assay\",\n",
    "#     y_title=\"test_y_title\",\n",
    "#     changing row titles\n",
    "#     row_titles=[\"avg\"]*subplot_rows\n",
    ")\n",
    "    \n",
    "    \n",
    "    # Add traces\n",
    "    for i in range(len(time_point_list)):\n",
    "        time = time_point_list[i]\n",
    "        for j in range(len(assay_name_list)):\n",
    "            assay = assay_name_list[j]\n",
    "            \n",
    "    \n",
    "    for i in range(len(time_point_list)):\n",
    "        time = time_point_list[i]\n",
    "        for j in range(len(assay_name_list)):\n",
    "            assay = assay_name_list[j]\n",
    "            try:\n",
    "                max_CV = max(tidy_data_df[\"CV%\"][(tidy_data_df[\"Assay_Name\"]==assay)&(tidy_data_df[\"Time_Point\"]==time)])\n",
    "            except ValueError:\n",
    "                max_CV = 0\n",
    "            fig_dq.add_trace(\n",
    "                    go.Violin(\n",
    "                        x=df_dq[\"Assay_Name\"][df_dq[\"Assay_Name\"]==assay], \n",
    "                        y=df_dq[\"CV%\"][(df_dq[\"Assay_Name\"]==assay)&(df_dq[\"Time_Point\"]==time)],\n",
    "                        name=assay,\n",
    "                        box_visible=True,\n",
    "                        meanline_visible=True,\n",
    "                        line_color=\"Black\",\n",
    "                        fillcolor=cols[assay_name_list.index(assay)],\n",
    "#                         span = [0, 100], spanmode='manual'\n",
    "#                         constrain the distribution shown on a violin plot to be within the bounds of the data that is being plotted\n",
    "                        span = [0, max_CV], spanmode='manual'\n",
    "\n",
    "                    ),\n",
    "\n",
    "                row=i+1,col=1\n",
    "                )\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    fig_dq.update_layout(\n",
    "            height=3000,\n",
    "            title_text=f'{user_identified_protocol.upper()} Data Quality Per Time Point'\n",
    "            )\n",
    "\n",
    "#     fig_dq.show(config={'displayModeBar': False})\n",
    "\n",
    "#     py.plot(fig_dq, filename=\"scatter_violin_table.html\")\n",
    "    return pyo.plot(fig_dq, filename = \"Violin_\"+study_name+\"_\"+user_identified_protocol.upper()+\"_\"+cutoff_peak_time+\".html\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "# Violin Visualization step Data Quality per Time point: \n",
    "\n",
    "# If there are assay replicates, provide an option to modify cutoff peak area\n",
    "if tidy_data[\"CV%\"].isnull().all():\n",
    "    pass\n",
    "else:\n",
    "    violin_visual(tidy_data_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "jupyter": {
     "source_hidden": true
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ESE Data Analysis",
   "language": "python",
   "name": "ese_data_analysis"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}