4.covariates.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating a covariate dataset that encodes categorical sample variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The sample data possesses covariates as shown in <a href='https://github.com/cognoma/cancer-data/blob/master/3.explore-mutations.ipynb'>this notebook</a>. These may provide a spurious signal that a classifier accommodates for, and could confound attempts to pick out the actual signal that we desire. This notebook will create a file with encoded information on these covariates. Classifiers being implemented by the machine learning group can use this as additional data to train on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's peak at the sample data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>sample_id</th>\n",
       "      <th>TCGA-02-0047-01</th>\n",
       "      <th>TCGA-02-0055-01</th>\n",
       "      <th>TCGA-02-2483-01</th>\n",
       "      <th>TCGA-02-2485-01</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>patient_id</th>\n",
       "      <td>TCGA-02-0047</td>\n",
       "      <td>TCGA-02-0055</td>\n",
       "      <td>TCGA-02-2483</td>\n",
       "      <td>TCGA-02-2485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acronym</th>\n",
       "      <td>GBM</td>\n",
       "      <td>GBM</td>\n",
       "      <td>GBM</td>\n",
       "      <td>GBM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disease</th>\n",
       "      <td>glioblastoma multiforme</td>\n",
       "      <td>glioblastoma multiforme</td>\n",
       "      <td>glioblastoma multiforme</td>\n",
       "      <td>glioblastoma multiforme</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>age_diagnosed</th>\n",
       "      <td>78</td>\n",
       "      <td>62</td>\n",
       "      <td>43</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gender</th>\n",
       "      <td>Male</td>\n",
       "      <td>Female</td>\n",
       "      <td>Male</td>\n",
       "      <td>Male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>race</th>\n",
       "      <td>White</td>\n",
       "      <td>White</td>\n",
       "      <td>Asian</td>\n",
       "      <td>Black Or African American</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ajcc_stage</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clinical_stage</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>histological_type</th>\n",
       "      <td>Untreated primary (de novo) GBM</td>\n",
       "      <td>Untreated primary (de novo) GBM</td>\n",
       "      <td>Untreated primary (de novo) GBM</td>\n",
       "      <td>Untreated primary (de novo) GBM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>histological_grade</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>initial_pathologic_dx_year</th>\n",
       "      <td>2005</td>\n",
       "      <td>2005</td>\n",
       "      <td>2008</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>menopause_status</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>birth_days_to</th>\n",
       "      <td>-28759</td>\n",
       "      <td>-22798</td>\n",
       "      <td>-15964</td>\n",
       "      <td>-19494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vital_status</th>\n",
       "      <td>Dead</td>\n",
       "      <td>Dead</td>\n",
       "      <td>Alive</td>\n",
       "      <td>Alive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tumor_status</th>\n",
       "      <td>WITH TUMOR</td>\n",
       "      <td>WITH TUMOR</td>\n",
       "      <td>WITH TUMOR</td>\n",
       "      <td>WITH TUMOR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>last_contact_days_to</th>\n",
       "      <td>448</td>\n",
       "      <td>76</td>\n",
       "      <td>466</td>\n",
       "      <td>470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>death_days_to</th>\n",
       "      <td>448</td>\n",
       "      <td>76</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cause_of_death</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_tumor_event_type</th>\n",
       "      <td>Progression of Disease</td>\n",
       "      <td>Progression of Disease</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Progression of Disease</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_tumor_event_site</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new_tumor_event_site_other</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_recurrence_free</th>\n",
       "      <td>57</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>treatment_outcome_first_course</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>margin_status</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>residual_tumor</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>event_status</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>event_days</th>\n",
       "      <td>448</td>\n",
       "      <td>76</td>\n",
       "      <td>466</td>\n",
       "      <td>470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dead</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>days_survived</th>\n",
       "      <td>448</td>\n",
       "      <td>76</td>\n",
       "      <td>466</td>\n",
       "      <td>470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disease_specific_survival_status</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disease_specific_survival_days</th>\n",
       "      <td>448</td>\n",
       "      <td>76</td>\n",
       "      <td>466</td>\n",
       "      <td>470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disease_free_interval_status</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>disease_free_interval_days</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>progression_free_interval_status</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>progression_free_interval_days</th>\n",
       "      <td>57</td>\n",
       "      <td>6</td>\n",
       "      <td>466</td>\n",
       "      <td>186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_type</th>\n",
       "      <td>Primary Solid Tumor</td>\n",
       "      <td>Primary Solid Tumor</td>\n",
       "      <td>Primary Solid Tumor</td>\n",
       "      <td>Primary Solid Tumor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_mutations</th>\n",
       "      <td>64</td>\n",
       "      <td>51</td>\n",
       "      <td>47</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recurred</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "sample_id                                         TCGA-02-0047-01  \\\n",
       "patient_id                                           TCGA-02-0047   \n",
       "acronym                                                       GBM   \n",
       "disease                                   glioblastoma multiforme   \n",
       "age_diagnosed                                                  78   \n",
       "gender                                                       Male   \n",
       "race                                                        White   \n",
       "ajcc_stage                                                    NaN   \n",
       "clinical_stage                                                NaN   \n",
       "histological_type                 Untreated primary (de novo) GBM   \n",
       "histological_grade                                            NaN   \n",
       "initial_pathologic_dx_year                                   2005   \n",
       "menopause_status                                              NaN   \n",
       "birth_days_to                                              -28759   \n",
       "vital_status                                                 Dead   \n",
       "tumor_status                                           WITH TUMOR   \n",
       "last_contact_days_to                                          448   \n",
       "death_days_to                                                 448   \n",
       "cause_of_death                                                NaN   \n",
       "new_tumor_event_type                       Progression of Disease   \n",
       "new_tumor_event_site                                          NaN   \n",
       "new_tumor_event_site_other                                    NaN   \n",
       "days_recurrence_free                                           57   \n",
       "treatment_outcome_first_course                                NaN   \n",
       "margin_status                                                 NaN   \n",
       "residual_tumor                                                NaN   \n",
       "event_status                                                    1   \n",
       "event_days                                                    448   \n",
       "dead                                                            1   \n",
       "days_survived                                                 448   \n",
       "disease_specific_survival_status                                1   \n",
       "disease_specific_survival_days                                448   \n",
       "disease_free_interval_status                                  NaN   \n",
       "disease_free_interval_days                                    NaN   \n",
       "progression_free_interval_status                                1   \n",
       "progression_free_interval_days                                 57   \n",
       "sample_type                                   Primary Solid Tumor   \n",
       "n_mutations                                                    64   \n",
       "recurred                                                        1   \n",
       "\n",
       "sample_id                                         TCGA-02-0055-01  \\\n",
       "patient_id                                           TCGA-02-0055   \n",
       "acronym                                                       GBM   \n",
       "disease                                   glioblastoma multiforme   \n",
       "age_diagnosed                                                  62   \n",
       "gender                                                     Female   \n",
       "race                                                        White   \n",
       "ajcc_stage                                                    NaN   \n",
       "clinical_stage                                                NaN   \n",
       "histological_type                 Untreated primary (de novo) GBM   \n",
       "histological_grade                                            NaN   \n",
       "initial_pathologic_dx_year                                   2005   \n",
       "menopause_status                                              NaN   \n",
       "birth_days_to                                              -22798   \n",
       "vital_status                                                 Dead   \n",
       "tumor_status                                           WITH TUMOR   \n",
       "last_contact_days_to                                           76   \n",
       "death_days_to                                                  76   \n",
       "cause_of_death                                                NaN   \n",
       "new_tumor_event_type                       Progression of Disease   \n",
       "new_tumor_event_site                                          NaN   \n",
       "new_tumor_event_site_other                                    NaN   \n",
       "days_recurrence_free                                            6   \n",
       "treatment_outcome_first_course                                NaN   \n",
       "margin_status                                                 NaN   \n",
       "residual_tumor                                                NaN   \n",
       "event_status                                                    1   \n",
       "event_days                                                     76   \n",
       "dead                                                            1   \n",
       "days_survived                                                  76   \n",
       "disease_specific_survival_status                                1   \n",
       "disease_specific_survival_days                                 76   \n",
       "disease_free_interval_status                                  NaN   \n",
       "disease_free_interval_days                                    NaN   \n",
       "progression_free_interval_status                                1   \n",
       "progression_free_interval_days                                  6   \n",
       "sample_type                                   Primary Solid Tumor   \n",
       "n_mutations                                                    51   \n",
       "recurred                                                        1   \n",
       "\n",
       "sample_id                                         TCGA-02-2483-01  \\\n",
       "patient_id                                           TCGA-02-2483   \n",
       "acronym                                                       GBM   \n",
       "disease                                   glioblastoma multiforme   \n",
       "age_diagnosed                                                  43   \n",
       "gender                                                       Male   \n",
       "race                                                        Asian   \n",
       "ajcc_stage                                                    NaN   \n",
       "clinical_stage                                                NaN   \n",
       "histological_type                 Untreated primary (de novo) GBM   \n",
       "histological_grade                                            NaN   \n",
       "initial_pathologic_dx_year                                   2008   \n",
       "menopause_status                                              NaN   \n",
       "birth_days_to                                              -15964   \n",
       "vital_status                                                Alive   \n",
       "tumor_status                                           WITH TUMOR   \n",
       "last_contact_days_to                                          466   \n",
       "death_days_to                                                 NaN   \n",
       "cause_of_death                                                NaN   \n",
       "new_tumor_event_type                                          NaN   \n",
       "new_tumor_event_site                                          NaN   \n",
       "new_tumor_event_site_other                                    NaN   \n",
       "days_recurrence_free                                          NaN   \n",
       "treatment_outcome_first_course                                NaN   \n",
       "margin_status                                                 NaN   \n",
       "residual_tumor                                                NaN   \n",
       "event_status                                                    0   \n",
       "event_days                                                    466   \n",
       "dead                                                            0   \n",
       "days_survived                                                 466   \n",
       "disease_specific_survival_status                                0   \n",
       "disease_specific_survival_days                                466   \n",
       "disease_free_interval_status                                  NaN   \n",
       "disease_free_interval_days                                    NaN   \n",
       "progression_free_interval_status                                0   \n",
       "progression_free_interval_days                                466   \n",
       "sample_type                                   Primary Solid Tumor   \n",
       "n_mutations                                                    47   \n",
       "recurred                                                        0   \n",
       "\n",
       "sample_id                                         TCGA-02-2485-01  \n",
       "patient_id                                           TCGA-02-2485  \n",
       "acronym                                                       GBM  \n",
       "disease                                   glioblastoma multiforme  \n",
       "age_diagnosed                                                  53  \n",
       "gender                                                       Male  \n",
       "race                                    Black Or African American  \n",
       "ajcc_stage                                                    NaN  \n",
       "clinical_stage                                                NaN  \n",
       "histological_type                 Untreated primary (de novo) GBM  \n",
       "histological_grade                                            NaN  \n",
       "initial_pathologic_dx_year                                   2009  \n",
       "menopause_status                                              NaN  \n",
       "birth_days_to                                              -19494  \n",
       "vital_status                                                Alive  \n",
       "tumor_status                                           WITH TUMOR  \n",
       "last_contact_days_to                                          470  \n",
       "death_days_to                                                 NaN  \n",
       "cause_of_death                                                NaN  \n",
       "new_tumor_event_type                       Progression of Disease  \n",
       "new_tumor_event_site                                          NaN  \n",
       "new_tumor_event_site_other                                    NaN  \n",
       "days_recurrence_free                                          186  \n",
       "treatment_outcome_first_course                                NaN  \n",
       "margin_status                                                 NaN  \n",
       "residual_tumor                                                NaN  \n",
       "event_status                                                    0  \n",
       "event_days                                                    470  \n",
       "dead                                                            0  \n",
       "days_survived                                                 470  \n",
       "disease_specific_survival_status                                0  \n",
       "disease_specific_survival_days                                470  \n",
       "disease_free_interval_status                                  NaN  \n",
       "disease_free_interval_days                                    NaN  \n",
       "progression_free_interval_status                                1  \n",
       "progression_free_interval_days                                186  \n",
       "sample_type                                   Primary Solid Tumor  \n",
       "n_mutations                                                    56  \n",
       "recurred                                                        1  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = os.path.join('data', 'samples.tsv')\n",
    "\n",
    "# Protect against int columns with missing values getting converted to floats\n",
    "dtypes = {\n",
    "    'dead': str,\n",
    "    'recurred': str,\n",
    "}\n",
    "covariates_df = pd.read_table(path, index_col=0, dtype=dtypes)\n",
    "covariates_df['recurred'] = covariates_df.days_recurrence_free.notnull().astype(int)\n",
    "covariates_df.head(4).transpose()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's get a sense of how much missing data there is."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of samples: 8,756\n",
      "Number of nulls in each column:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "patient_id                             0\n",
       "acronym                                0\n",
       "disease                                0\n",
       "age_diagnosed                         50\n",
       "gender                                 0\n",
       "race                                 646\n",
       "ajcc_stage                          2716\n",
       "clinical_stage                      6878\n",
       "histological_type                    469\n",
       "histological_grade                  5103\n",
       "initial_pathologic_dx_year           122\n",
       "menopause_status                    7319\n",
       "birth_days_to                        105\n",
       "vital_status                           1\n",
       "tumor_status                         556\n",
       "last_contact_days_to                2322\n",
       "death_days_to                       6292\n",
       "cause_of_death                      8549\n",
       "new_tumor_event_type                6928\n",
       "new_tumor_event_site                7515\n",
       "new_tumor_event_site_other          8293\n",
       "days_recurrence_free                6299\n",
       "treatment_outcome_first_course      3889\n",
       "margin_status                       7815\n",
       "residual_tumor                      7612\n",
       "event_status                           7\n",
       "event_days                            45\n",
       "dead                                   7\n",
       "days_survived                         45\n",
       "disease_specific_survival_status     291\n",
       "disease_specific_survival_days        45\n",
       "disease_free_interval_status        4175\n",
       "disease_free_interval_days          4184\n",
       "progression_free_interval_status       7\n",
       "progression_free_interval_days        46\n",
       "sample_type                            0\n",
       "n_mutations                            0\n",
       "recurred                               0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Total number of samples: {:,}'.format(len(covariates_df)))\n",
    "print('Number of nulls in each column:')\n",
    "covariates_df.isnull().sum(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "acronym     32\n",
       "gender       2\n",
       "dead         2\n",
       "recurred     2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify which variables are categorical\n",
    "categorical_variables = ['acronym', 'gender', 'dead', 'recurred']\n",
    "\n",
    "# Number of categories per categorical variable\n",
    "covariates_df[categorical_variables].apply(lambda x: x.nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acronym</th>\n",
       "      <th>gender</th>\n",
       "      <th>dead</th>\n",
       "      <th>recurred</th>\n",
       "      <th>age_diagnosed</th>\n",
       "      <th>days_survived</th>\n",
       "      <th>days_recurrence_free</th>\n",
       "      <th>n_mutations</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0047-01</th>\n",
       "      <td>GBM</td>\n",
       "      <td>Male</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>78.0</td>\n",
       "      <td>448.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0055-01</th>\n",
       "      <td>GBM</td>\n",
       "      <td>Female</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>62.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                acronym  gender dead  recurred  age_diagnosed  days_survived  \\\n",
       "sample_id                                                                      \n",
       "TCGA-02-0047-01     GBM    Male    1         1           78.0          448.0   \n",
       "TCGA-02-0055-01     GBM  Female    1         1           62.0           76.0   \n",
       "\n",
       "                 days_recurrence_free  n_mutations  \n",
       "sample_id                                           \n",
       "TCGA-02-0047-01                  57.0           64  \n",
       "TCGA-02-0055-01                   6.0           51  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "continuous_variables = ['age_diagnosed', 'days_survived', 'days_recurrence_free', 'n_mutations']\n",
    "covariates_df = covariates_df[categorical_variables + continuous_variables]\n",
    "covariates_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before encoding, we're going to use the disease categories below. So let's store them for later use."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inspecting the head of the covariates DataFrame above, we see that two columns, namely <code>dead</code> and <code>recurred</code>, need some attention. We're going to encode categorical variables using panda's get_dummies. Since these columns are indicated by a 1 or 0, this will become the column header when encoded, as below."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's rename the values in each of these so that they more accurately reflect the underlying variable."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age_diagnosed</th>\n",
       "      <th>days_survived</th>\n",
       "      <th>days_recurrence_free</th>\n",
       "      <th>n_mutations</th>\n",
       "      <th>acronym_ACC</th>\n",
       "      <th>acronym_BLCA</th>\n",
       "      <th>acronym_BRCA</th>\n",
       "      <th>acronym_CESC</th>\n",
       "      <th>acronym_CHOL</th>\n",
       "      <th>acronym_COAD</th>\n",
       "      <th>...</th>\n",
       "      <th>acronym_THYM</th>\n",
       "      <th>acronym_UCEC</th>\n",
       "      <th>acronym_UCS</th>\n",
       "      <th>acronym_UVM</th>\n",
       "      <th>female</th>\n",
       "      <th>male</th>\n",
       "      <th>alive</th>\n",
       "      <th>dead</th>\n",
       "      <th>has_not_recurred</th>\n",
       "      <th>has_recurred</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0047-01</th>\n",
       "      <td>78.0</td>\n",
       "      <td>448.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>64</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0055-01</th>\n",
       "      <td>62.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>51</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 age_diagnosed  days_survived  days_recurrence_free  \\\n",
       "sample_id                                                             \n",
       "TCGA-02-0047-01           78.0          448.0                  57.0   \n",
       "TCGA-02-0055-01           62.0           76.0                   6.0   \n",
       "\n",
       "                 n_mutations  acronym_ACC  acronym_BLCA  acronym_BRCA  \\\n",
       "sample_id                                                               \n",
       "TCGA-02-0047-01           64          0.0           0.0           0.0   \n",
       "TCGA-02-0055-01           51          0.0           0.0           0.0   \n",
       "\n",
       "                 acronym_CESC  acronym_CHOL  acronym_COAD      ...       \\\n",
       "sample_id                                                      ...        \n",
       "TCGA-02-0047-01           0.0           0.0           0.0      ...        \n",
       "TCGA-02-0055-01           0.0           0.0           0.0      ...        \n",
       "\n",
       "                 acronym_THYM  acronym_UCEC  acronym_UCS  acronym_UVM  female  \\\n",
       "sample_id                                                                       \n",
       "TCGA-02-0047-01           0.0           0.0          0.0          0.0     0.0   \n",
       "TCGA-02-0055-01           0.0           0.0          0.0          0.0     1.0   \n",
       "\n",
       "                 male  alive  dead  has_not_recurred  has_recurred  \n",
       "sample_id                                                           \n",
       "TCGA-02-0047-01   1.0    0.0   1.0               0.0           1.0  \n",
       "TCGA-02-0055-01   0.0    0.0   1.0               0.0           1.0  \n",
       "\n",
       "[2 rows x 42 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "renamer = {\n",
    "    'dead_0': 'alive',\n",
    "    'dead_1': 'dead',\n",
    "    'recurred_0': 'has_not_recurred',\n",
    "    'recurred_1': 'has_recurred',\n",
    "    'gender_Female': 'female',\n",
    "    'gender_Male': 'male',\n",
    "}\n",
    "covariates_df = pd.get_dummies(covariates_df, columns=categorical_variables).rename(columns=renamer)\n",
    "covariates_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now the column name more accurately reflects the underlying variable. The categorical values are encoded as numeric data that can be input to the types of classifiers that we have been using.\n",
    "\n",
    "Another useful covariate will be the logarithm plus one function of the number mutations that was calculated in the aforementioned notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age_diagnosed</th>\n",
       "      <th>days_survived</th>\n",
       "      <th>days_recurrence_free</th>\n",
       "      <th>acronym_ACC</th>\n",
       "      <th>acronym_BLCA</th>\n",
       "      <th>acronym_BRCA</th>\n",
       "      <th>acronym_CESC</th>\n",
       "      <th>acronym_CHOL</th>\n",
       "      <th>acronym_COAD</th>\n",
       "      <th>acronym_DLBC</th>\n",
       "      <th>...</th>\n",
       "      <th>acronym_UCEC</th>\n",
       "      <th>acronym_UCS</th>\n",
       "      <th>acronym_UVM</th>\n",
       "      <th>female</th>\n",
       "      <th>male</th>\n",
       "      <th>alive</th>\n",
       "      <th>dead</th>\n",
       "      <th>has_not_recurred</th>\n",
       "      <th>has_recurred</th>\n",
       "      <th>n_mutations_log1p</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0047-01</th>\n",
       "      <td>78.0</td>\n",
       "      <td>448.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.174387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TCGA-02-0055-01</th>\n",
       "      <td>62.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.951244</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 age_diagnosed  days_survived  days_recurrence_free  \\\n",
       "sample_id                                                             \n",
       "TCGA-02-0047-01           78.0          448.0                  57.0   \n",
       "TCGA-02-0055-01           62.0           76.0                   6.0   \n",
       "\n",
       "                 acronym_ACC  acronym_BLCA  acronym_BRCA  acronym_CESC  \\\n",
       "sample_id                                                                \n",
       "TCGA-02-0047-01          0.0           0.0           0.0           0.0   \n",
       "TCGA-02-0055-01          0.0           0.0           0.0           0.0   \n",
       "\n",
       "                 acronym_CHOL  acronym_COAD  acronym_DLBC        ...          \\\n",
       "sample_id                                                        ...           \n",
       "TCGA-02-0047-01           0.0           0.0           0.0        ...           \n",
       "TCGA-02-0055-01           0.0           0.0           0.0        ...           \n",
       "\n",
       "                 acronym_UCEC  acronym_UCS  acronym_UVM  female  male  alive  \\\n",
       "sample_id                                                                      \n",
       "TCGA-02-0047-01           0.0          0.0          0.0     0.0   1.0    0.0   \n",
       "TCGA-02-0055-01           0.0          0.0          0.0     1.0   0.0    0.0   \n",
       "\n",
       "                 dead  has_not_recurred  has_recurred  n_mutations_log1p  \n",
       "sample_id                                                                 \n",
       "TCGA-02-0047-01   1.0               0.0           1.0           4.174387  \n",
       "TCGA-02-0055-01   1.0               0.0           1.0           3.951244  \n",
       "\n",
       "[2 rows x 42 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "covariates_df['n_mutations_log1p'] = np.log1p(covariates_df.pop('n_mutations'))\n",
    "covariates_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, let's save this to a <code>.tsv</code> file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = os.path.join('data', 'covariates.tsv')\n",
    "covariates_df.to_csv(path, sep='\\t', float_format='%.5g')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:cognoma-cancer-data]",
   "language": "python",
   "name": "conda-env-cognoma-cancer-data-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}