evaluate_model_on_test_set.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rcParams\n",
    "import seaborn as sns\n",
    "sns.set(font_scale = 2)\n",
    "sns.set_style(\"white\")\n",
    "rcParams['svg.fonttype'] = 'none'\n",
    "\n",
    "from scipy.stats import pearsonr,spearmanr\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "regression_models = {\"LCL\":              load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_lcl/model_11_hg19_tf.hdf5\", compile=False),\n",
    "                     \"Microglia\":        load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_gosselin_microglia/model_7_tf.hdf5\", compile=False),\n",
    "                     \"Neuron\":           load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_fullard_neun/model_2_tf.hdf5\", compile=False),\n",
    "                     \"Monocyte\":         load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_monocyte/model_8_tf.hdf5\", compile=False),\n",
    "                     \"mono_scatac\":      load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/scatac_snigdha_models_tf_versions/mono_subset_model_tf.hdf5\", compile=False),\n",
    "                     \"dendritic_scatac\": load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/scatac_snigdha_models_tf_versions/dendritic_subset_model_tf.hdf5\", compile=False),\n",
    "                     \"nkCells_scatac\":   load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/scatac_snigdha_models_tf_versions/nkCells_subset_model_tf.hdf5\", compile=False),\n",
    "                     \"basophil_scatac\":  load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/scatac_snigdha_models_tf_versions/basophil_subset_model_tf.hdf5\", compile=False),\n",
    "                     \"progen_scatac\":    load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/scatac_snigdha_models_tf_versions/progen_subset_model_tf.hdf5\", compile=False),\n",
    "                     \"cd4_scatac\":       load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/scatac_snigdha_models_tf_versions/cd4_subset_model_tf.hdf5\", compile=False),\n",
    "                     \"cd8_scatac\":       load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/scatac_snigdha_models_tf_versions/cd8_subset_model_tf.hdf5\", compile=False),\n",
    "                     \"B_cells_scatac\":   load_model(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/scatac_snigdha_models_tf_versions/B_cells_subset_model_tf.hdf5\", compile=False)\n",
    "                    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_1 (Conv1D)            (None, 493, 500)          16500     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 123, 500)          0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 123, 500)          0         \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, 116, 250)          1000250   \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 29, 250)           0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 29, 250)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_3 (Conv1D)            (None, 22, 100)           200100    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1 (None, 5, 100)            0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 5, 100)            0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 501       \n",
      "=================================================================\n",
      "Total params: 1,217,351\n",
      "Trainable params: 1,217,351\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "regression_models[\"mono_scatac\"].summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_1 (Conv1D)            (None, 493, 500)          16500     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_1 (MaxPooling1 (None, 123, 500)          0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 123, 500)          0         \n",
      "_________________________________________________________________\n",
      "conv1d_2 (Conv1D)            (None, 116, 250)          1000250   \n",
      "_________________________________________________________________\n",
      "max_pooling1d_2 (MaxPooling1 (None, 29, 250)           0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 29, 250)           0         \n",
      "_________________________________________________________________\n",
      "conv1d_3 (Conv1D)            (None, 22, 100)           200100    \n",
      "_________________________________________________________________\n",
      "max_pooling1d_3 (MaxPooling1 (None, 5, 100)            0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 5, 100)            0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 500)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 501       \n",
      "=================================================================\n",
      "Total params: 1,217,351\n",
      "Trainable params: 1,217,351\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "regression_models[\"mono_scatac\"].summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'name': 'conv1d_1', 'trainable': True, 'batch_input_shape': (None, 500, 4), 'dtype': 'float32', 'filters': 500, 'kernel_size': (8,), 'strides': (1,), 'padding': 'valid', 'data_format': 'channels_last', 'dilation_rate': (1,), 'groups': 1, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'class_name': 'VarianceScaling', 'config': {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'uniform', 'seed': None}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {}}, 'kernel_regularizer': {'class_name': 'L1L2', 'config': {'l1': 0.0, 'l2': 0.0010000000474974513}}, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}\n",
      "{'name': 'max_pooling1d_1', 'trainable': True, 'dtype': 'float32', 'strides': (4,), 'pool_size': (4,), 'padding': 'valid', 'data_format': 'channels_last'}\n",
      "{'name': 'dropout_1', 'trainable': True, 'dtype': 'float32', 'rate': 0.25, 'noise_shape': None, 'seed': None}\n",
      "{'name': 'conv1d_2', 'trainable': True, 'dtype': 'float32', 'filters': 250, 'kernel_size': (8,), 'strides': (1,), 'padding': 'valid', 'data_format': 'channels_last', 'dilation_rate': (1,), 'groups': 1, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'class_name': 'VarianceScaling', 'config': {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'uniform', 'seed': None}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {}}, 'kernel_regularizer': {'class_name': 'L1L2', 'config': {'l1': 0.0, 'l2': 0.009999999776482582}}, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}\n",
      "{'name': 'max_pooling1d_2', 'trainable': True, 'dtype': 'float32', 'strides': (4,), 'pool_size': (4,), 'padding': 'valid', 'data_format': 'channels_last'}\n",
      "{'name': 'dropout_2', 'trainable': True, 'dtype': 'float32', 'rate': 0.25, 'noise_shape': None, 'seed': None}\n",
      "{'name': 'conv1d_3', 'trainable': True, 'dtype': 'float32', 'filters': 100, 'kernel_size': (8,), 'strides': (1,), 'padding': 'valid', 'data_format': 'channels_last', 'dilation_rate': (1,), 'groups': 1, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'class_name': 'VarianceScaling', 'config': {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'uniform', 'seed': None}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {}}, 'kernel_regularizer': {'class_name': 'L1L2', 'config': {'l1': 0.0, 'l2': 0.009999999776482582}}, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}\n",
      "{'name': 'max_pooling1d_3', 'trainable': True, 'dtype': 'float32', 'strides': (4,), 'pool_size': (4,), 'padding': 'valid', 'data_format': 'channels_last'}\n",
      "{'name': 'dropout_3', 'trainable': True, 'dtype': 'float32', 'rate': 0.25, 'noise_shape': None, 'seed': None}\n",
      "{'name': 'flatten_1', 'trainable': True, 'dtype': 'float32', 'data_format': 'channels_last'}\n",
      "{'name': 'dense_1', 'trainable': True, 'dtype': 'float32', 'units': 1, 'activation': 'relu', 'use_bias': True, 'kernel_initializer': {'class_name': 'VarianceScaling', 'config': {'scale': 1.0, 'mode': 'fan_avg', 'distribution': 'uniform', 'seed': None}}, 'bias_initializer': {'class_name': 'Zeros', 'config': {}}, 'kernel_regularizer': {'class_name': 'L1L2', 'config': {'l1': 0.0, 'l2': 0.009999999776482582}}, 'bias_regularizer': None, 'activity_regularizer': None, 'kernel_constraint': None, 'bias_constraint': None}\n"
     ]
    }
   ],
   "source": [
    "for layer in regression_models[\"mono_scatac\"].layers:\n",
    "    print(layer.get_config())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_sets_regression = {\"LCL\":              np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_lcl/chr4_summit_centered_validation_set_hg19_X.npy\"),          \n",
    "                              \"Microglia\":        np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_gosselin_microglia/chr4_summit_centered_validation_set_X.npy\"),\n",
    "                              \"Neuron\":           np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_fullard_neun/chr4_summit_centered_validation_set_X.npy\"),\n",
    "                              \"Monocyte\":         np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_monocyte/chr4_summit_centered_validation_set_X.npy\"),\n",
    "                              \"mono_scatac\":      np.load(\"/home/snigdhaa/diff_acess/Archr/mono/subsetPeaks_validationInput.npy\"),\n",
    "                              \"dendritic_scatac\": np.load(\"/home/snigdhaa/diff_acess/Archr/dendritic/subsetPeaks_validationInput.npy\"),\n",
    "                              \"nkCells_scatac\":   np.load(\"/home/snigdhaa/diff_acess/Archr/nkCells/subsetPeaks_validationInput.npy\"),\n",
    "                              \"basophil_scatac\":  np.load(\"/home/snigdhaa/diff_acess/Archr/basophil/subsetPeaks_validationInput.npy\"),\n",
    "                              \"progen_scatac\":    np.load(\"/home/snigdhaa/diff_acess/Archr/progen/subsetPeaks_validationInput.npy\"),\n",
    "                              \"cd4_scatac\":       np.load(\"/home/snigdhaa/diff_acess/Archr/cd4/subsetPeaks_validationInput.npy\"),\n",
    "                              \"cd8_scatac\":       np.load(\"/home/snigdhaa/diff_acess/Archr/cd8/subsetPeaks_validationInput.npy\"),\n",
    "                              \"B_cells_scatac\":   np.load(\"/home/snigdhaa/diff_acess/Archr/B_cells/subsetPeaks_validationInput.npy\")\n",
    "                             }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_sets_regression_labels = {\"LCL\":              np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_lcl/chr4_summit_centered_validation_set_hg19_Y.npy\"),          \n",
    "                                     \"Microglia\":        np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_gosselin_microglia/chr4_summit_centered_validation_set_Y.npy\"),\n",
    "                                     \"Neuron\":           np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_fullard_neun/chr4_summit_centered_validation_set_Y.npy\"),\n",
    "                                     \"Monocyte\":         np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_monocyte/chr4_summit_centered_validation_set_Y.npy\"),\n",
    "                                     \"mono_scatac\":      np.load(\"/home/snigdhaa/diff_acess/Archr/mono/subsetPeaks_validationLabels.npy\"),\n",
    "                                     \"dendritic_scatac\": np.load(\"/home/snigdhaa/diff_acess/Archr/dendritic/subsetPeaks_validationLabels.npy\"),\n",
    "                                     \"nkCells_scatac\":   np.load(\"/home/snigdhaa/diff_acess/Archr/nkCells/subsetPeaks_validationLabels.npy\"),\n",
    "                                     \"basophil_scatac\":  np.load(\"/home/snigdhaa/diff_acess/Archr/basophil/subsetPeaks_validationLabels.npy\"),\n",
    "                                     \"progen_scatac\":    np.load(\"/home/snigdhaa/diff_acess/Archr/progen/subsetPeaks_validationLabels.npy\"),\n",
    "                                     \"cd4_scatac\":       np.load(\"/home/snigdhaa/diff_acess/Archr/cd4/subsetPeaks_validationLabels.npy\"),\n",
    "                                     \"cd8_scatac\":       np.load(\"/home/snigdhaa/diff_acess/Archr/cd8/subsetPeaks_validationLabels.npy\"),\n",
    "                                     \"B_cells_scatac\":   np.load(\"/home/snigdhaa/diff_acess/Archr/B_cells/subsetPeaks_validationLabels.npy\")\n",
    "                                    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sets_regression = {\"LCL\":              np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_lcl/chr8_9_summit_centered_test_set_hg19_X.npy\"),          \n",
    "                        \"Microglia\":        np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_gosselin_microglia/chr8_9_summit_centered_test_set_X.npy\"),\n",
    "                        \"Neuron\":           np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_fullard_neun/chr8_9_summit_centered_test_set_X.npy\"),\n",
    "                        \"Monocyte\":         np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_monocyte/chr8_9_summit_centered_test_set_X.npy\"),\n",
    "                        \"mono_scatac\":      np.load(\"/home/snigdhaa/diff_acess/Archr/mono/subsetPeaks_testInput.npy\"),\n",
    "                        \"dendritic_scatac\": np.load(\"/home/snigdhaa/diff_acess/Archr/dendritic/subsetPeaks_testInput.npy\"),\n",
    "                        \"nkCells_scatac\":   np.load(\"/home/snigdhaa/diff_acess/Archr/nkCells/subsetPeaks_testInput.npy\"),\n",
    "                        \"basophil_scatac\":  np.load(\"/home/snigdhaa/diff_acess/Archr/basophil/subsetPeaks_testInput.npy\"),\n",
    "                        \"progen_scatac\":    np.load(\"/home/snigdhaa/diff_acess/Archr/progen/subsetPeaks_testInput.npy\"),\n",
    "                        \"cd4_scatac\":       np.load(\"/home/snigdhaa/diff_acess/Archr/cd4/subsetPeaks_testInput.npy\"),\n",
    "                        \"cd8_scatac\":       np.load(\"/home/snigdhaa/diff_acess/Archr/cd8/subsetPeaks_testInput.npy\"),\n",
    "                        \"B_cells_scatac\":   np.load(\"/home/snigdhaa/diff_acess/Archr/B_cells/subsetPeaks_testInput.npy\")\n",
    "                        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_sets_regression_labels = {\"LCL\":              np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_lcl/chr8_9_summit_centered_test_set_hg19_Y.npy\"),          \n",
    "                               \"Microglia\":        np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_gosselin_microglia/chr8_9_summit_centered_test_set_Y.npy\"),\n",
    "                               \"Neuron\":           np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_fullard_neun/chr8_9_summit_centered_test_set_Y.npy\"),\n",
    "                               \"Monocyte\":         np.load(\"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_monocyte/chr8_9_summit_centered_test_set_Y.npy\"),\n",
    "                               \"mono_scatac\":      np.load(\"/home/snigdhaa/diff_acess/Archr/mono/subsetPeaks_testLabels.npy\"),\n",
    "                               \"dendritic_scatac\": np.load(\"/home/snigdhaa/diff_acess/Archr/dendritic/subsetPeaks_testLabels.npy\"),\n",
    "                               \"nkCells_scatac\":   np.load(\"/home/snigdhaa/diff_acess/Archr/nkCells/subsetPeaks_testLabels.npy\"),\n",
    "                               \"basophil_scatac\":  np.load(\"/home/snigdhaa/diff_acess/Archr/basophil/subsetPeaks_testLabels.npy\"),\n",
    "                               \"progen_scatac\":    np.load(\"/home/snigdhaa/diff_acess/Archr/progen/subsetPeaks_testLabels.npy\"),\n",
    "                               \"cd4_scatac\":       np.load(\"/home/snigdhaa/diff_acess/Archr/cd4/subsetPeaks_testLabels.npy\"),\n",
    "                               \"cd8_scatac\":       np.load(\"/home/snigdhaa/diff_acess/Archr/cd8/subsetPeaks_testLabels.npy\"),\n",
    "                               \"B_cells_scatac\":   np.load(\"/home/snigdhaa/diff_acess/Archr/B_cells/subsetPeaks_testLabels.npy\")\n",
    "                               }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_sets_predictions = {}\n",
    "test_sets_predictions = {}\n",
    "\n",
    "for cell_type in test_sets_regression:\n",
    "    validation_sets_predictions[cell_type] = regression_models[cell_type].predict(validation_sets_regression[cell_type])\n",
    "    test_sets_predictions[cell_type] = regression_models[cell_type].predict(test_sets_regression[cell_type])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "bulk_ocr_cell_type_names = {\"LCL\",\n",
    "                            \"Microglia\",\n",
    "                            \"Neuron\",\n",
    "                            \"Monocyte\"\n",
    "                           }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "replicateCorrelationInFiles = {\"LCL\": \"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/lcl_dnase_encode/atac/5840747c-53d2-4b55-8091-72684bd9fc7f/call-idr/shard-0/execution/rep1_rep2.idr0.05.thresholded-peaks.bfilt.txt.gz\",\n",
    "           \"Monocyte\": \"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_human_monocyte_brain/furtherProcessedPeaks/humanMonocyteDHS/peak/macs2/idr/pooled_pseudo_reps/humanMonocyteDHS_ppr.IDR0.1.thresholded-peaks.bfilt.txt.gz\",\n",
    "           \"Microglia\": \"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/gosselin_microglia_atac/exvivo_idr_pipeline_out/atac/0c4b0010-a158-4176-a424-059a4fb12cef/call-idr_ppr/execution/ppr.idr0.05.thresholded-peaks.bfilt.txt.gz\",\n",
    "           \"Neuron\": \"/projects/pfenninggroup/machineLearningForComputationalBiology/eramamur_stuff/ml_fullard_neun/ppr.idr0.1.thresholded-peaks.bfilt.txt.gz\"\n",
    "          }\n",
    "\n",
    "repNotPpr = {\"LCL\": True,\n",
    "             \"Monocyte\": False,\n",
    "             \"Microglia\": False,\n",
    "             \"Neuron\": False\n",
    "            }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotRegressionModelPerformance(true_signal, predicted_signal, cell_type, label):\n",
    "    p = sns.jointplot(x=true_signal.flatten(),\n",
    "                      y=predicted_signal.flatten(),\n",
    "                      kind='hex',\n",
    "                      color='#1f77b4',\n",
    "                      #rasterized=True\n",
    "                     ).plot_joint(sns.kdeplot,\n",
    "                                  color='#1f77b4',\n",
    "                                  linewidths=0.5\n",
    "                                 )\n",
    "    \n",
    "    p.ax_marg_x.remove()\n",
    "    p.ax_marg_y.remove()\n",
    "    \n",
    "    print(true_signal.shape)\n",
    "    r, rp = pearsonr(true_signal.flatten(), predicted_signal.flatten())\n",
    "    rho, rhop = spearmanr(true_signal.flatten(), predicted_signal.flatten())\n",
    "    \n",
    "    p.set_axis_labels(\"Signal value\", \"Predicted signal\")\n",
    "    \n",
    "    p.ax_joint.annotate(\"r={:0.2}, p={:0.2}\".format(r,rp)+ \"\\n\" + r\"$\\rho$={:0.2}, p={:0.2}\".format(rho, rhop),\n",
    "                        xy=(0.5,0.1),\n",
    "                        xycoords=\"axes fraction\",\n",
    "                        ha=\"left\",\n",
    "                        va=\"center\",\n",
    "                        bbox={'fc': 'none', 'ec': 'none'},\n",
    "                        color='red',\n",
    "                        fontsize=15,\n",
    "                        weight='bold'\n",
    "                       )\n",
    "\n",
    "    plt.title(\" \".join([cell_type, label, \"set\"]))\n",
    "    plt.savefig(\"_\".join([cell_type,\"regression\",label,\"performance\"])+\".svg\")\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotReplicateCorrelations(inFiles, repNotPpr, cell_type, label):\n",
    "    inFile = inFiles[cell_type]\n",
    "    replicate_data = pd.read_csv(inFile,\n",
    "                                 sep=\"\\t\", compression=\"gzip\",\n",
    "                                 names=[\"CHR\",\n",
    "                                 \"START\",\n",
    "                                 \"END\",\n",
    "                                 \"NAME\",\n",
    "                                 \"SCORE\",\n",
    "                                 \"STRAND\",\n",
    "                                 \"SIGNAL\",\n",
    "                                 \"P\",\n",
    "                                 \"Q\",\n",
    "                                 \"SUMMIT\",\n",
    "                                 \"LOCALIDR\",\n",
    "                                 \"GLOBALIDR\",\n",
    "                                 \"REP1_START\",\n",
    "                                 \"REP1_END\",\n",
    "                                 \"REP1_SIGNAL\",\n",
    "                                 \"REP1_SUMMIT\",\n",
    "                                 \"REP2_START\",\n",
    "                                 \"REP2_END\",\n",
    "                                 \"REP2_SIGNAL\",\n",
    "                                 \"REP2_SUMMIT\"]\n",
    "                                )\n",
    "    \n",
    "    labelToChrMapping = {\"validation\": [\"chr4\"],\n",
    "                         \"test\": [\"chr8\", \"chr9\"],\n",
    "                        }\n",
    "\n",
    "    \n",
    "    \n",
    "    subsetted_replicate_data = replicate_data.loc[replicate_data[\"CHR\"].isin(labelToChrMapping[label])]    \n",
    "    print(subsetted_replicate_data.shape[0])\n",
    "    addStr = \"ppr\"\n",
    "    if repNotPpr[cell_type]:\n",
    "        addStr = \"rep\"\n",
    "\n",
    "        \n",
    "    p = sns.jointplot(x=subsetted_replicate_data[\"REP1_SIGNAL\"],\n",
    "                      y=subsetted_replicate_data[\"REP2_SIGNAL\"],\n",
    "                      kind='hex',\n",
    "                      color='#1f77b4',\n",
    "                     ).plot_joint(sns.kdeplot,\n",
    "                                  color='#1f77b4',\n",
    "                                  linewidths=0.5\n",
    "                                 )\n",
    "\n",
    "    p.ax_marg_x.remove()\n",
    "    p.ax_marg_y.remove()\n",
    "    \n",
    "    r, rp = pearsonr(subsetted_replicate_data[\"REP1_SIGNAL\"], subsetted_replicate_data[\"REP2_SIGNAL\"])\n",
    "    rho, rhop = spearmanr(subsetted_replicate_data[\"REP1_SIGNAL\"], subsetted_replicate_data[\"REP2_SIGNAL\"])\n",
    "    \n",
    "    p.ax_joint.annotate(\"r={:0.2}, p={:0.2}\".format(r,rp)+ \"\\n\" + r\"$\\rho$={:0.2}, p={:0.2}\".format(rho, rhop),\n",
    "                        xy=(0.5,0.1),\n",
    "                        xycoords=\"axes fraction\",\n",
    "                        ha=\"left\",\n",
    "                        va=\"center\",\n",
    "                        bbox={'fc': 'none', 'ec': 'none'},\n",
    "                        color='red',\n",
    "                        fontsize=15,\n",
    "                        weight='bold'\n",
    "                       )    \n",
    "    p.set_axis_labels(\"Signal (\"+cell_type+\" \"+addStr+\"1)\", \"Signal (\"+cell_type+\" \"+addStr+\"2)\")\n",
    "    plt.title(\" \".join([cell_type, addStr, \"correlations\", label, \"set\"]))\n",
    "    plt.savefig(\"_\".join([cell_type,addStr+\"_correlations\",label])+\".svg\")\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3894, 1)\n",
      "3894\n",
      "(9196, 1)\n",
      "9196\n",
      "(8888, 1)\n",
      "8888\n",
      "(15982, 1)\n",
      "15982\n",
      "(13557, 1)\n",
      "13557\n",
      "(17003, 1)\n",
      "17003\n",
      "(4684, 1)\n",
      "4684\n",
      "(10513, 1)\n",
      "10513\n",
      "(29615, 1)\n",
      "(13700, 1)\n",
      "(33780, 1)\n",
      "(15362, 1)\n",
      "(17673, 1)\n",
      "(7822, 1)\n",
      "(12601, 1)\n",
      "(5422, 1)\n",
      "(36164, 1)\n",
      "(16481, 1)\n",
      "(24603, 1)\n",
      "(10709, 1)\n",
      "(22980, 1)\n",
      "(9906, 1)\n",
      "(20894, 1)\n",
      "(9643, 1)\n"
     ]
    }
   ],
   "source": [
    "for cell_type in test_sets_regression:\n",
    "    plotRegressionModelPerformance(validation_sets_predictions[cell_type],\n",
    "                                   validation_sets_regression_labels[cell_type],\n",
    "                                   cell_type,\n",
    "                                   \"validation\")\n",
    "    if cell_type in bulk_ocr_cell_type_names:\n",
    "        plotReplicateCorrelations(replicateCorrelationInFiles, repNotPpr, cell_type, \"validation\")    \n",
    "    \n",
    "    plotRegressionModelPerformance(test_sets_predictions[cell_type],\n",
    "                                   test_sets_regression_labels[cell_type],\n",
    "                                   cell_type,\n",
    "                                   \"test\")    \n",
    "\n",
    "    if cell_type in bulk_ocr_cell_type_names:\n",
    "        plotReplicateCorrelations(replicateCorrelationInFiles, repNotPpr, cell_type, \"test\")    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:keras_tf_new_mar_18_2021]",
   "language": "python",
   "name": "conda-env-keras_tf_new_mar_18_2021-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}