FullStudy_RepeatabilityStudy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Full PyRadiomics-mpReview Repeatability Study\n",
    "Analyse pyRadiomics features on mpReview prostate data for repeatability.\n",
    "\n",
    "Code cells in which you are epxected to provide user input/set parameters to customize the anylsis are headed by \"*__User Input__*\" (of course you are free to customize any part of the code, this is just to help you find the parameters which are most intersting for most users).\n",
    "\n",
    "NOTE: There are some asserts included in the code to make sure some basic aspects of the data handling turned out as expected. If any of these fail, smth most likely went wrong with the data processing (or some crucial thing about the data is different than expected when this code was written)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "# tell pandas the display width (more ore less), since it cannot autodetect in a browser, \n",
    "# lower this if it is too high for your display (and you encounter ugly wrapping effects)\n",
    "pd.set_option('display.width', 120)\n",
    "# Disable warnings about changes to a df copy not making it back to the original df\n",
    "pd.options.mode.chained_assignment = None  # default='warn'\n",
    "\n",
    "# This line tells the notebook to show plots inside of the notebook\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "import seaborn as sb\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### User Input - Set the CSV files containing the extracted pyRadiomics features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "BASE_FILE_NAMES = [#\"pyRadiomicsFeatures_FullStudySettings_2D_bin5\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_2D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_2D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_2D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_2D_bin40\",\n",
    "                   #\"pyRadiomicsFeatures_FullStudySettings_3D_bin5\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_3D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_3D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_3D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_3D_bin40\",\n",
    "                   #\"pyRadiomicsFeatures_FullStudySettings_noNormalization_2D_bin5\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_noNormalization_2D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_noNormalization_2D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_noNormalization_2D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_noNormalization_2D_bin40\",\n",
    "                   #\"pyRadiomicsFeatures_FullStudySettings_noNormalization_3D_bin5\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_noNormalization_3D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_noNormalization_3D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_noNormalization_3D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_FullStudySettings_noNormalization_3D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_2D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_2D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_2D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_2D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_noNormalization_2D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_noNormalization_2D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_noNormalization_2D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_noNormalization_2D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_3D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_3D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_3D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_3D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_noNormalization_3D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_noNormalization_3D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_noNormalization_3D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrectedT2AX_FullStudySettings_noNormalization_3D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_2D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_2D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_2D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_2D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_noNormalization_2D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_noNormalization_2D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_noNormalization_2D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_noNormalization_2D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_3D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_3D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_3D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_3D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_noNormalization_3D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_noNormalization_3D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_noNormalization_3D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+MuscleRefNormT2AX_FullStudySettings_noNormalization_3D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_2D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_2D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_2D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_2D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_noNormalization_2D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_noNormalization_2D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_noNormalization_2D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_noNormalization_2D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_3D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_3D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_3D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_3D_bin20\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_noNormalization_3D_bin40\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_noNormalization_3D_bin10\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_noNormalization_3D_bin15\",\n",
    "                   \"pyRadiomicsFeatures_biasCorrected+TP2RegisteredT2AX_FullStudySettings_noNormalization_3D_bin20\"]\n",
    "\n",
    "featuresFileNames = [os.path.join(os.getcwd(), \"EvalData\", b + \".csv\") for b in BASE_FILE_NAMES]\n",
    "#print(featuresFileNames)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the feature data and add some more derived meta information (which will be convenient later)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dfFeatures = None\n",
    "for i, fileName in enumerate(featuresFileNames):\n",
    "  dfImport = pd.read_csv(fileName, sep = \";\")\n",
    "  assert \"_bin\" in fileName\n",
    "  dfImport[\"binwidth\"] = int(fileName[fileName.find(\"_bin\")+4:].split(\".\")[0])\n",
    "  assert \"_3D\" in fileName or \"_2D\" in fileName\n",
    "  dfImport[\"dim\"] = 3 if \"_3D\" in fileName else 2\n",
    "  dfImport[\"normalized\"] = False if \"_noNormalization\" in fileName else True\n",
    "  baseImageType = \"ORIG\"\n",
    "  if \"_biasCorrectedT2AX\" in fileName:\n",
    "    baseImageType = \"BC\"\n",
    "  elif \"_biasCorrected+MuscleRefNormT2AX\" in fileName:\n",
    "    baseImageType = \"BC+MRN\"\n",
    "  elif \"_biasCorrected+TP2RegisteredT2AX\" in fileName:\n",
    "    baseImageType = \"BC+REG\"\n",
    "  dfImport[\"baseImageType\"] = baseImageType\n",
    "  assert (dfImport.shape[0] == 102) or (dfImport.shape[0] == 22) or (dfImport.shape[0] == 326)\n",
    "  if i == 0:\n",
    "    dfFeatures = dfImport\n",
    "  else:\n",
    "    dfFeatures = pd.concat([dfFeatures, dfImport], ignore_index=True)\n",
    "\n",
    "dfFeatures[\"series\"] = dfFeatures[\"series\"].astype(str)\n",
    "dfFeatures[\"date\"] = dfFeatures.apply(lambda row: row.study.split(\"_\")[1], axis = 1)\n",
    "dfFeatures[\"date\"] = pd.to_numeric(dfFeatures[\"date\"])\n",
    "dfFeatures[\"patient\"] = dfFeatures.apply(lambda row: row.study.split(\"_\")[0], axis = 1)\n",
    "\n",
    "#assert dfFeatures.shape[0] == len(featuresFileNames) * 102  #326\n",
    "#display(dfFeatures[(dfFeatures.patient == \"PCAMPMRI-00775\") & (dfFeatures.segmentedStructure == \"NormalROI_PZ_1\")])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merge the data in such a way that ascociated measurments for both timepoints for each patient are combined in one row with \"\\_t1\" indicating measurements from the earlier (first) timepoint and \"\\_t2\" indicating measurements from the later (second) timepoint."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "dfFeaturesJointTP = pd.merge(dfFeatures, dfFeatures, how = \"inner\", suffixes = (\"_t1\", \"_t2\"),\n",
    "                             on = [\"patient\", \"canonicalType\", \"segmentedStructure\", \"binwidth\", \"dim\", \"normalized\", \"baseImageType\"])\n",
    "dfFeaturesJointTP = dfFeaturesJointTP[(dfFeaturesJointTP.date_t1 < dfFeaturesJointTP.date_t2)]\n",
    "print(dfFeaturesJointTP.shape)\n",
    "\n",
    "# exclude PCAMPMRI-00775 (case 1) ADC1400, because the 2nd timepoint image is not ok\n",
    "dfFeaturesJointTP = dfFeaturesJointTP[~((dfFeaturesJointTP.patient == \"PCAMPMRI-00775\") &\n",
    "                                        (dfFeaturesJointTP.canonicalType == \"ADC1400\"))]\n",
    "\n",
    "#dfFeaturesJointTP.to_csv(\"D:/pyRadiomicsFeaturesJointTimepoints.csv\", sep = \";\", index = False)\n",
    "\n",
    "#assert dfFeaturesJointTP.shape[0] == len(featuresFileNames) * 48   #160\n",
    "\n",
    "#display(dfFeaturesJointTP[[\"series_t1\", \"series_t2\", \"patient\", \"canonicalType\", \"segmentedStructure\", \"binwidth\", \"dim\", \"normalized\"]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Compute Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "selectedFeatures = [c[0:-3] for c in dfFeaturesJointTP.columns if any(xc in c for xc in [\"original\", \n",
    "                                                                                         \"log-sigma\", \n",
    "                                                                                         \"wavelet\",\n",
    "                                                                                         \"square\",\n",
    "                                                                                         \"squareroot\",\n",
    "                                                                                         \"logarithm\",\n",
    "                                                                                         \"exponential\"\n",
    "                                                                                        ])]\n",
    "\n",
    "\n",
    "selectedFeatures = set(selectedFeatures)\n",
    "#selectedFeatures = [\"original_shape_Volume\"]\n",
    "#print(\"No. Selected Features:\", len(selectedFeatures), \"\\nSelected Features:\", selectedFeatures)\n",
    "\n",
    "# Select image types: choose from [\"ADC1400\", \"SUB\", \"T2AX\"]\n",
    "selectedTypes = [\"ADC1400\", \"SUB\", \"T2AX\"]\n",
    "\n",
    "# Select segmented structures: choose from [\"WholeGland\", \"PeripheralZone\", \"TumorROI_PZ_1\", \"NormalROI_PZ_1\"]\n",
    "selectedStructures = [\"WholeGland\", \"PeripheralZone\", \"TumorROI_PZ_1\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get metrics for selected data, add some meta data to the metrics and create a unsorted and a sorted DataFrame from the metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import statsUtils.metrics as sm\n",
    "\n",
    "metrics = []\n",
    "for ctype in selectedTypes:\n",
    "  for structure in selectedStructures:\n",
    "    for baseImageType in dfFeaturesJointTP[\"baseImageType\"].unique().tolist():\n",
    "      for binwidth in dfFeaturesJointTP[\"binwidth\"].unique().tolist():\n",
    "        for dim in dfFeaturesJointTP[\"dim\"].unique().tolist():\n",
    "          for normalized in dfFeaturesJointTP[\"normalized\"].unique().tolist():\n",
    "            dfSelection = dfFeaturesJointTP[(dfFeaturesJointTP.canonicalType == ctype) & \n",
    "                                            (dfFeaturesJointTP.segmentedStructure == structure) &\n",
    "                                            (dfFeaturesJointTP.baseImageType == baseImageType) &\n",
    "                                            (dfFeaturesJointTP.binwidth == binwidth) &\n",
    "                                            (dfFeaturesJointTP.dim == dim) &\n",
    "                                            (dfFeaturesJointTP.normalized == normalized)]\n",
    "            if dfSelection.shape[0] > 0:\n",
    "              for feature in selectedFeatures:\n",
    "                f1 = feature + \"_t1\"\n",
    "                f2 = feature + \"_t2\"\n",
    "                t1Data = dfSelection[f1]\n",
    "                t2Data = dfSelection[f2]\n",
    "                \n",
    "                assert t1Data.shape[0] == t2Data.shape[0]\n",
    "                expectedCases = 15\n",
    "                if structure == \"TumorROI_PZ_1\":\n",
    "                  expectedCases = 11\n",
    "                if ctype == \"ADC1400\":\n",
    "                  expectedCases -= 1\n",
    "                assert t1Data.shape[0] == expectedCases\n",
    "                \n",
    "                m = sm.getMetrics(t1Data, t2Data)\n",
    "                m[\"featureName\"] = feature\n",
    "                m[\"featureGroup\"] = feature.split(\"_\")[1]\n",
    "                m[\"featureBaseName\"] = feature.rsplit(\"_\", maxsplit = 1)[1]\n",
    "                m[\"featureGroupAndBaseName\"] = feature.split(\"_\", maxsplit = 1)[1]\n",
    "                m[\"structure\"] = structure\n",
    "                m[\"ctype\"] = ctype\n",
    "                m[\"filter\"] = feature.split(\"_\")[0]\n",
    "                filterGroup = (feature.split(\"_\")[0]).split(\"-\")[0]\n",
    "                if filterGroup not in [\"wavelet\", \"log\", \"original\"]:\n",
    "                  filterGroup = \"singlepix\"\n",
    "                if filterGroup == \"original\":\n",
    "                  filterGroup = \"a_original\"\n",
    "                m[\"filterGroup\"] = filterGroup\n",
    "                m[\"binwidth\"] = binwidth\n",
    "                m[\"dim\"] = dim\n",
    "                m[\"normalized\"] = normalized\n",
    "                m[\"baseImageType\"] = baseImageType\n",
    "                metrics.append(m)\n",
    "      \n",
    "dfMetrics = pd.DataFrame(metrics)\n",
    "#print(dfMetrics.sort_values(by = \"structure\")[[\"ctype\", \"structure\", \"icc\", \"rc\", \"rcp\", \"meanOfAbsDiff\", \"meanOfAbsDiffPercent\"]])\n",
    "#print(dfMetrics[\"filter\"].unique())\n",
    "\n",
    "dfSortedMetrics = dfMetrics.sort_values(by = \"icc\", ascending = False)\n",
    "#print(dfSortedMetrics.head(5))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## General Convenience Functions for Plotting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Small helper function to reduce a list to unique values while keeping the order of the elements (keep first)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def unique_keep_order(seq):\n",
    "  seen = set()\n",
    "  seen_add = seen.add  # this is for speed optimization, resolving a local variable is cheaper\n",
    "  return [x for x in seq if not (x in seen or seen_add(x))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A small helper class for mapping mapping keys to values in a user defined dictionary or returning the key if as value if the key is not defined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Mapper(object):\n",
    "  def __init__(self, customDict):\n",
    "    self._customDict = customDict\n",
    "\n",
    "  def lookup(self, key):\n",
    "    if key in self._customDict:\n",
    "      return self._customDict[key]\n",
    "    else:\n",
    "      return key\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simple map from indices to letters for subfigures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FIGURE_SUBIDX_MAP = {0: \"a\", 1: \"b\", 2: \"c\", 3: \"d\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bokeh Customizable Scatterplot\n",
    "This function defines a Bokeh Scatterplot that can be configured to encode different parameters in color and glyph shape.\n",
    "Currently only supports 1 source for color and 1 or 2 sources for glyph shape. Also currently limited to a maximum of 4 glyph shapes (i.e. this function is not super generalized yet, just enough for the purposes of this analysis, I should think about cleaning this up and puttin git into statsUtils)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from bokeh.models import ColumnDataSource, OpenURL, TapTool, Span\n",
    "from bokeh.plotting import figure, output_file, show, save, reset_output\n",
    "from bokeh.io import output_notebook, export_png\n",
    "from bokeh.colors import RGB\n",
    "from bokeh.palettes import Category20, Category10\n",
    "\n",
    "from bokeh.models import HoverTool, PanTool, WheelZoomTool, BoxZoomTool, ResetTool, TapTool\n",
    "\n",
    "def showBokehPlot(dfSelectedMetrics = None, plotWhat = \"icc\",\n",
    "                  glyphColorSrc = \"binwidth\", glyphShapeSrc = [\"dim\", \"normalized\"], \n",
    "                  colorLegendLookupDict = {}, shapeLegendLookupDict = {},\n",
    "                  outFilePrefix = \"\", outFileSuffix = \"\", writeToFile = False,\n",
    "                  includeFeatGroupNameInLegend = False,\n",
    "                  userSingleGlyphSelection = -1):\n",
    "  cMapper = Mapper(colorLegendLookupDict)\n",
    "  sMapper = Mapper(shapeLegendLookupDict)\n",
    "  if isinstance(glyphShapeSrc, str):\n",
    "    glyphShapeSrc = [glyphShapeSrc]\n",
    "  \n",
    "  reset_output()\n",
    "  output_notebook()\n",
    "\n",
    "  # add a unique color for each value of the designated source for the glyph color\n",
    "  colormap = {}\n",
    "  requiredNoColors = len(dfSelectedMetrics[glyphColorSrc].unique())\n",
    "  colorScheme = Category10[10]\n",
    "  if requiredNoColors > 10:\n",
    "    colorScheme = Category20[20]\n",
    "  for i, f in enumerate(np.sort(dfSelectedMetrics[glyphColorSrc].unique())):\n",
    "    colormap[f] = colorScheme[i]\n",
    "  dfSelectedMetrics[\"color\"] = dfSelectedMetrics.apply(lambda row: colormap[row[glyphColorSrc]], axis = 1)\n",
    "  \n",
    "  # add legend for glyphs\n",
    "  if len(glyphShapeSrc) == 1:\n",
    "    dfSelectedMetrics[\"legend\"] = dfSelectedMetrics.apply(lambda row: str(sMapper.lookup(row[glyphShapeSrc[0]])) + \" \" + \n",
    "                                                                      str(cMapper.lookup(row[glyphColorSrc])), \n",
    "                                                          axis = 1)\n",
    "  else:\n",
    "    dfSelectedMetrics[\"legend\"] = dfSelectedMetrics.apply(lambda row: str(sMapper.lookup(row[glyphShapeSrc[0]])) + \" \" + \n",
    "                                                                      str(sMapper.lookup(row[glyphShapeSrc[1]]))  + \" \" +\n",
    "                                                                      str(cMapper.lookup(row[glyphColorSrc])), \n",
    "                                                          axis = 1)\n",
    "\n",
    "  # declare data sources, we need one source for each glyph shape\n",
    "  sources = []\n",
    "  if len(glyphShapeSrc) == 1:  \n",
    "    shapeSource0Values = np.sort(dfSelectedMetrics[glyphShapeSrc[0]].unique()).tolist()\n",
    "    for ss0 in shapeSource0Values:\n",
    "      source = ColumnDataSource(data = dfSelectedMetrics[(dfSelectedMetrics[glyphShapeSrc[0]] == ss0)])\n",
    "      sources.append(source)\n",
    "  else:\n",
    "    shapeSource0Values = np.sort(dfSelectedMetrics[glyphShapeSrc[0]].unique()).tolist()\n",
    "    shapeSource1Values = np.sort(dfSelectedMetrics[glyphShapeSrc[1]].unique()).tolist()\n",
    "    for ss0 in shapeSource0Values:\n",
    "      for ss1 in shapeSource1Values:\n",
    "        source = ColumnDataSource(data = dfSelectedMetrics[(dfSelectedMetrics[glyphShapeSrc[0]] == ss0) &\n",
    "                                                           (dfSelectedMetrics[glyphShapeSrc[1]] == ss1)])\n",
    "        sources.append(source)\n",
    "    \n",
    "    \n",
    "  # create the plot\n",
    "  hover = HoverTool(tooltips = [(\"rc\", \"@rc\"),\n",
    "                                (\"rcp\", \"@rcp\"),\n",
    "                                (\"icc\", \"@icc\"),\n",
    "                                (\"madp\", \"@meanOfAbsDiffPercent\"),\n",
    "                                (\"mean\", \"@mean\"),\n",
    "                                (\"std\", \"@std\"),\n",
    "                                (\"type\", \"@ctype\"),\n",
    "                                (\"structure\", \"@structure\"),\n",
    "                                (\"filter\", \"@filter\"),\n",
    "                                (\"featName\", \"@featureName\")])\n",
    "  wZoom = WheelZoomTool()\n",
    "  bZoom = BoxZoomTool()\n",
    "  reset = ResetTool()\n",
    "  pan = PanTool()\n",
    "  \n",
    "  titleStr = \"Legend Code: \" + \" \".join(glyphShapeSrc) + \" \" + glyphColorSrc + \"     (\" + outFileSuffix + \")\"\n",
    "  \n",
    "  # this is a \"trick\" to get only the base feature names as y tick labels, but keep the feature groups together\n",
    "  yRange = np.sort(dfSelectedMetrics[\"featureGroupAndBaseName\"].unique()).tolist()\n",
    "  if not includeFeatGroupNameInLegend:\n",
    "    yRange = [yl.split(\"_\")[-1] for yl in yRange]\n",
    "  \n",
    "  p = figure(x_range = [-0.6, 1], #x_range=[min(-1, np.min(dfSelectedMetrics[plotWhat].values)),max(1, np.max(dfSelectedMetrics[plotWhat].values))], \n",
    "             y_range = yRange, \n",
    "             x_axis_label = plotWhat.upper(),\n",
    "             y_axis_label = \"Feature Name\",\n",
    "             tools = [hover, wZoom, bZoom, reset, pan], \n",
    "             title = None,#titleStr, \n",
    "             plot_width=1200, plot_height=(145 + (26 * len(yRange))),\n",
    "             min_border_left = 255\n",
    "            )\n",
    "    \n",
    "  # add a reference line where the volume ICC is\n",
    "  volumeIcc = Span(location=dfSelectedMetrics[(dfSelectedMetrics[\"featureBaseName\"] == \"Volume\")][plotWhat].tolist()[0],\n",
    "                   dimension='height', line_color='gray',\n",
    "                   line_dash='dashed', line_width=2)\n",
    "  p.add_layout(volumeIcc)\n",
    "  \n",
    "  if includeFeatGroupNameInLegend:\n",
    "    yLabel = 'featureGroupAndBaseName'\n",
    "  else:\n",
    "    yLabel = 'featureBaseName'\n",
    "  \n",
    "  if (userSingleGlyphSelection < 0) or (len(sources) > 1):\n",
    "    if len(sources) > 1:\n",
    "      p.square(plotWhat, yLabel, source = sources[1], color = \"color\", legend = \"legend\", size = 18, line_width = 2, fill_alpha = 0.5)\n",
    "    if len(sources) > 0:\n",
    "      p.asterisk(plotWhat, yLabel, source = sources[0], color = \"color\", legend = \"legend\", size = 22, line_width = 2)\n",
    "    if len(sources) > 2:  \n",
    "      p.diamond(plotWhat, yLabel, source = sources[2], color = \"color\", legend = \"legend\", size = 22, line_width = 2, fill_alpha = 0.5)\n",
    "    if len(sources) > 3:  \n",
    "      p.circle(plotWhat, yLabel, source = sources[3], color = \"color\", legend = \"legend\", size = 18, line_width = 2, fill_alpha = 0.5)\n",
    "  else:\n",
    "    if userSingleGlyphSelection == 1:\n",
    "      p.square(plotWhat, yLabel, source = sources[0], color = \"color\", legend = \"legend\", size = 18, line_width = 2, fill_alpha = 0.5)\n",
    "    if userSingleGlyphSelection == 0:\n",
    "      p.asterisk(plotWhat, yLabel, source = sources[0], color = \"color\", legend = \"legend\", size = 22, line_width = 2)\n",
    "    if userSingleGlyphSelection == 2:  \n",
    "      p.diamond(plotWhat, yLabel, source = sources[0], color = \"color\", legend = \"legend\", size = 22, line_width = 2, fill_alpha = 0.5)\n",
    "    if userSingleGlyphSelection == 3:  \n",
    "      p.circle(plotWhat, yLabel, source = sources[0], color = \"color\", legend = \"legend\", size = 18, line_width = 2, fill_alpha = 0.5)\n",
    "    \n",
    "  \n",
    "  p.legend.location = \"top_left\"\n",
    "  p.legend.background_fill_color = \"snow\"\n",
    "  p.legend.background_fill_alpha = 0.9\n",
    "  p.legend.glyph_height = 38\n",
    "  p.legend.glyph_width = 38\n",
    "  p.legend.padding = 5\n",
    "  p.legend.spacing = -12\n",
    "\n",
    "  #p.toolbar.active_scroll = wZoom\n",
    "  p.toolbar.logo = None\n",
    "  p.toolbar_location = None\n",
    "  \n",
    "  # changing default sizes of fonts\n",
    "  p.xaxis.axis_label_text_font_size = \"16pt\"\n",
    "  p.xaxis.major_label_text_font_size = \"14pt\"\n",
    "  p.yaxis.axis_label_text_font_size = \"16pt\"\n",
    "  p.yaxis.major_label_text_font_size = \"14pt\"\n",
    "  p.legend.label_text_font_size = \"14pt\"\n",
    "\n",
    "  \n",
    "  show(p)\n",
    "\n",
    "  if writeToFile:\n",
    "    plotOutBaseFileName = outFilePrefix + plotWhat + \"__\" + \"_\".join(glyphShapeSrc) + \"_\" + glyphColorSrc + \"__\" + outFileSuffix\n",
    "    reset_output()\n",
    "    output_file(os.path.join(os.getcwd(), \"EvalData\", \"plots\", plotOutBaseFileName + \".html\"))\n",
    "    #save(p) #this line saves the plot in html\n",
    "    export_png(p, filename = os.path.join(os.getcwd(), \"EvalData\", \"plots\", plotOutBaseFileName + \".png\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figures 6-8\n",
    "### ICC Bin-width and Normlization on Selected Features (Texture)\n",
    "ICC for selected features computed on different ROIs on ADC, SUB, and T2w images. Texture features are computed in 2D. Colors represent the bin width for the texture computations, glyph shape represents if the image was normalized or not. No filtering was applied to the image. Dashed line indicates the reference Volume ICC."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selectedFeatureNames = [\"glcm_JointEnergy\",\"glcm_JointEntropy\", \"glcm_Idm\", \"glcm_Correlation\", \n",
    "                        \"glcm_Contrast\", \"shape_Volume\"]\n",
    "\n",
    "\n",
    "# Select image type: choose from [\"ADC1400\", \"SUB\", \"T2AX\"]\n",
    "for i, SELECTED_TYPE in enumerate([\"ADC1400\", \"SUB\", \"T2AX\"], 6):\n",
    "  BASE_IMAGE_TYPE = \"BC\" if SELECTED_TYPE == \"T2AX\" else \"ORIG\"\n",
    "  # Select segmented structures: choose from [\"TumorROI_PZ_1\", \"PeripheralZone\", \"WholeGland\"]\n",
    "  for j, SELECTED_STRUCT in enumerate([\"TumorROI_PZ_1\", \"PeripheralZone\", \"WholeGland\"]):\n",
    "    for DIM in [2]:\n",
    "      dfSelectedMetrics = dfMetrics[(dfMetrics[\"featureGroupAndBaseName\"].isin(selectedFeatureNames)) &\n",
    "                                    (dfMetrics.icc.notnull()) &\n",
    "                                    (dfMetrics[\"filter\"] == \"original\") &\n",
    "                                    (dfMetrics[\"baseImageType\"] == BASE_IMAGE_TYPE) &\n",
    "                                    (dfMetrics[\"ctype\"] == SELECTED_TYPE) &\n",
    "                                    (dfMetrics[\"structure\"] == SELECTED_STRUCT) &\n",
    "                                    (dfMetrics[\"dim\"] == DIM)\n",
    "                                   ]\n",
    "      dfSelectedMetrics = dfSelectedMetrics.sort_values(by = \"binwidth\", ascending = True)\n",
    "      if dfSelectedMetrics.shape[0] == 0:\n",
    "        continue\n",
    "      myColorDict = {5: \"binw-5\", 10: \"binw-10\", 15: \"binw-15\", 20: \"binw-20\", 40: \"binw-40\"}\n",
    "      myShapeDict = {True: \"Normed\", False: \"Original\"}\n",
    "      showBokehPlot(dfSelectedMetrics, plotWhat = \"icc\",\n",
    "                    glyphColorSrc = \"binwidth\", glyphShapeSrc = [\"normalized\"], \n",
    "                    colorLegendLookupDict = myColorDict, shapeLegendLookupDict = myShapeDict,\n",
    "                    outFilePrefix = \"Fig\" + str(i) + FIGURE_SUBIDX_MAP[j] + \"__\",\n",
    "                    outFileSuffix = \"__Texture__Imge_Type_Struct_Dim_Tex=\" + BASE_IMAGE_TYPE + \"_\" + SELECTED_TYPE + \"_\" + SELECTED_STRUCT + \"_\" + str(DIM),\n",
    "                    includeFeatGroupNameInLegend = True,\n",
    "                    writeToFile = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ICC Bin-width and Normlization on Selected Features (Firstorder)\n",
    "ICC for selected features computed on different ROIs on ADC, SUB, and T2w images. No filtering was applied to the image. Dashed line indicates the reference Volume ICC."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selectedFeatureNames = [\"firstorder_Mean\", \"firstorder_Median\", \"firstorder_10Percentile\", \n",
    "                        \"firstorder_Variance\", \"firstorder_Kurtosis\", \"firstorder_Skewness\", \"shape_Volume\"]\n",
    "\n",
    "# Select image type: choose from [\"ADC1400\", \"SUB\", \"T2AX\"]\n",
    "for i, SELECTED_TYPE in enumerate([\"ADC1400\", \"SUB\", \"T2AX\"], 6):\n",
    "  BASE_IMAGE_TYPE = \"BC\" if SELECTED_TYPE == \"T2AX\" else \"ORIG\"\n",
    "  # Select segmented structures: choose from [\"TumorROI_PZ_1\", \"PeripheralZone\", \"WholeGland\"]\n",
    "  for j, SELECTED_STRUCT in enumerate([\"TumorROI_PZ_1\", \"PeripheralZone\", \"WholeGland\"]):\n",
    "    for DIM in [2]:\n",
    "      dfSelectedMetrics = dfMetrics[(dfMetrics[\"featureGroupAndBaseName\"].isin(selectedFeatureNames)) &\n",
    "                                    (dfMetrics.icc.notnull()) &\n",
    "                                    (dfMetrics[\"filter\"] == \"original\") &\n",
    "                                    (dfMetrics[\"baseImageType\"] == BASE_IMAGE_TYPE) &\n",
    "                                    (dfMetrics[\"ctype\"] == SELECTED_TYPE) &\n",
    "                                    (dfMetrics[\"structure\"] == SELECTED_STRUCT) &\n",
    "                                    (dfMetrics[\"dim\"] == DIM) & \n",
    "                                    (dfMetrics[\"binwidth\"] == 10)\n",
    "                                   ]\n",
    "      dfSelectedMetrics = dfSelectedMetrics.sort_values(by = \"binwidth\", ascending = True)\n",
    "      if dfSelectedMetrics.shape[0] == 0:\n",
    "        continue\n",
    "      myColorDict = {True: \"\", False: \"\"}\n",
    "      myShapeDict = {True: \"Normed\", False: \"Original\"}\n",
    "      showBokehPlot(dfSelectedMetrics, plotWhat = \"icc\",\n",
    "                    glyphColorSrc = \"normalized\", glyphShapeSrc = [\"normalized\"], \n",
    "                    colorLegendLookupDict = myColorDict, shapeLegendLookupDict = myShapeDict,\n",
    "                    outFilePrefix = \"Fig\" + str(i) + FIGURE_SUBIDX_MAP[j] + \"__\",\n",
    "                    outFileSuffix = \"__Firstorder__Imge_Type_Struct_Dim_Tex=\" + BASE_IMAGE_TYPE + \"_\" + SELECTED_TYPE + \"_\" + SELECTED_STRUCT + \"_\" + str(DIM),\n",
    "                    includeFeatGroupNameInLegend = True,\n",
    "                    writeToFile = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## *Figure not in paper: Comparing bias corrected vs original T2AX*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for SELECTED_TYPE in [\"T2AX\"]:\n",
    "  # Select segmented structures: choose from [\"WholeGland\", \"PeripheralZone\", \"TumorROI_PZ_1\", \"NormalROI_PZ_1\"]\n",
    "  for SELECTED_STRUCT in [\"WholeGland\", \"PeripheralZone\", \"TumorROI_PZ_1\"]:\n",
    "    for DIM in [2]:\n",
    "      dfSelectedMetrics = dfMetrics[(dfMetrics.icc.notnull()) &\n",
    "                                    (dfMetrics[\"filter\"] == \"original\") &\n",
    "                                    (dfMetrics[\"baseImageType\"].isin([\"ORIG\", \"BC\"])) &\n",
    "                                    (dfMetrics[\"ctype\"] == SELECTED_TYPE) &\n",
    "                                    (dfMetrics[\"structure\"] == SELECTED_STRUCT) &\n",
    "                                    (dfMetrics[\"dim\"] == DIM) & \n",
    "                                    (dfMetrics[\"normalized\"] == False) &\n",
    "                                    (dfMetrics[\"binwidth\"] == 15)\n",
    "                                   ]\n",
    "      dfSelectedMetrics = dfSelectedMetrics.sort_values(by = \"binwidth\", ascending = True)\n",
    "      if dfSelectedMetrics.shape[0] == 0:\n",
    "        continue\n",
    "      myColorDict = {\"BC\": \"\", \"ORIG\": \"\"}\n",
    "      myShapeDict = {\"BC\": \"Bias-corrected\", \"ORIG\": \"Original\"}\n",
    "      showBokehPlot(dfSelectedMetrics, plotWhat = \"icc\",\n",
    "                    glyphColorSrc = \"baseImageType\", glyphShapeSrc = [\"baseImageType\"], \n",
    "                    colorLegendLookupDict = myColorDict, shapeLegendLookupDict = myShapeDict,\n",
    "                    outFileSuffix = \"Imge_Type_Struct_Dim=\" + SELECTED_TYPE + \"_\" + SELECTED_STRUCT + \"_\" + str(DIM),\n",
    "                    includeFeatGroupNameInLegend = True,\n",
    "                    writeToFile = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figures 9 and 10 \n",
    "KDE plots and Rank Count for bin width influence\n",
    "### Preparation for 9a and 10a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_IMAGE_TYPE = \"ORIG_or_BC\"\n",
    "selectedFeatureNames = [\"glcm_JointEnergy\",\"glcm_JointEntropy\", \"glcm_Idm\", \"glcm_Correlation\", \n",
    "                        \"glcm_Contrast\", \"shape_Volume\"]\n",
    "dfSelectedMetrics = dfMetrics[(dfMetrics.icc.notnull()) &\n",
    "                              (((dfMetrics[\"baseImageType\"] == \"ORIG\") & (dfMetrics[\"ctype\"] != \"T2AX\")) |\n",
    "                               ((dfMetrics[\"baseImageType\"] == \"BC\") & (dfMetrics[\"ctype\"] == \"T2AX\"))) &\n",
    "                              (dfMetrics[\"featureGroupAndBaseName\"].isin(selectedFeatureNames)) &\n",
    "                              (dfMetrics[\"filter\"] == \"original\")][['binwidth',\n",
    "                                                                    'ctype',\n",
    "                                                                    'dim',\n",
    "                                                                    'featureName',\n",
    "                                                                    'featureGroup',\n",
    "                                                                    'filter',\n",
    "                                                                    'icc',\n",
    "                                                                    'normalized',\n",
    "                                                                    'structure'\n",
    "                                                                   ]]\n",
    "\n",
    "dfSelectedMetrics = dfSelectedMetrics[(dfSelectedMetrics[\"featureGroup\"] != \"firstorder\") & \n",
    "                                      (dfSelectedMetrics[\"featureGroup\"] != \"shape\")]\n",
    "\n",
    "dfGroup = dfSelectedMetrics.groupby(['ctype','dim','featureName','featureGroup','filter','normalized','structure'])\n",
    "dfSelectedMetrics[\"rank\"] = dfGroup[\"icc\"].rank(ascending=False, method='min').astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 9a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "font = {'family' : 'sans',\n",
    "        'weight' : 'normal',\n",
    "        'size'   : 18}\n",
    "plt.rc('font', **font)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "fig.set_size_inches(11.7, 8.27)\n",
    "\n",
    "maxIccDiffPerGroup = dfGroup[\"icc\"].agg(['max','min'])\n",
    "maxIccDiffPerGroup['maxDiff'] = maxIccDiffPerGroup['max']-maxIccDiffPerGroup['min']\n",
    "plot = sb.kdeplot(maxIccDiffPerGroup['maxDiff'], shade = True)\n",
    "plot.set(xlim = (-0.15,1.5))\n",
    "fig.savefig(os.path.join(os.getcwd(), \"EvalData\", \"plots\", \"Fig9a__\" + BASE_IMAGE_TYPE + \"_bin__maxdiff_kdeplot__select_glcm.png\"), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 10a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "font = {'family' : 'sans',\n",
    "        'weight' : 'normal',\n",
    "        'size'   : 18}\n",
    "plt.rc('font', **font)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "fig.set_size_inches(11.7, 8.27)\n",
    "sb.countplot(x=\"binwidth\", hue=\"rank\", data=dfSelectedMetrics)\n",
    "fig.savefig(os.path.join(os.getcwd(), \"EvalData\", \"plots\", \"Fig10a__\" + BASE_IMAGE_TYPE + \"_bin__rank_countplot__select_glcm.png\"), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparation for 9b and 10b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_IMAGE_TYPE = \"ORIG_or_BC\"\n",
    "dfSelectedMetrics = dfMetrics[(dfMetrics.icc.notnull()) &\n",
    "                              (((dfMetrics[\"baseImageType\"] == \"ORIG\") & (dfMetrics[\"ctype\"] != \"T2AX\")) |\n",
    "                               ((dfMetrics[\"baseImageType\"] == \"BC\") & (dfMetrics[\"ctype\"] == \"T2AX\")))][['binwidth',\n",
    "                                                                                                          'ctype',\n",
    "                                                                                                          'dim',\n",
    "                                                                                                          'featureName',\n",
    "                                                                                                          'featureGroup',\n",
    "                                                                                                          'filter',\n",
    "                                                                                                          'icc',\n",
    "                                                                                                          'normalized',\n",
    "                                                                                                          'structure'\n",
    "                                                                                                         ]]\n",
    "\n",
    "dfSelectedMetrics = dfSelectedMetrics[(dfSelectedMetrics[\"featureGroup\"] != \"firstorder\") & \n",
    "                                      (dfSelectedMetrics[\"featureGroup\"] != \"shape\")]\n",
    "\n",
    "dfGroup = dfSelectedMetrics.groupby(['ctype','dim','featureName','featureGroup','filter','normalized','structure'])\n",
    "dfSelectedMetrics[\"rank\"] = dfGroup[\"icc\"].rank(ascending=False, method='min').astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 9b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "font = {'family' : 'sans',\n",
    "        'weight' : 'normal',\n",
    "        'size'   : 18}\n",
    "plt.rc('font', **font)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "fig.set_size_inches(11.7, 8.27)\n",
    "\n",
    "maxIccDiffPerGroup = dfGroup[\"icc\"].agg(['max','min'])\n",
    "maxIccDiffPerGroup['maxDiff'] = maxIccDiffPerGroup['max']-maxIccDiffPerGroup['min']\n",
    "plot = sb.kdeplot(maxIccDiffPerGroup['maxDiff'], shade = True)\n",
    "plot.set(xlim = (-0.15,1.5))\n",
    "fig.savefig(os.path.join(os.getcwd(), \"EvalData\", \"plots\", \"Fig9b__\" + BASE_IMAGE_TYPE + \"_bin__maxdiff_kdeplot.png\"), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 10b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "font = {'family' : 'sans',\n",
    "        'weight' : 'normal',\n",
    "        'size'   : 18}\n",
    "plt.rc('font', **font)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "fig.set_size_inches(11.7, 8.27)\n",
    "sb.countplot(x=\"binwidth\", hue=\"rank\", data=dfSelectedMetrics)\n",
    "fig.savefig(os.path.join(os.getcwd(), \"EvalData\", \"plots\", \"Fig10b__\" + BASE_IMAGE_TYPE + \"_bin__rank_countplot.png\"), bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 11\n",
    "ICC for selected features computed on the Tumor ROI on T2w images with (a) 2D and (b) 3D wavelet pre-filtering and feature extraction.\n",
    "### Figure 11a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selectedFeatureNames = [\"firstorder_Mean\", \"firstorder_Median\", \"firstorder_10Percentile\", \n",
    "                        \"firstorder_Variance\", \"firstorder_Kurtosis\", \"firstorder_Skewness\",\n",
    "                        \"glcm_JointEnergy\",\"glcm_JointEntropy\", \"glcm_Idm\", \"glcm_Correlation\", \n",
    "                        \"glcm_Contrast\", \"shape_Volume\"]\n",
    "BASE_IMAGE_TYPE = \"BC\"\n",
    "BIN = 15\n",
    "DIM = 2\n",
    "NORM = False\n",
    "CTYPE = \"T2AX\"\n",
    "STRUCTURE = \"TumorROI_PZ_1\"\n",
    "dfSelectedMetrics = dfMetrics[(dfMetrics.icc.notnull()) & \n",
    "                              (dfMetrics[\"featureGroupAndBaseName\"].isin(selectedFeatureNames)) &\n",
    "                              (dfMetrics[\"baseImageType\"] == BASE_IMAGE_TYPE) &\n",
    "                              (dfMetrics[\"binwidth\"] == BIN) &\n",
    "                              (dfMetrics[\"dim\"] == DIM) &\n",
    "                              (dfMetrics[\"normalized\"] == NORM) &\n",
    "                              (dfMetrics[\"ctype\"] == CTYPE) &\n",
    "                              (dfMetrics[\"structure\"] == STRUCTURE) &\n",
    "                              ((dfMetrics[\"featureGroup\"] == \"shape\") | (dfMetrics[\"featureGroup\"] == \"firstorder\") \n",
    "                                                                      | (dfMetrics[\"featureGroup\"] == \"glcm\")) &\n",
    "                              ((dfMetrics[\"filter\"] == \"original\") | (dfMetrics[\"filter\"] == \"wavelet-LL\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"wavelet-LH\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"wavelet-HL\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"wavelet-HH\"))\n",
    "                             ]\n",
    "dfSelectedMetrics = dfSelectedMetrics.sort_values(by = \"filter\")\n",
    "\n",
    "myShapeDict = {True: \"\", False: \"\"}\n",
    "showBokehPlot(dfSelectedMetrics, plotWhat = \"icc\",\n",
    "              glyphColorSrc = \"filter\", glyphShapeSrc = \"normalized\",\n",
    "              shapeLegendLookupDict = myShapeDict,\n",
    "              outFilePrefix = \"Fig11a__\",\n",
    "              outFileSuffix = \"Wavelet__Bin_Dim_Normalized_CType_Struct=\" + str(BIN) + \"_\" + str(DIM) + \"_\" + str(NORM) + \"_\" + str(CTYPE) + \"_\" + str(STRUCTURE),\n",
    "              writeToFile = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 11b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selectedFeatureNames = [\"firstorder_Mean\", \"firstorder_Median\", \"firstorder_10Percentile\", \n",
    "                        \"firstorder_Variance\", \"firstorder_Kurtosis\", \"firstorder_Skewness\",\n",
    "                        \"glcm_JointEnergy\",\"glcm_JointEntropy\", \"glcm_Idm\", \"glcm_Correlation\", \n",
    "                        \"glcm_Contrast\", \"shape_Volume\"]\n",
    "BASE_IMAGE_TYPE = \"BC\"\n",
    "BIN = 15\n",
    "DIM = 3\n",
    "NORM = False\n",
    "CTYPE = \"T2AX\"\n",
    "STRUCTURE = \"TumorROI_PZ_1\"\n",
    "dfSelectedMetrics = dfMetrics[(dfMetrics.icc.notnull()) & \n",
    "                              (dfMetrics[\"featureGroupAndBaseName\"].isin(selectedFeatureNames)) &\n",
    "                              (dfMetrics[\"baseImageType\"] == BASE_IMAGE_TYPE) &\n",
    "                              (dfMetrics[\"binwidth\"] == BIN) &\n",
    "                              (dfMetrics[\"dim\"] == DIM) &\n",
    "                              (dfMetrics[\"normalized\"] == NORM) &\n",
    "                              (dfMetrics[\"ctype\"] == CTYPE) &\n",
    "                              (dfMetrics[\"structure\"] == STRUCTURE) &\n",
    "                              ((dfMetrics[\"featureGroup\"] == \"shape\") | (dfMetrics[\"featureGroup\"] == \"firstorder\") \n",
    "                                                                      | (dfMetrics[\"featureGroup\"] == \"glcm\")) &\n",
    "                              ((dfMetrics[\"filter\"] == \"original\") | (dfMetrics[\"filter\"] == \"wavelet-LLL\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"wavelet-HLL\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"wavelet-LHL\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"wavelet-LLH\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"wavelet-LHH\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"wavelet-HLH\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"wavelet-HHL\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"wavelet-HHH\"))\n",
    "                             ]\n",
    "dfSelectedMetrics = dfSelectedMetrics.sort_values(by = \"filter\")\n",
    "\n",
    "myShapeDict = {True: \"\", False: \"\"}\n",
    "showBokehPlot(dfSelectedMetrics, plotWhat = \"icc\",\n",
    "              glyphColorSrc = \"filter\", glyphShapeSrc = \"normalized\",\n",
    "              shapeLegendLookupDict = myShapeDict,\n",
    "              outFilePrefix = \"Fig11b__\",\n",
    "              outFileSuffix = \"Wavelet__Bin_Dim_Normalized_CType_Struct=\" + str(BIN) + \"_\" + str(DIM) + \"_\" + str(NORM) + \"_\" + str(CTYPE) + \"_\" + str(STRUCTURE),\n",
    "              writeToFile = True, userSingleGlyphSelection = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 12\n",
    "ICC for selected features computed on the Tumor ROI on T2w images with 3D LoG pre-filtering and texture features extracted in 2D (a) and 3D (b). \n",
    "### Figure 12a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selectedFeatureNames = [\"firstorder_Mean\", \"firstorder_Median\", \"firstorder_10Percentile\", \n",
    "                        \"firstorder_Variance\", \"firstorder_Kurtosis\", \"firstorder_Skewness\",\n",
    "                        \"glcm_JointEnergy\",\"glcm_JointEntropy\", \"glcm_Idm\", \"glcm_Correlation\", \n",
    "                        \"glcm_Contrast\", \"shape_Volume\"]\n",
    "BASE_IMAGE_TYPE = \"BC\"\n",
    "BIN = 15\n",
    "NORM = False\n",
    "CTYPE = \"T2AX\"\n",
    "STRUCTURE = \"TumorROI_PZ_1\"\n",
    "DIM = 2\n",
    "dfSelectedMetrics = dfMetrics[(dfMetrics.icc.notnull()) & \n",
    "                              (dfMetrics[\"featureGroupAndBaseName\"].isin(selectedFeatureNames)) &\n",
    "                              (dfMetrics[\"baseImageType\"] == BASE_IMAGE_TYPE) &\n",
    "                              (dfMetrics[\"binwidth\"] == BIN) &\n",
    "                              (dfMetrics[\"normalized\"] == NORM) &\n",
    "                              (dfMetrics[\"ctype\"] == CTYPE) &\n",
    "                              (dfMetrics[\"structure\"] == STRUCTURE) &\n",
    "                              (dfMetrics[\"dim\"] == DIM) &\n",
    "                              ((dfMetrics[\"featureGroup\"] == \"shape\") | (dfMetrics[\"featureGroup\"] == \"firstorder\") \n",
    "                                                                      | (dfMetrics[\"featureGroup\"] == \"glcm\")) &\n",
    "                              ((dfMetrics[\"filter\"] == \"original\") | (dfMetrics[\"filter\"] == \"log-sigma-2-0-mm-3D\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"log-sigma-3-0-mm-3D\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"log-sigma-4-0-mm-3D\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"log-sigma-5-0-mm-3D\"))\n",
    "                             ]\n",
    "dfSelectedMetrics = dfSelectedMetrics.sort_values(by = \"filter\")\n",
    "\n",
    "myColorDict = {}\n",
    "myColorDict = {\"log-sigma-2-0-mm-3D\": \"log-sigma-2.0mm\", \n",
    "               \"log-sigma-3-0-mm-3D\": \"log-sigma-3.0mm\", \n",
    "               \"log-sigma-4-0-mm-3D\": \"log-sigma-4.0mm\", \n",
    "               \"log-sigma-5-0-mm-3D\": \"log-sigma-5.0mm\"}\n",
    "myShapeDict = {2: \"\", 3: \"\"}\n",
    "showBokehPlot(dfSelectedMetrics, plotWhat = \"icc\",\n",
    "              glyphColorSrc = \"filter\", glyphShapeSrc = \"dim\",\n",
    "              colorLegendLookupDict = myColorDict, shapeLegendLookupDict = myShapeDict,\n",
    "              outFilePrefix = \"Fig12a__\",\n",
    "              outFileSuffix = \"Log__Bin_Normalized_Ctype_Struct_Dim=\" + str(BIN) + \"_\" + str(NORM) + \"_\" + str(CTYPE) + \"_\" + str(STRUCTURE) + \"_\" + str(DIM),\n",
    "              writeToFile = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 12b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selectedFeatureNames = [\"glcm_JointEnergy\",\"glcm_JointEntropy\", \"glcm_Idm\", \"glcm_Correlation\", \n",
    "                        \"glcm_Contrast\", \"shape_Volume\"]\n",
    "BASE_IMAGE_TYPE = \"BC\"\n",
    "BIN = 15\n",
    "NORM = False\n",
    "CTYPE = \"T2AX\"\n",
    "STRUCTURE = \"TumorROI_PZ_1\"\n",
    "DIM = 3\n",
    "dfSelectedMetrics = dfMetrics[(dfMetrics.icc.notnull()) & \n",
    "                              (dfMetrics[\"featureGroupAndBaseName\"].isin(selectedFeatureNames)) &\n",
    "                              (dfMetrics[\"baseImageType\"] == BASE_IMAGE_TYPE) &\n",
    "                              (dfMetrics[\"binwidth\"] == BIN) &\n",
    "                              (dfMetrics[\"normalized\"] == NORM) &\n",
    "                              (dfMetrics[\"ctype\"] == CTYPE) &\n",
    "                              (dfMetrics[\"structure\"] == STRUCTURE) &\n",
    "                              (dfMetrics[\"dim\"] == DIM) &\n",
    "                              ((dfMetrics[\"featureGroup\"] == \"shape\") | (dfMetrics[\"featureGroup\"] == \"firstorder\") \n",
    "                                                                      | (dfMetrics[\"featureGroup\"] == \"glcm\")) &\n",
    "                              ((dfMetrics[\"filter\"] == \"original\") | (dfMetrics[\"filter\"] == \"log-sigma-2-0-mm-3D\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"log-sigma-3-0-mm-3D\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"log-sigma-4-0-mm-3D\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"log-sigma-5-0-mm-3D\"))\n",
    "                             ]\n",
    "dfSelectedMetrics = dfSelectedMetrics.sort_values(by = \"filter\")\n",
    "\n",
    "myColorDict = {}\n",
    "myColorDict = {\"log-sigma-2-0-mm-3D\": \"log-sigma-2.0mm\", \n",
    "               \"log-sigma-3-0-mm-3D\": \"log-sigma-3.0mm\", \n",
    "               \"log-sigma-4-0-mm-3D\": \"log-sigma-4.0mm\", \n",
    "               \"log-sigma-5-0-mm-3D\": \"log-sigma-5.0mm\"}\n",
    "myShapeDict = {2: \"\", 3: \"\"}\n",
    "showBokehPlot(dfSelectedMetrics, plotWhat = \"icc\",\n",
    "              glyphColorSrc = \"filter\", glyphShapeSrc = \"dim\",\n",
    "              colorLegendLookupDict = myColorDict, shapeLegendLookupDict = myShapeDict,\n",
    "              outFilePrefix = \"Fig12b__\",\n",
    "              outFileSuffix = \"Log__Bin_Normalized_Ctype_Struct_Dim=\" + str(BIN) + \"_\" + str(NORM) + \"_\" + str(CTYPE) + \"_\" + str(STRUCTURE) + \"_\" + str(DIM),\n",
    "              writeToFile = True, userSingleGlyphSelection = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 13\n",
    "ICC for selected features computed on the Tumor ROI on T2w images with single pixel pre-filtering and texture features extracted in 2D (a) and 3D (b)\n",
    "### Figure 13a "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selectedFeatureNames = [\"firstorder_Mean\", \"firstorder_Median\", \"firstorder_10Percentile\", \n",
    "                        \"firstorder_Variance\", \"firstorder_Kurtosis\", \"firstorder_Skewness\",\n",
    "                        \"glcm_JointEnergy\",\"glcm_JointEntropy\", \"glcm_Idm\", \"glcm_Correlation\", \n",
    "                        \"glcm_Contrast\", \"shape_Volume\"]\n",
    "BASE_IMAGE_TYPE = \"BC\"\n",
    "BIN = 15\n",
    "NORM = False\n",
    "CTYPE = \"T2AX\"\n",
    "STRUCTURE = \"TumorROI_PZ_1\"\n",
    "DIM = 2\n",
    "dfSelectedMetrics = dfMetrics[(dfMetrics.icc.notnull()) & \n",
    "                              (dfMetrics[\"featureGroupAndBaseName\"].isin(selectedFeatureNames)) &\n",
    "                              (dfMetrics[\"baseImageType\"] == BASE_IMAGE_TYPE) &\n",
    "                              (dfMetrics[\"binwidth\"] == BIN) &\n",
    "                              (dfMetrics[\"normalized\"] == NORM) &\n",
    "                              (dfMetrics[\"ctype\"] == CTYPE) &\n",
    "                              (dfMetrics[\"structure\"] == STRUCTURE) &\n",
    "                              (dfMetrics[\"dim\"] == DIM) &\n",
    "                              ((dfMetrics[\"featureGroup\"] == \"shape\") | (dfMetrics[\"featureGroup\"] == \"firstorder\") \n",
    "                                                                      | (dfMetrics[\"featureGroup\"] == \"glcm\")) &\n",
    "                              ((dfMetrics[\"filter\"] == \"original\") | (dfMetrics[\"filter\"] == \"square\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"squareroot\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"exponential\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"logarithm\"))\n",
    "                             ]\n",
    "dfSelectedMetrics = dfSelectedMetrics.sort_values(by = \"filter\")\n",
    "\n",
    "myShapeDict = {2: \"\", 3: \"\"}\n",
    "showBokehPlot(dfSelectedMetrics, plotWhat = \"icc\",\n",
    "              glyphColorSrc = \"filter\", glyphShapeSrc = \"dim\",\n",
    "              shapeLegendLookupDict = myShapeDict,\n",
    "              outFilePrefix = \"Fig13a__\",\n",
    "              outFileSuffix = \"ExpEtc__Bin_Normalized_Ctype_Struct_Dim=\" + str(BIN) + \"_\" + str(NORM) + \"_\" + str(CTYPE) + \"_\" + str(STRUCTURE) + \"_\" + str(DIM),\n",
    "              writeToFile = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 13b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "selectedFeatureNames = [\"glcm_JointEnergy\",\"glcm_JointEntropy\", \"glcm_Idm\", \"glcm_Correlation\", \n",
    "                        \"glcm_Contrast\", \"shape_Volume\"]\n",
    "BASE_IMAGE_TYPE = \"BC\"\n",
    "BIN = 15\n",
    "NORM = False\n",
    "CTYPE = \"T2AX\"\n",
    "STRUCTURE = \"TumorROI_PZ_1\"\n",
    "DIM = 3\n",
    "dfSelectedMetrics = dfMetrics[(dfMetrics.icc.notnull()) & \n",
    "                              (dfMetrics[\"featureGroupAndBaseName\"].isin(selectedFeatureNames)) &\n",
    "                              (dfMetrics[\"baseImageType\"] == BASE_IMAGE_TYPE) &\n",
    "                              (dfMetrics[\"binwidth\"] == BIN) &\n",
    "                              (dfMetrics[\"normalized\"] == NORM) &\n",
    "                              (dfMetrics[\"ctype\"] == CTYPE) &\n",
    "                              (dfMetrics[\"structure\"] == STRUCTURE) &\n",
    "                              (dfMetrics[\"dim\"] == DIM) &\n",
    "                              ((dfMetrics[\"featureGroup\"] == \"shape\") | (dfMetrics[\"featureGroup\"] == \"firstorder\") \n",
    "                                                                      | (dfMetrics[\"featureGroup\"] == \"glcm\")) &\n",
    "                              ((dfMetrics[\"filter\"] == \"original\") | (dfMetrics[\"filter\"] == \"square\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"squareroot\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"exponential\")\n",
    "                                                                   | (dfMetrics[\"filter\"] == \"logarithm\"))\n",
    "                             ]\n",
    "dfSelectedMetrics = dfSelectedMetrics.sort_values(by = \"filter\")\n",
    "\n",
    "myShapeDict = {2: \"\", 3: \"\"}\n",
    "showBokehPlot(dfSelectedMetrics, plotWhat = \"icc\",\n",
    "              glyphColorSrc = \"filter\", glyphShapeSrc = \"dim\",\n",
    "              shapeLegendLookupDict = myShapeDict,\n",
    "              outFilePrefix = \"Fig13b__\",\n",
    "              outFileSuffix = \"ExpEtc__Bin_Normalized_Ctype_Struct_Dim=\" + str(BIN) + \"_\" + str(NORM) + \"_\" + str(CTYPE) + \"_\" + str(STRUCTURE) + \"_\" + str(DIM),\n",
    "              writeToFile = True, userSingleGlyphSelection = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convenience function for Top3 Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTop3FeaturesPerGroupAndBaseImage(bin_width, dim, norm, ctype, structure, \n",
    "                                        base_image_types, \n",
    "                                        groups = [\"shape\", \"firstorder\", \"glcm\", \"glrlm\", \"glszm\"]):\n",
    "  featuresNames = []\n",
    "  for GROUP in groups:\n",
    "    for BASE_IMAGE_TYPE in base_image_types: # \"ORIG\", \"BC\", \"BC+MRN\", \"BC+REG\"\n",
    "      dfSelectedMetrics = dfSortedMetrics[(dfSortedMetrics.icc.notnull()) &\n",
    "                                          (dfMetrics[\"baseImageType\"] == BASE_IMAGE_TYPE) &\n",
    "                                          (dfSortedMetrics[\"binwidth\"] == bin_width) &\n",
    "                                          (dfSortedMetrics[\"dim\"] == dim) &\n",
    "                                          (dfSortedMetrics[\"normalized\"] == norm) &\n",
    "                                          (dfSortedMetrics[\"ctype\"] == ctype) &\n",
    "                                          (dfSortedMetrics[\"structure\"] == structure) &\n",
    "                                          (dfSortedMetrics[\"featureGroup\"] == GROUP)\n",
    "                                         ]\n",
    "      dfSelectedMetrics = dfSelectedMetrics.drop_duplicates(subset = \"featureGroupAndBaseName\", keep = \"first\")\n",
    "      featuresNames.extend((dfSelectedMetrics[\"featureName\"].values.tolist())[:3])\n",
    "  return featuresNames\n",
    "\n",
    "\n",
    "def getAllMetrics(bin_width, dim, norm, ctype, structure, \n",
    "                  featureGroupAndBaseNames,\n",
    "                  base_image_types, \n",
    "                  groups = [\"shape\", \"firstorder\", \"glcm\", \"glrlm\", \"glszm\"]):\n",
    "  dfSelectedMetrics = dfMetrics[(dfMetrics.icc.notnull()) & \n",
    "                                (dfMetrics[\"baseImageType\"].isin(base_image_types)) &\n",
    "                                (dfMetrics[\"binwidth\"] == bin_width) &\n",
    "                                (dfMetrics[\"dim\"] == dim) &\n",
    "                                (dfMetrics[\"normalized\"] == norm) &\n",
    "                                (dfMetrics[\"ctype\"] == ctype) &\n",
    "                                (dfMetrics[\"structure\"] == structure) &\n",
    "                                ((dfMetrics[\"featureGroupAndBaseName\"].isin(featureGroupAndBaseNames)) | \n",
    "                                 (dfMetrics[\"featureBaseName\"] == \"Volume\")) &\n",
    "                                (dfMetrics[\"featureGroup\"].isin(groups)) &\n",
    "                                ((dfMetrics[\"filter\"] == \"original\") | (dfMetrics[\"filter\"] == \"wavelet-LL\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-LH\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HL\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HH\")\n",
    "                                                                     | ((dfMetrics[\"filter\"] == \"wavelet-LLL\") & \n",
    "                                                                        (dfMetrics[\"dim\"] == 3))\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HLL\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-LHL\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-LLH\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-LHH\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HLH\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HHL\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HHH\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"log-sigma-2-0-mm-3D\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"log-sigma-3-0-mm-3D\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"log-sigma-4-0-mm-3D\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"log-sigma-5-0-mm-3D\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"square\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"squareroot\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"exponential\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"logarithm\"))\n",
    "                               ]\n",
    "  return dfSelectedMetrics\n",
    "\n",
    "\n",
    "def top3Plot(base_image_type, bin_width, dim, norm, ctype, structure, outFilePrefix = \"\"):\n",
    "  featuresNames = getTop3FeaturesPerGroupAndBaseImage(bin_width, dim, norm, ctype, structure, [base_image_type])\n",
    "  featureGroupAndBaseNames = [f.split(\"_\", maxsplit = 1)[1] for f in featuresNames]\n",
    "  dfSelectedMetrics = getAllMetrics(bin_width, dim, norm, ctype, structure, featureGroupAndBaseNames, [base_image_type])\n",
    "\n",
    "  dfSelectedMetrics = dfSelectedMetrics.sort_values(by = [\"filterGroup\", \"filter\"])\n",
    "  myShapeDict = {\"a_original\": \"\", \"log\": \"\", \"singlepix\": \"\", \"wavelet\": \"\"}\n",
    "  showBokehPlot(dfSelectedMetrics, plotWhat = \"icc\",\n",
    "                glyphColorSrc = \"filter\", glyphShapeSrc = \"filterGroup\",\n",
    "                shapeLegendLookupDict = myShapeDict,\n",
    "                outFilePrefix = outFilePrefix,\n",
    "                outFileSuffix = \"Top3_AllFilters__Bin_Dim_Normalized_CType_Struct=\" + str(bin_width) + \"_\" + str(dim) + \"_\" + str(norm) + \"_\" + str(ctype) + \"_\" + str(structure),\n",
    "                writeToFile = True,\n",
    "                includeFeatGroupNameInLegend = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 14\n",
    "Top 3 features for each feature group by ICC in T2w images for (a) the Tumor ROI and (b) the Peripheral Zone\n",
    "### Figure 14a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top3Plot(\"BC\", 15, 2, False, \"T2AX\", \"TumorROI_PZ_1\", \"Fig14a__\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 14b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top3Plot(\"BC\", 15, 2, False, \"T2AX\", \"PeripheralZone\", \"Fig14b__\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 15\n",
    "Top 3 features for each feature group by ICC in ADC images for (a) the Tumor ROI and (b) the Peripheral Zone\n",
    "### Figure 15a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top3Plot(\"ORIG\", 15, 2, True, \"ADC1400\", \"TumorROI_PZ_1\", \"Fig15a__\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 15b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top3Plot(\"ORIG\", 15, 2, True, \"ADC1400\", \"PeripheralZone\", \"Fig15b__\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convenience Function for AboveVolumeICC Countplots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotAboveVolumeIccCount(base_image_type, bin_width, dim, norm, ctype, structure, outFilePrefix = \"\"):\n",
    "  dfSelectedMetrics = dfMetrics[(dfMetrics.icc.notnull()) &\n",
    "                                (dfMetrics[\"baseImageType\"] == base_image_type) &\n",
    "                                (dfMetrics[\"binwidth\"] == bin_width) &\n",
    "                                (dfMetrics[\"dim\"] == dim) &\n",
    "                                (dfMetrics[\"normalized\"] == norm) &\n",
    "                                (dfMetrics[\"ctype\"] == ctype) &\n",
    "                                (dfMetrics[\"structure\"] == structure) &\n",
    "                                ((dfMetrics[\"filter\"] == \"original\") | (dfMetrics[\"filter\"] == \"wavelet-LL\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-LH\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HL\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HH\")\n",
    "                                                                     | ((dfMetrics[\"filter\"] == \"wavelet-LLL\") & (dfMetrics[\"dim\"] == 3))\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HLL\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-LHL\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-LLH\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-LHH\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HLH\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HHL\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"wavelet-HHH\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"log-sigma-2-0-mm-3D\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"log-sigma-3-0-mm-3D\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"log-sigma-4-0-mm-3D\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"log-sigma-5-0-mm-3D\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"square\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"squareroot\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"exponential\")\n",
    "                                                                     | (dfMetrics[\"filter\"] == \"logarithm\"))\n",
    "                               ]\n",
    "  volumeIcc = dfSelectedMetrics[(dfSelectedMetrics[\"featureBaseName\"] == \"Volume\")][\"icc\"].values[0]\n",
    "  dfSelectedMetrics = dfSelectedMetrics[(dfSelectedMetrics[\"icc\"] > volumeIcc)]\n",
    "  dfSelectedMetrics = dfSelectedMetrics.sort_values(by = [\"filterGroup\", \"filter\"])\n",
    "  dfFeatures = dfSelectedMetrics[\"featureGroupAndBaseName\"].unique()\n",
    "  noFeaturesAboveVolIcc = len(dfFeatures)\n",
    "\n",
    "  sb.set(style=\"ticks\", font_scale=2.0)\n",
    "  fig, ax = plt.subplots()\n",
    "  fig.set_size_inches(11.7, 8.27)\n",
    "  g = sb.countplot(x=\"filter\", data=dfSelectedMetrics)\n",
    "  for item in g.get_xticklabels():\n",
    "      item.set_rotation(-90)\n",
    "  plt.plot([-1, 14], [noFeaturesAboveVolIcc, noFeaturesAboveVolIcc], linewidth=2, linestyle='--', color='gray')\n",
    "  fig.savefig(os.path.join(os.getcwd(), \"EvalData\", \"plots\", outFilePrefix + \"aboveVolumeIcc_filter_countplot_\" + ctype + \"_\"+ structure + \".png\"), bbox_inches='tight')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 16\n",
    "Overview of how often the particular pre-filters appear among the features which reach an ICC higher than Volume on T2w images for (a) Tumor ROI and (b) Peripheral Zone.\n",
    "### Figure 16a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotAboveVolumeIccCount(\"BC\", 15, 2, False, \"T2AX\", \"TumorROI_PZ_1\", \"Fig16a__\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 16b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotAboveVolumeIccCount(\"BC\", 15, 2, False, \"T2AX\", \"PeripheralZone\", \"Fig16b__\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 17\n",
    "Overview of how often the particular pre-filters appear among the features which reach an ICC higher than Volume on ADC images for (a) Tumor ROI and (b) Peripheral Zone.\n",
    "### Figure 17a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotAboveVolumeIccCount(\"ORIG\", 15, 2, True, \"ADC1400\", \"TumorROI_PZ_1\", \"Fig17a__\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 17b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotAboveVolumeIccCount(\"ORIG\", 15, 2, True, \"ADC1400\", \"PeripheralZone\", \"Fig17b__\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convenience Functions for Trend Line Plots\n",
    "Also uses some convenience functions from the Top3 plots, so they need to be executed for this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plotTrendLines(left_image_type, right_image_type, bin_width, dim, norm, ctype, structure, featuresNames, outFilePrefix = \"\"):\n",
    "  postfix = \"_Selected\"\n",
    "  if not featuresNames or len(featuresNames) == 0:\n",
    "    postfix = \"_Top3\"\n",
    "    featuresNames = getTop3FeaturesPerGroupAndBaseImage(bin_width, dim, norm, ctype, structure, [left_image_type, right_image_type])\n",
    "  featureGroupAndBaseNames = [f.split(\"_\", maxsplit = 1)[1] for f in featuresNames]\n",
    "  featureGroupAndBaseNames = unique_keep_order(featureGroupAndBaseNames)\n",
    "  \n",
    "  dfSelectedMetrics = getAllMetrics(bin_width, dim, norm, ctype, structure, featureGroupAndBaseNames, [left_image_type, right_image_type])\n",
    "\n",
    "  # Map from codes to human readable text\n",
    "  xLabelMap = {\n",
    "    \"ORIG\": \"Original\",\n",
    "    \"BC\": \"T2w\", # Bias Corrected\n",
    "    \"BC+MRN\": \"T2w Normalized by Reference\", # Bias Corrected + Muscle Ref Norm\n",
    "    \"BC+REG\": \"T2w with Registration\" # Bias Corrected + Registration\n",
    "  }\n",
    "  \n",
    "  dfSelectedMetricsLeft = dfSelectedMetrics[(dfSelectedMetrics[\"baseImageType\"] == left_image_type)]\n",
    "  dfSelectedMetricsLeft = dfSelectedMetricsLeft.sort_values(by = \"icc\", ascending = False)\n",
    "  dfSelectedMetricsLeft = dfSelectedMetricsLeft[(dfSelectedMetricsLeft[\"featureName\"].isin(featuresNames))]\n",
    "  #dfSelectedMetricsLeft = dfSelectedMetricsLeft.drop_duplicates(subset = \"featureGroupAndBaseName\", keep = \"first\")\n",
    "  \n",
    "  dfSelectedMetricsRight = dfSelectedMetrics[(dfSelectedMetrics[\"baseImageType\"] == right_image_type)]\n",
    "  dfSelectedMetricsRight = dfSelectedMetricsRight.sort_values(by = \"icc\", ascending = False)\n",
    "  dfSelectedMetricsRight = dfSelectedMetricsRight[(dfSelectedMetricsRight[\"featureName\"].isin(featuresNames))]\n",
    "  #dfSelectedMetricsRight = dfSelectedMetricsRight.drop_duplicates(subset = \"featureGroupAndBaseName\", keep = \"first\")\n",
    "\n",
    "  font = {'family' : 'sans',\n",
    "          'weight' : 'normal',\n",
    "          'size'   : 18}\n",
    "  colorMap = {\"firstorder\": \"C0\",\n",
    "              \"glcm\"      : \"C1\",\n",
    "              \"glrlm\"     : \"C2\",\n",
    "              \"glszm\"     : \"C3\",\n",
    "              \"shape\"     : \"C4\"}\n",
    "\n",
    "  matplotlib.style.use(\"default\")\n",
    "  plt.rc('font', **font)\n",
    "  plt.figure(figsize=(10,10))\n",
    "  plt.ylabel('$\\it{ICC}$', fontsize = 20)\n",
    "\n",
    "  textY = 1.02\n",
    "  iccAndFilters = dfSelectedMetricsRight[[\"icc\", \"filter\", \"featureName\", \"featureGroup\"]]\n",
    "  for idx, row in iccAndFilters.iterrows():\n",
    "    group = row[\"featureGroup\"]\n",
    "    y2 = row[\"icc\"]\n",
    "    y1 = dfSelectedMetricsLeft[(dfSelectedMetricsLeft[\"featureName\"] == row[\"featureName\"])\n",
    "                              ][\"icc\"].values[0]\n",
    "    plt.plot([1, 2], [y1, y2], linestyle='-', color=colorMap[group])\n",
    "\n",
    "    plt.text(2.3, textY, row[\"featureName\"], fontsize=14, horizontalalignment='left', verticalalignment='center', color=colorMap[group])\n",
    "    ll = plt.plot([2.01, 2.28], [y2, textY], linestyle=':', color=colorMap[group]) # returns a sequence of line objects \n",
    "    ll[0].set_clip_on(False)\n",
    "    textY -= 0.05\n",
    "\n",
    "    plt.axis([0.95,2.05,-0.2,1.05]) \n",
    "\n",
    "\n",
    "  plt.xticks([1,2], [xLabelMap[left_image_type], xLabelMap[right_image_type]])\n",
    "\n",
    "  plt.savefig(os.path.join(os.getcwd(), \"EvalData\", \"plots\", outFilePrefix + \"TrendLines_\" + STRUCTURE + \"_\" + CTYPE + \"_\" + left_image_type + \"_vs_\" + right_image_type + postfix + \".png\"), bbox_inches='tight')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 18 a + b\n",
    "Change in Tumor ROI ICCs from T2w to T2w normalized by a reference region (muscle) for (a) literature recommended features, (b) 3 most stable features from each of the feature classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hard define features, REMOVE if I want to use the top 3 features\n",
    "featuresNames = [\"original_firstorder_Mean\", \"original_firstorder_Median\", \"original_firstorder_10Percentile\", \"original_firstorder_Variance\",\n",
    "                 \"original_firstorder_Kurtosis\", \"original_firstorder_Skewness\", \"original_glcm_JointEnergy\",\"original_glcm_JointEntropy\",\n",
    "                 \"original_glcm_Idm\", \"original_glcm_Correlation\", \"original_glcm_Contrast\", \"original_shape_Volume\"]\n",
    "\n",
    "plotTrendLines(\"BC\", \"BC+MRN\", 15, 2, False, \"T2AX\", \"TumorROI_PZ_1\", featuresNames, \"Fig18a__\")\n",
    "plotTrendLines(\"BC\", \"BC+MRN\", 15, 2, False, \"T2AX\", \"TumorROI_PZ_1\", None, \"Fig18b__\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 19 a + b\n",
    "Change in ICCs from T2w Tumor ROIs to registered T2w Tumor ROIs for (a) literature recommended features, (b) top 3 features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hard define features, REMOVE if I want to use the top 3 features\n",
    "featuresNames = [\"original_firstorder_Mean\", \"original_firstorder_Median\", \"original_firstorder_10Percentile\", \"original_firstorder_Variance\",\n",
    "                 \"original_firstorder_Kurtosis\", \"original_firstorder_Skewness\", \"original_glcm_JointEnergy\",\"original_glcm_JointEntropy\",\n",
    "                 \"original_glcm_Idm\", \"original_glcm_Correlation\", \"original_glcm_Contrast\", \"original_shape_Volume\"]\n",
    "\n",
    "plotTrendLines(\"BC\", \"BC+REG\", 15, 2, False, \"T2AX\", \"TumorROI_PZ_1\", featuresNames, \"Fig19a__\")\n",
    "plotTrendLines(\"BC\", \"BC+REG\", 15, 2, False, \"T2AX\", \"TumorROI_PZ_1\", None, \"Fig19b__\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convenience Function for Plotting Paired Cases Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pairedCasesPlot(ctype, structure, feature, imageType, normalized):\n",
    "  BIN = 15\n",
    "  DIM = 2\n",
    "  CTYPE = ctype  # ADC1400 T2AX\n",
    "  STRUCTURE = structure # TumorROI_PZ_1 PeripheralZone\n",
    "  FEATURE = feature    # \"original_firstorder_Mean\"\n",
    "\n",
    "  dfSelectedFeatures = dfFeaturesJointTP[(dfFeaturesJointTP[FEATURE+\"_t1\"].notnull()) &\n",
    "                                         (dfFeaturesJointTP[FEATURE+\"_t2\"].notnull()) &\n",
    "                                         (dfFeaturesJointTP[\"binwidth\"] == BIN) &\n",
    "                                         (dfFeaturesJointTP[\"dim\"] == DIM) &\n",
    "                                         (dfFeaturesJointTP[\"canonicalType\"] == CTYPE) &\n",
    "                                         (dfFeaturesJointTP[\"segmentedStructure\"] == STRUCTURE)]\n",
    "\n",
    "  # Select from \"ORIG\", \"BC\", \"BC+MRN\"\n",
    "  imageTypeLabelMap = {\n",
    "    \"ORIG\": \"Original\",\n",
    "    \"BC\": \"Bias Corrected\",\n",
    "    \"BC+MRN\": \"Bias Corrected + Muscle Ref Norm\",\n",
    "    \"BC+REG\": \"Bias Corrected + Registration\"\n",
    "  }\n",
    "  IMAGE_TYPE = imageType\n",
    "  NORM = normalized\n",
    "\n",
    "  dfSelectedFeatures = dfSelectedFeatures[(dfSelectedFeatures[\"baseImageType\"] == IMAGE_TYPE) &\n",
    "                                          (dfSelectedFeatures[\"normalized\"] == NORM)]\n",
    "\n",
    "  patNoLookup = { \"PCAMPMRI-00775\": 1  ,\n",
    "                  \"PCAMPMRI-00784\": 2  ,\n",
    "                  \"PCAMPMRI-00812\": 3  ,\n",
    "                  \"PCAMPMRI-00821\": 4  ,\n",
    "                  \"PCAMPMRI-00824\": 5  ,\n",
    "                  \"PCAMPMRI-00838\": 6  ,\n",
    "                  \"PCAMPMRI-00840\": 7  ,\n",
    "                  \"PCAMPMRI-00936\": 8  ,\n",
    "                  \"PCAMPMRI-00945\": 9  ,\n",
    "                  \"PCAMPMRI-00949\": 10 ,\n",
    "                  \"PCAMPMRI-00967\": 11 ,\n",
    "                  \"PCAMPMRI-00990\": 12 ,\n",
    "                  \"PCAMPMRI-01020\": 13 ,\n",
    "                  \"PCAMPMRI-01030\": 14 ,\n",
    "                  \"PCAMPMRI-01033\": 15 }\n",
    "\n",
    "  patientNames = dfSelectedFeatures[\"patient\"].values\n",
    "  fTp1 = dfSelectedFeatures[FEATURE+\"_t1\"].values\n",
    "  fTp2 = dfSelectedFeatures[FEATURE+\"_t2\"].values\n",
    "  n = fTp1.shape[0]\n",
    "  overallMean = np.mean([fTp1, fTp2])\n",
    "  pairwiseMean = np.mean([fTp1, fTp2], axis=0)\n",
    "  brms = np.sqrt(np.sum(np.square(pairwiseMean - overallMean)) / (n - 1))\n",
    "  wrms = np.sqrt(np.sum(np.square(fTp1 - pairwiseMean) + np.square(fTp2 - pairwiseMean)) / (2 * n))\n",
    "  #print(bms, wms)\n",
    "  featValues = []\n",
    "  featValues.append(fTp1)\n",
    "  featValues.append(pairwiseMean)\n",
    "  featValues.append(fTp2)\n",
    "  #featValues\n",
    "\n",
    "  plt.figure(figsize=(16,9))\n",
    "  sb.swarmplot(data=featValues, orient=\"h\")\n",
    "  for i in range(len(featValues[0])):\n",
    "    plt.plot([featValues[0][i], featValues[2][i]], [0, 2], linestyle=':', color=\"gray\")\n",
    "    plt.text(featValues[0][i], -0.05, str(patNoLookup[patientNames[i]]), fontsize=10, horizontalalignment='center', verticalalignment='bottom', color=\"gray\")\n",
    "    plt.text(featValues[2][i], 2.05, str(patNoLookup[patientNames[i]]), fontsize=10, horizontalalignment='center', verticalalignment='top', color=\"gray\")\n",
    "\n",
    "  plt.plot([overallMean, overallMean], [-0.2, 2.2], linestyle='-', color=\"gray\")\n",
    "  plt.text(overallMean, -0.3, \"Overall Mean\", fontsize=14, horizontalalignment='center', verticalalignment='center', color=\"gray\")\n",
    "\n",
    "  plt.plot([overallMean - (0.5 * brms), overallMean + (0.5 * brms)], [2.3, 2.3], linestyle='-', color=\"orange\")\n",
    "  plt.plot([overallMean - (0.5 * wrms), overallMean + (0.5 * wrms)], [2.4, 2.4], linestyle='-', color=\"gray\")\n",
    "  x = overallMean + (0.6 * max(wrms,brms))\n",
    "  plt.text(x, 2.3, \"between cases rmse\", fontsize=14, horizontalalignment='left', verticalalignment='center', color=\"orange\")\n",
    "  plt.text(x, 2.4, \"within cases rmse\", fontsize=14, horizontalalignment='left', verticalalignment='center', color=\"gray\")\n",
    "  plt.title(CTYPE + \" \" + STRUCTURE + \" \" + FEATURE + \" \" + IMAGE_TYPE + \"-normalized=\" + str(NORM))\n",
    "\n",
    "\n",
    "\n",
    "  plt.yticks([0,1,2], [\"TP 1\", \"PairMean\", \"TP 2\"])\n",
    "  #plt.savefig(os.path.join(os.getcwd(), \"EvalData\", \"plots\", \"PairedCases_\" + CTYPE + \"_\" + STRUCTURE + \"_\" + FEATURE + \"_\" + IMAGE_TYPE + \"-normed-\" + str(NORM) + \".png\"), bbox_inches='tight')\n",
    "  #plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Paired Cases Plots (not used in Paper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pairedCasesPlot(\"ADC1400\", \"PeripheralZone\", \"original_firstorder_Mean\", \"ORIG\", False)\n",
    "pairedCasesPlot(\"ADC1400\", \"PeripheralZone\", \"original_firstorder_Mean\", \"ORIG\", True)\n",
    "pairedCasesPlot(\"ADC1400\", \"PeripheralZone\", \"original_glcm_Contrast\", \"ORIG\", False)\n",
    "pairedCasesPlot(\"ADC1400\", \"PeripheralZone\", \"original_glcm_Contrast\", \"ORIG\", True)\n",
    "pairedCasesPlot(\"ADC1400\", \"PeripheralZone\", \"original_glcm_Idm\", \"ORIG\", False)\n",
    "pairedCasesPlot(\"ADC1400\", \"PeripheralZone\", \"original_glcm_Idm\", \"ORIG\", True)\n",
    "\n",
    "pairedCasesPlot(\"ADC1400\", \"TumorROI_PZ_1\", \"original_firstorder_Mean\", \"ORIG\", False)\n",
    "pairedCasesPlot(\"ADC1400\", \"TumorROI_PZ_1\", \"original_firstorder_Mean\", \"ORIG\", True)\n",
    "pairedCasesPlot(\"ADC1400\", \"TumorROI_PZ_1\", \"original_glcm_Idm\", \"ORIG\", False)\n",
    "pairedCasesPlot(\"ADC1400\", \"TumorROI_PZ_1\", \"original_glcm_Idm\", \"ORIG\", True)\n",
    "\n",
    "pairedCasesPlot(\"T2AX\", \"PeripheralZone\", \"original_firstorder_Mean\", \"ORIG\", False)\n",
    "pairedCasesPlot(\"T2AX\", \"PeripheralZone\", \"original_firstorder_Mean\", \"ORIG\", True)\n",
    "\n",
    "pairedCasesPlot(\"T2AX\", \"TumorROI_PZ_1\", \"original_firstorder_Mean\", \"ORIG\", False)\n",
    "pairedCasesPlot(\"T2AX\", \"TumorROI_PZ_1\", \"original_firstorder_Mean\", \"ORIG\", True)\n",
    "\n",
    "pairedCasesPlot(\"T2AX\", \"TumorROI_PZ_1\", \"original_glcm_Contrast\", \"BC+MRN\", False)\n",
    "pairedCasesPlot(\"T2AX\", \"TumorROI_PZ_1\", \"original_glcm_Contrast\", \"BC+REG\", False)\n",
    "pairedCasesPlot(\"T2AX\", \"TumorROI_PZ_1\", \"original_glcm_Contrast\", \"BC\", False)\n",
    "pairedCasesPlot(\"T2AX\", \"TumorROI_PZ_1\", \"original_glcm_Contrast\", \"ORIG\", False)\n",
    "pairedCasesPlot(\"T2AX\", \"TumorROI_PZ_1\", \"original_glcm_Contrast\", \"ORIG\", True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Correlation Matrix Plot (not used in Paper)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BIN = 15\n",
    "DIM = 2\n",
    "CTYPE = \"T2AX\"  # ADC1400 T2AX\n",
    "STRUCTURE = \"PeripheralZone\" # TumorROI_PZ_1 PeripheralZone\n",
    "IMAGE_TYPE = \"BC\"\n",
    "NORM = True\n",
    "FILTER = \"original\"\n",
    "\n",
    "dfSelectedFeatures = dfFeaturesJointTP[(dfFeaturesJointTP[\"binwidth\"] == BIN) &\n",
    "                                       (dfFeaturesJointTP[\"dim\"] == DIM) &\n",
    "                                       (dfFeaturesJointTP[\"canonicalType\"] == CTYPE) &\n",
    "                                       (dfFeaturesJointTP[\"segmentedStructure\"] == STRUCTURE) &\n",
    "                                       (dfFeaturesJointTP[\"baseImageType\"] == IMAGE_TYPE) &\n",
    "                                       (dfFeaturesJointTP[\"normalized\"] == NORM)]\n",
    "\n",
    "\n",
    "\n",
    "featuresNames = [\"firstorder_Mean\", \"firstorder_Median\", \"firstorder_10Percentile\", \"firstorder_Variance\",\n",
    "                 \"firstorder_Kurtosis\", \"firstorder_Skewness\", \"glcm_JointEnergy\",\"glcm_JointEntropy\",\n",
    "                 \"glcm_Idm\", \"glcm_Correlation\", \"glcm_Contrast\", \"shape_Volume\"]\n",
    "featuresNames = [\"firstorder_Mean\", \"firstorder_Median\", \"firstorder_10Percentile\", \"firstorder_Variance\",\n",
    "                 \"firstorder_Kurtosis\", \"firstorder_Skewness\"]\n",
    "featuresNames2 = [\"glcm_JointEnergy\",\"glcm_JointEntropy\",\n",
    "                 \"glcm_Idm\", \"glcm_Correlation\", \"glcm_Contrast\", \"shape_Volume\"]\n",
    "\n",
    "featuresNames = [FILTER + \"_\" + f for f in featuresNames]\n",
    "ft1 = [f + \"_t1\" for f in featuresNames]\n",
    "ft2 = [f + \"_t2\" for f in featuresNames]\n",
    "featuresNames = ft1 + ft2\n",
    "\n",
    "corr = dfSelectedFeatures[featuresNames].corr()\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "fig.set_size_inches(16, 16)\n",
    "sb.heatmap(corr, annot=True, linewidths=.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}