04_experiment-dim-reduction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dimensionality reduction\n",
    "========"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import glob, os, os.path, sys, warnings, math, time, re\n",
    "# warnings.filterwarnings('ignore')\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import Imputer, StandardScaler, LabelEncoder\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import silhouette_score\n",
    "\n",
    "from techniques import sortedDatasets\n",
    "from dimred import trustworthiness, continuity\n",
    "\n",
    "from metric_learn import LMNN, NCA, LFDA, Covariance, MetricEvolution, NeuralNetworkTransformer, FullMatrixTransformer\n",
    "from metric_learn import ITML_Supervised, SDML_Supervised, LSML_Supervised, RCA_Supervised\n",
    "ME = MetricEvolution\n",
    "\n",
    "%matplotlib inline\n",
    "from plotting.plots import *\n",
    "\n",
    "datasetsDirectory = 'datasets'\n",
    "resultsDirectory = 'results/dim-reduction'\n",
    "graphsDirectory = 'img/dim-reduction'\n",
    "\n",
    "if not os.path.exists(resultsDirectory):\n",
    "    os.makedirs(resultsDirectory)\n",
    "    \n",
    "if not os.path.exists(graphsDirectory):\n",
    "    os.makedirs(graphsDirectory)\n",
    "\n",
    "np.set_printoptions(formatter={'float': lambda x: \"{0:0.5f}\".format(x)})\n",
    "\n",
    "import pickle\n",
    "def save_obj(obj, name):\n",
    "    with open('{}/{}.pkl'.format(resultsDirectory, name), 'wb') as f:\n",
    "        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "def load_obj(name):\n",
    "    with open('{}/{}.pkl'.format(resultsDirectory, name), 'rb') as f:\n",
    "        return pickle.load(f)\n",
    "    \n",
    "def exists_obj(name):\n",
    "    return os.path.exists('{}/{}.pkl'.format(resultsDirectory, name))\n",
    "\n",
    "def gfn(filename, folder='dimred'):\n",
    "    odir = '../thesis-distance-metric-learning/graphs/{}'.format(folder)\n",
    "    if not os.path.exists(odir):\n",
    "        os.makedirs(odir)\n",
    "    return '{}/{}'.format(odir, filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import glob, os\n",
    "\n",
    "datasets = []\n",
    "for file in glob.glob(\"{}/*.csv\".format(datasetsDirectory)):\n",
    "    datasets.append(file)\n",
    "datasets.sort()\n",
    "\n",
    "for x in datasets:\n",
    "    print(x, pd.read_csv(x, sep=',', skiprows=1, header=0).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "skipDone = False\n",
    "def makeGraphBuilder(datasetName, X_train, X_test, y_train, y_test, perRow=4):\n",
    "    scores = []\n",
    "    \n",
    "    imputer = Imputer(missing_values='NaN', strategy='mean', axis=0, verbose=0, copy=False)\n",
    "    X_train = imputer.fit_transform(X_train)\n",
    "    X_test = imputer.transform(X_test)\n",
    "    \n",
    "    sscaler = StandardScaler()\n",
    "    X_train = sscaler.fit_transform(X_train)\n",
    "    X_test = sscaler.transform(X_test)\n",
    "    \n",
    "    def add(methodName, model, canTransform=True):\n",
    "        filename = '{}__{}'.format(datasetName, methodName)\n",
    "        if skipDone and exists_obj(filename):\n",
    "            print('skipping {}, already exists'.format(filename))\n",
    "            return\n",
    "\n",
    "        previousRun = None\n",
    "        scores = []\n",
    "        if exists_obj(filename):\n",
    "            previousRun = load_obj(filename)\n",
    "            if 'scores' not in previousRun:\n",
    "                previousRun['scores'] = []\n",
    "            scores = previousRun['scores']\n",
    "\n",
    "        print('previous scores: {}'.format(scores))\n",
    "        if len(scores)>=10:\n",
    "            print('ALREADY HAVE 10 RUNS, FINISHED {}'.format(methodName))\n",
    "            return\n",
    "        \n",
    "        try:\n",
    "            if canTransform: # TSNE cant transform\n",
    "                Xt_train = model.fit_transform(X_train, y_train)\n",
    "                Xt_test = model.transform(X_test)\n",
    "            else:\n",
    "                Xt = model.fit_transform(np.concatenate((X_train, X_test)), np.concatenate((y_train, y_test)))\n",
    "                l = len(X_train)\n",
    "                Xt_train, Xt_test = Xt[:l], Xt[l:]\n",
    "\n",
    "            knn = KNeighborsClassifier(n_neighbors=4, n_jobs=-1)\n",
    "            knn.fit(Xt_train, y_train)\n",
    "\n",
    "            score = knn.score(Xt_test, y_test)\n",
    "            wrong = knn.predict(Xt_test) != y_test\n",
    "            \n",
    "            print('{} has {}%'.format(methodName, score*100))\n",
    "            \n",
    "            if (previousRun is None) or score>previousRun['score']:\n",
    "                print('SAVING BETTER {} {}'.format(datasetName, methodName))\n",
    "                data = {\n",
    "                    'scores': scores+[score],\n",
    "                    'score': score,\n",
    "                    'X_train': Xt_train,\n",
    "                    'y_train': y_train,\n",
    "                    'X_test': Xt_test,\n",
    "                    'y_test': y_test,\n",
    "                    'wrong': wrong,\n",
    "                }\n",
    "            else:\n",
    "                previousRun['scores'].append(score)\n",
    "                data = previousRun\n",
    "                \n",
    "            save_obj(data, filename)\n",
    "        except:\n",
    "            import traceback\n",
    "            traceback.print_exc()\n",
    "#             raise\n",
    "\n",
    "    return add\n",
    "\n",
    "def P(methods):\n",
    "    def c(**kwargs):\n",
    "        return Pipeline([(x.__name__.lower(),x()) for x in methods])\n",
    "    return c\n",
    "\n",
    "def P(methods):\n",
    "    def c(**kwargs):\n",
    "        return Pipeline([(type(x).__name__.lower(),x) for x in methods])\n",
    "    return c()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sortedDatasetsBySize = [\n",
    " 'datasets/balance-scale.csv',\n",
    " 'datasets/breast-cancer.csv',\n",
    " 'datasets/gaussians.csv',\n",
    " 'datasets/iris.csv',\n",
    " 'datasets/pima-indians.csv',\n",
    " 'datasets/wine.csv',\n",
    "\n",
    " 'datasets/sonar.csv',\n",
    " 'datasets/mice-protein.csv',\n",
    " 'datasets/digits10.csv',\n",
    " 'datasets/digits6.csv',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for filename in sortedDatasetsBySize:#['datasets/balance-scale.csv', 'datasets/gaussians.csv']:#\n",
    "    results = []\n",
    "    datasetName = filename[len(datasetsDirectory)+1:-4]\n",
    "\n",
    "    data = pd.read_csv(filename, sep=',', skiprows=1, header=0)\n",
    "    y = data['class']\n",
    "    X = data.drop(['class'], axis=1).values\n",
    "\n",
    "    le = LabelEncoder()\n",
    "    y = le.fit_transform(y)\n",
    "\n",
    "#     X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)\n",
    "    \n",
    "    skf = StratifiedKFold(n_splits=10, shuffle=True, random_state=42)\n",
    "    for train_index, test_index in skf.split(X, y):\n",
    "        X_train, X_test = X[train_index], X[test_index]\n",
    "        y_train, y_test = y[train_index], y[test_index]\n",
    "\n",
    "        g = makeGraphBuilder(datasetName, X_train, X_test, y_train, y_test, perRow=3)\n",
    "\n",
    "        defaults = {\n",
    "    #         'transformer':'neuralnetwork',\n",
    "    #         't__layers':(2,),\n",
    "\n",
    "            'n_gen': 200,\n",
    "    #         'class_separation': True,\n",
    "    #         'verbose': False,\n",
    "    #         'random_state': 41,\n",
    "        }\n",
    "\n",
    "        print(datasetName)\n",
    "\n",
    "        start = time.clock()\n",
    "\n",
    "    #     g('pca', PCA(n_components=2))\n",
    "    #     g('tsne', TSNE(n_components=2)) # , 'perplexity':50\n",
    "    #     g('lfda', LFDA(num_dims=2, k=2))\n",
    "    #     g('cmaes', ME(t__n_components=2))\n",
    "\n",
    "        g('normpca', P([PCA(n_components=2) ]))\n",
    "        g('normlfda', P([LFDA(num_dims=2, k=2), StandardScaler() ]))\n",
    "        g('normnca', P([NCA(num_dims=2) ]))\n",
    "        g('normtsne', P([TSNE(n_components=2) ]) ,canTransform=False)\n",
    "        g('normcmaes', P([ME(**defaults, t__n_components=2) ]))\n",
    "        g('normcmaestsne', P([ME(**defaults, ), TSNE(n_components=2) ]), canTransform=False)\n",
    "\n",
    "    #     g('nnnone', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=(16,8,4,2,), activation=None)) ]))\n",
    "        deepL = (16,12,8,6,4,2,)\n",
    "        g('nnrelu', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=deepL, activation='relu')) ]))\n",
    "        g('nnsigm', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=deepL, activation='sigm')) ]))\n",
    "        g('nntanh', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=deepL, activation='tanh')) ]))\n",
    "\n",
    "        shallowL = (4,2,)\n",
    "        g('nnsigmshallow', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=shallowL, activation='sigm')), StandardScaler() ]))\n",
    "    #     g('nnsigmdeep', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=(2,2,2,2,2,2,2,2,2,2,), activation='sigm')), StandardScaler() ]))\n",
    "\n",
    "        g('nntanhshallow', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=shallowL, activation='tanh')), StandardScaler() ]))\n",
    "    #     g('nntanhdeep', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=(2,2,2,2,2,2,2,2,2,2,), activation='tanh')), StandardScaler() ]))\n",
    "\n",
    "        g('nnrelushallow', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=shallowL, activation='relu')), StandardScaler() ]))\n",
    "    #     g('nnreludeep', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=(2,2,2,2,2,2,2,2,2,2,), activation='relu')), StandardScaler() ]))\n",
    "\n",
    "    #     g('nnnoneshallow', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=shallowL, activation=None)), StandardScaler() ]))\n",
    "    #     g('nnnonedeep', P([ME(**defaults, transformer=NeuralNetworkTransformer(layers=(2,2,2,2,2,2,2,2,2,2,), activation=None)), StandardScaler() ]))\n",
    "\n",
    "#     end = time.clock()\n",
    "#     print(end - start)\n",
    "#     start = time.clock()\n",
    "\n",
    "#     g(ME, {**defaults, 'classifier':'knn','c__n_neighbors':1,'c__n_jobs':-1,'c__weights':'uniform'})\n",
    "#     g(ME, {**defaults, 'classifier':'svc',})\n",
    "#     g(ME, {**defaults, 'classifier':'lsvc','c__dual':False,})\n",
    "#     g(ME, {**defaults, 'transformer':NeuralNetworkTransformer(layers=(2,2,), activation='tanh'), 'c__n_neighbors':1, })\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,2,)), 'n_neighbors':1, 'n_gen':n_gen,'class_separation':0, 'verbose':verbose})\n",
    "\n",
    "#     end = time.clock()\n",
    "#     print(end - start)\n",
    "#     start = time.clock()\n",
    "\n",
    "#     good_transformer = 'full'\n",
    "#     if X.shape[1]>15:\n",
    "#         good_transformer = NeuralNetworkTransformer(layers=(10,8,6,4,))\n",
    "#         good_transformer = 'diagonal'\n",
    "\n",
    "#     g(P([ME, PCA]), {'pca__n_components':2, 'metricevolution__transformer': good_transformer})\n",
    "#     g(P([ME, TSNE]), {'tsne__n_components':2, 'tsne__perplexity': 30, 'metricevolution__transformer': good_transformer})\n",
    "#     if False and X.shape[1]>7:\n",
    "#         g(ME, {'transformer': NeuralNetworkTransformer(layers=(10,8,6,4,2))})\n",
    "#     else:\n",
    "#     g(P([ME, TSNE]), {'tsne__n_components':2, 'tsne__perplexity': 30, \n",
    "#                       'metricevolution__transformer': 'full', 'metricevolution__evolution_strategy':'dde', 'metricevolution__verbose':True, \n",
    "#                       'metricevolution__fitnesses':('class_separation',)})\n",
    "\n",
    "#     end = time.clock()\n",
    "#     print(end - start)\n",
    "#     start = time.clock()\n",
    "\n",
    "#     g(P([ME, TSNE]), {'tsne__n_components':2, 'tsne__perplexity': 30, \n",
    "#                       'metricevolution__transformer': 'full', 'metricevolution__evolution_strategy':'de', 'metricevolution__verbose':True})\n",
    "\n",
    "#     end = time.clock()\n",
    "#     print(end - start)\n",
    "#     start = time.clock()\n",
    "\n",
    "#     g(P([ME, TSNE]), {'tsne__n_components':2, 'tsne__perplexity': 30, \n",
    "#                       'metricevolution__transformer': 'full', 'metricevolution__evolution_strategy':'cmaes', 'metricevolution__verbose':True})\n",
    "\n",
    "#     end = time.clock()\n",
    "#     print(end - start)\n",
    "#     start = time.clock()\n",
    "\n",
    "#     g(ME, {'transformer': 'full', 't__n_components': 2, 'evolution_strategy':'metricevolution'})\n",
    "#     g(ME, {'transformer': 'full', 't__n_components': 2, 'evolution_strategy':'metricevolution'})\n",
    "#     g(ME, {'transformer': 'kmeans', 't__transformer': 'full'})\n",
    "#     g(ME, {'transformer': 'kmeans', 't__transformer': 'diagonal', 't__n_clusters':2})\n",
    "#     g(ME, {'transformer': 'kmeans', 't__transformer': 'diagonal', 't__n_clusters':2, 't__function':'product'})\n",
    "#     g(P([ME, TSNE]), {'tsne__n_components':2, 'tsne__perplexity': 30, 'metricevolution__transformer': 'kmeans', 'metricevolution__t__transformer':'diagonal', 'metricevolution__t__n_classes': 'same'})\n",
    "#     g(P([ME, TSNE]), {'tsne__n_components':2, 'tsne__perplexity': 30, 'metricevolution__transformer': 'kmeans', 'metricevolution__t__transformer':'diagonal', 'metricevolution__t__n_classes': 'same', 'metricevolution__t__function':'product'})\n",
    "\n",
    "#     end = time.clock()\n",
    "#     print(end - start)\n",
    "#     start = time.clock()\n",
    "\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,)), 'n_neighbors':1, 'n_gen':n_gen,'classifier':'svm', 'verbose':verbose})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,)), 'n_neighbors':1, 'n_gen':n_gen,'classifier':'svm', 'verbose':verbose})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,2,)), 'n_neighbors':1, 'n_gen':n_gen,'classifier':'svm', 'verbose':verbose})\n",
    "\n",
    "#     end = time.clock()\n",
    "#     print(end - start)\n",
    "#     start = time.clock()\n",
    "\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,)), 'n_neighbors':1, 'n_gen':n_gen,'classifier':'lsvm', 'verbose':verbose})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,)), 'n_neighbors':1, 'n_gen':n_gen,'classifier':'lsvm', 'verbose':verbose})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,2,)), 'n_neighbors':1, 'n_gen':n_gen,'classifier':'lsvm', 'verbose':verbose})\n",
    "\n",
    "#     end = time.clock()\n",
    "#     print(end - start)\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,), activation='tanh'), 'n_neighbors':1, 'n_gen':n_gen,'class_separation':0, 'verbose':verbose})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,2,), activation='tanh'), 'n_neighbors':1, 'n_gen':n_gen,'class_separation':0, 'verbose':verbose})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(4,3,2,2,), activation='tanh'), 'n_neighbors':1, 'n_gen':n_gen,'class_separation':0, 'verbose':verbose})\n",
    "\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,), activation='none'), 'n_neighbors':1, 'n_gen':n_gen,'class_separation':0, 'verbose':verbose})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,2,), activation='none'), 'n_neighbors':1, 'n_gen':n_gen,'class_separation':0, 'verbose':verbose})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(4,3,2,2,), activation='none'), 'n_neighbors':1, 'n_gen':n_gen,'class_separation':0, 'verbose':verbose})\n",
    "\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,)), 'n_neighbors':4, 'n_gen':n_gen,'class_separation':0, 'verbose':True})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,)), 'n_neighbors':4, 'n_gen':n_gen,'class_separation':0, 'verbose':True})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,2,), activation='relu'), 'n_neighbors':4, 'n_gen':n_gen,'class_separation':0, 'verbose':True})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,2,2,2,)), 'n_neighbors':4, 'n_gen':n_gen,'class_separation':0, 'verbose':True})\n",
    "\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,)), 'n_neighbors':8, 'n_gen':n_gen,'class_separation':1, 'verbose':True})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(4,3,2,)), 'n_neighbors':8, 'n_gen':n_gen,'class_separation':1, 'verbose':True})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(5,4,3,2,)), 'n_neighbors':8, 'n_gen':n_gen,'class_separation':1, 'verbose':True})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(16,8,4,2,)), 'n_neighbors':8, 'n_gen':n_gen,'class_separation':1, 'verbose':True})\n",
    "\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(2,)), 'n_neighbors':8, 'n_gen':n_gen,'class_separation':-1, 'verbose':True})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(4,3,2,)), 'n_neighbors':8, 'n_gen':n_gen,'class_separation':-1, 'verbose':True})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(5,4,3,2,)), 'n_neighbors':8, 'n_gen':n_gen,'class_separation':-1, 'verbose':True})\n",
    "#     g(ME, {'transformer':NeuralNetworkTransformer(layers=(16,8,4,2,)), 'n_neighbors':8, 'n_gen':n_gen,'class_separation':-1, 'verbose':True})\n",
    "\n",
    "#     break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "methodTitles = [\n",
    "    ('normpca', 's:PCA'),\n",
    "    ('normnca', 's:NCA'),\n",
    "    ('normlfda', 's:LFDA'),\n",
    "\n",
    "    ('normcmaes', 's:CMAES.kNN'),\n",
    "    ('normtsne', 's:t-SNE'),\n",
    "    ('normcmaestsne', 's:CMAES.kNN+t-SNE'),\n",
    "    \n",
    "    ('nnsigmshallow', 's:NN.Sigm (shallow)'),\n",
    "    ('nntanhshallow', 's:NN.Tanh (shallow)'),\n",
    "    ('nnrelushallow', 's:NN.ReLU (shallow)'),\n",
    "\n",
    "    ('nnsigm', 's:NN.Sigmoid (deep)'),\n",
    "    ('nntanh', 's:NN.Tanh (deep)'),\n",
    "    ('nnrelu', 's:NN.ReLU (deep)'),\n",
    "]\n",
    "\n",
    "results = []\n",
    "for file in glob.glob(\"{}/*.pkl\".format(resultsDirectory)):\n",
    "    results.append(file)\n",
    "results.sort()\n",
    "\n",
    "resultsByDataset = {}\n",
    "\n",
    "for x in results:\n",
    "    _,_,filename = re.split('/|\\\\\\\\', x)\n",
    "    datasetName,methodName = filename[:-4].split('__')\n",
    "    \n",
    "    if datasetName not in resultsByDataset:\n",
    "        resultsByDataset[datasetName] = {}\n",
    "\n",
    "    resultsByDataset[datasetName][methodName] = load_obj(filename[:-4])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Graphs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for datasetName, alldata in resultsByDataset.items():\n",
    "#     if datasetName not in ['wine']: continue\n",
    "#     N = sum([1 if x in alldata else 0 for x,y in methodTitles])\n",
    "    N = len(methodTitles)\n",
    "    \n",
    "    i = 0\n",
    "    cols = 3\n",
    "    fig, axes = startGraphing('`{}` dataset'.format(datasetName), cols, N, size=(8, 3.2*(N//cols)))\n",
    "    for method, title in methodTitles:\n",
    "        \n",
    "        if method not in alldata:\n",
    "            plotEmpty(axes[i], title, 'Memory Error', hideAxis=True)\n",
    "            i += 1\n",
    "            continue\n",
    "        \n",
    "        data = alldata[method]\n",
    "        plotScatter(axes[i],title,**data, scoreIsAproximation='t-SNE' in title)\n",
    "        i += 1\n",
    "    endGraphing(fig, filename=gfn('{}'.format(datasetName), folder='dimred'), move_title=.93)\n",
    "    plt.subplots_adjust(wspace=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Error rates table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def printCell(x, y, best, bestNoTsne):\n",
    "    return \" & {}\".format(printEmph(printBold(printNumbers(x, y), best), bestNoTsne))\n",
    "# & \\emph{\\textbf{44.26\\,$\\pm$13.07}}\n",
    "\n",
    "\n",
    "def printBold(text, bold):\n",
    "    if bold:\n",
    "        return \"\\\\textbf{{{}}}\".format(text)\n",
    "    return text\n",
    "\n",
    "def printEmph(text, emph):\n",
    "    if emph:\n",
    "        return \"\\\\emph{{{}}}\".format(text)\n",
    "    return text\n",
    "    \n",
    "def printNumbers(x,y):\n",
    "    return \"{:.2f}\\,$\\pm$\\,{:.2f}\".format(x,y)\n",
    "    \n",
    "def printError():\n",
    "    return \" & Error \".format()\n",
    "\n",
    "def printEmpty():\n",
    "    return \" & Timeout \".format()\n",
    "\n",
    "def printErrorTable(datasets, results):\n",
    "    for dataset in datasets:\n",
    "        print(\"& \\\\multicolumn{{1}}{{c}}{{{}}} \".format(dataset), end='')\n",
    "\n",
    "    print('\\\\\\\\ \\n\\\\midrule')\n",
    "\n",
    "    for i, (techniqueIndex, technique) in enumerate(methodTitles):\n",
    "        print(technique, end='')\n",
    "\n",
    "        for dataset in datasets:\n",
    "            \n",
    "            errors = [100.0]+[100*(1-np.average(y['scores'])) for x,y in results[dataset].items() if 'scores' in y]\n",
    "            beste = min(errors)\n",
    "            \n",
    "            errorsNoTsne = [100.0]+[100*(1-np.average(y['scores'])) for x,y in results[dataset].items() if 'scores' in y and 'tsne' not in x]\n",
    "            besteNoTsne = min(errorsNoTsne)\n",
    "            \n",
    "#             print(len(errors), len(errorsNoTsne))\n",
    "            \n",
    "            re = 'E'\n",
    "            if techniqueIndex in results[dataset] and 'scores' in results[dataset][techniqueIndex]:\n",
    "                data = results[dataset][techniqueIndex]\n",
    "                re = 100*(1-np.average(data['scores']))\n",
    "                rs = 100*np.std(data['scores'])\n",
    "\n",
    "            if re=='E':\n",
    "                print(printError(), end='')\n",
    "            elif math.isnan(re):\n",
    "                print(printEmpty(), end='')\n",
    "            else:\n",
    "#                 print('{:.2f}'.format(re), '{:.2f}'.format(beste))\n",
    "                print(printCell(re, rs, \n",
    "                                best='{:.2f}'.format(re)=='{:.2f}'.format(beste),\n",
    "                                bestNoTsne='{:.2f}'.format(re)=='{:.2f}'.format(besteNoTsne)\n",
    "                               ), end='')\n",
    "        print(' \\\\\\\\')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "printErrorTable(sortedDatasets[:5], resultsByDataset)\n",
    "print('\\\\midrule')\n",
    "printErrorTable(sortedDatasets[5:], resultsByDataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Statistical significance table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.stats import ttest_ind\n",
    "\n",
    "def stars(p, error=False):\n",
    "    if error: return \"?\"\n",
    "    if p < 0.0001: return \"****\"\n",
    "    elif (p < 0.001): return \"***\"\n",
    "    elif (p < 0.01): return \"**\"\n",
    "    elif (p < 0.05): return \"*\"\n",
    "    else: return \".\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from techniques import sortedDatasets\n",
    "\n",
    "for dataset in sortedDatasets:\n",
    "    print(\"\"\"\\\\begin{table}[ht] \\\\centering\n",
    "{\\\\small\\\\renewcommand{\\\\arraystretch}{0.95}\n",
    "\\\\setlength{\\\\tabcolsep}{1pt}\n",
    "\\\\begin{tabular}{rC{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}}\n",
    "\\\\toprule\"\"\")\n",
    "\n",
    "#     print('\\shortstack{{{} \\\\\\\\ dataset}}'.format(dataset), end='')\n",
    "    for i,T in methodTitles:\n",
    "    #     print(\"& \\\\rot{{{}}} \".format(dataset), end='')\n",
    "    #     print(\"& {} \".format(dataset), end='')\n",
    "        print(\" & \\\\rot{{{}}}\".format(T), end='')\n",
    "\n",
    "    print(' \\\\\\\\ \\\\midrule')\n",
    "    for T1,technique in methodTitles:\n",
    "        print(\"{}\".format(technique), end='')\n",
    "        for T2,_ in methodTitles:\n",
    "            try:\n",
    "                A = resultsByDataset[dataset][T1]['scores']\n",
    "                B = resultsByDataset[dataset][T2]['scores']\n",
    "                t,p = ttest_ind(A,B)\n",
    "                if t < 0: p = 1-p\n",
    "                hasErrors = False\n",
    "            except:\n",
    "                hasErrors = True\n",
    "\n",
    "            print(\" & {}\".format(stars(p, error=hasErrors)), end='')\n",
    "        print(' \\\\\\\\')\n",
    "\n",
    "    print(\"\"\"\\\\bottomrule\n",
    "\\multicolumn{{10}}{{l}}{{**** $p<0.0001$, *** $p<0.001$, ** $p<0.01$, * $p<0.05$}}\n",
    "\\\\end{{tabular}} }}\n",
    "\\\\caption{{Statistical significance for the `{}` dataset in the dimensionality reduction experiment}} \\\\label{{tab:statsign:dimred:{}}}\n",
    "\\\\end{{table}}\"\"\".format(dataset,dataset))\n",
    "    print()\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from techniques import sortedDatasets\n",
    "\n",
    "for dataset in sortedDatasets:\n",
    "    data = pd.read_csv('datasets/{}.csv'.format(dataset), sep=',', skiprows=1, header=0)\n",
    "#     y = data['class']\n",
    "    X = data.drop(['class'], axis=1).values\n",
    "    \n",
    "#     print(\"\"\"\\\\begin{table}[ht] \\\\centering\n",
    "# {\\\\small\\\\renewcommand{\\\\arraystretch}{0.95}\n",
    "# \\\\setlength{\\\\tabcolsep}{1pt}\n",
    "# \\\\begin{tabular}{rC{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}C{2em}}\n",
    "# \\\\toprule\"\"\")\n",
    "\n",
    "#     print('\\shortstack{{{} \\\\\\\\ dataset}}'.format(dataset), end='')\n",
    "    for i,T in methodTitles:\n",
    "        pass\n",
    "    #     print(\"& \\\\rot{{{}}} \".format(dataset), end='')\n",
    "    #     print(\"& {} \".format(dataset), end='')\n",
    "#         print(\" & \\\\rot{{{}}}\".format(T), end='')\n",
    "\n",
    "#     print(' \\\\\\\\ \\\\midrule')\n",
    "    for T1,technique in methodTitles:\n",
    "#         print(\"{}\".format(technique), end='')\n",
    "        try:\n",
    "            X_train = resultsByDataset[dataset][T1]['X_train']\n",
    "            X_test = resultsByDataset[dataset][T1]['X_test']\n",
    "\n",
    "            S1 = silhouette_score(X_train, resultsByDataset[dataset][T1]['y_train'])\n",
    "            S2 = silhouette_score(X_test, resultsByDataset[dataset][T1]['y_test'])\n",
    "            print(dataset, technique, S1, S2)\n",
    "\n",
    "#             [T] = trustworthiness(X, np.concatenate((X_train, X_test)), [4])\n",
    "#             [C] = continuity(X, np.concatenate((X_train, X_test)), [4])\n",
    "#             print(dataset, technique, T, C, 2*T*C/(T+C))\n",
    "\n",
    "#             hasErrors = False\n",
    "        except:\n",
    "            hasErrors = True\n",
    "\n",
    "#             print(\" & {}\".format(stars(p, error=hasErrors)), end='')\n",
    "#         print(' \\\\\\\\')\n",
    "\n",
    "#     print(\"\"\"\\\\bottomrule\n",
    "# \\multicolumn{{10}}{{l}}{{**** $p<0.0001$, *** $p<0.001$, ** $p<0.01$, * $p<0.05$}}\n",
    "# \\\\end{{tabular}} }}\n",
    "# \\\\caption{{Statistical significance for the `{}` dataset in the dimensionality reduction experiment}} \\\\label{{tab:statsign:dimred:{}}}\n",
    "# \\\\end{{table}}\"\"\".format(dataset,dataset))\n",
    "#     print()\n",
    "#     print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}