Examples/CimeExample.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First we need to import the packages and functions needed for this analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################\n",
    "### import the used functions\n",
    "#####################################\n",
    "#for umap you need the umap-learn package\n",
    "import umap.umap_ as umap\n",
    "from rdkit import Chem\n",
    "from rdkit.Chem import AllChem, DataStructs, Descriptors\n",
    "from rdkit.Chem import rdMolDescriptors\n",
    "from rdkit.Chem import PandasTools\n",
    "import pandas as pd\n",
    "import catboost\n",
    "import shap\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After this we will define several functions needed for our analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#######################################\n",
    "###define the used functions\n",
    "#######################################\n",
    "\n",
    "#define a function that trnasforms smiles RDKit Mol objects\n",
    "def smiles_to_mols(smis):\n",
    "    \"\"\"Convert a list of smiles to a list of RDKit Mol objects\"\"\"\n",
    "    if not isinstance(smis, list):\n",
    "        raise TypeError(\"Please provide smiles as a list\")\n",
    "    mols = []\n",
    "    successful_inds = []\n",
    "    for ind, smi in enumerate(smis):\n",
    "        # RDKit will sometimes 'fail' but instead of throwing an\n",
    "        # error it will (sometimes!) print a warning and return None.\n",
    "        m = Chem.MolFromSmiles(smi, sanitize=True)\n",
    "        if m is None:\n",
    "            print(\"Mol generation failed for\", smi, \"(index\", ind, \")\\n\")\n",
    "        else:\n",
    "            mols.append(m)\n",
    "            successful_inds.append(ind)\n",
    "    return mols, successful_inds\n",
    "\n",
    "\n",
    "# define a function to calculate count based morgan fingerprints\n",
    "def calc_morgan_counts_for_smiles(mol, radius=2, nBits=2048):\n",
    "    try:\n",
    "        fpCounts = AllChem.GetHashedMorganFingerprint(\n",
    "            mol, radius, nBits=nBits, includeRedundantEnvironments=False)\n",
    "        array = np.zeros((0,), dtype=np.int8)\n",
    "        DataStructs.ConvertToNumpyArray(fpCounts, array)\n",
    "    except:\n",
    "        print(\"Failure in calc_morgan_counts_for_smiles()\")\n",
    "        array = np.repeat(np.nan, nBits)\n",
    "\n",
    "    return array\n",
    "\n",
    "\n",
    "#get the substructure indices for a specfied radius around an atomID\n",
    "def getSubstructIndex(mol,atomID,radius):\n",
    "    if radius>0:\n",
    "        env = Chem.FindAtomEnvironmentOfRadiusN(mol,radius,atomID)\n",
    "        atomsToUse=[]\n",
    "        for b in env:\n",
    "            atomsToUse.append(mol.GetBondWithIdx(b).GetBeginAtomIdx())\n",
    "            atomsToUse.append(mol.GetBondWithIdx(b).GetEndAtomIdx())\n",
    "        atomsToUse = list(set(atomsToUse))\n",
    "    else:\n",
    "        atomsToUse = [atomID]\n",
    "        env=None\n",
    "    return(atomsToUse)\n",
    "\n",
    "#get the substructure indices for each fingerprint bit\n",
    "def calc_indices_for_fingerprints(smiles,radius = 2,nBits=2048):\n",
    "   mol = Chem.MolFromSmiles(smiles,sanitize = True)\n",
    "   bi = {}\n",
    "   fp = AllChem.GetHashedMorganFingerprint(mol,radius = radius, nBits=nBits, includeRedundantEnvironments=False,bitInfo=bi)\n",
    "   fp = fp.GetNonzeroElements()\n",
    "   #weights = [0  for i in mol.GetAtoms()]\n",
    "   #weights[slice(mol.GetSubstructMatch(Chem.MolFromSmiles('CNC(=O)C(C)(C)C',sanitize = True)))] += 1\n",
    "\n",
    "\n",
    "   fp_indices = {}\n",
    "   for fragment_id, count in fp.items():\n",
    "      sub_indices = []\n",
    "      for i in range(count):\n",
    "         #print(i)\n",
    "         root, radius = bi[fragment_id][i]\n",
    "         subindex = getSubstructIndex(mol,root,radius)\n",
    "         sub_indices = sub_indices + subindex\n",
    "\n",
    "      sub_indices = list(set(sub_indices))\n",
    "\n",
    "      fp_indices[fragment_id] = sub_indices\n",
    "   fp_indices = {\"fp_morgan_counts_\" +  str(k): v for k, v in fp_indices.items() }\n",
    "\n",
    "   return(fp_indices)\n",
    "\n",
    "#we also need a function that calculates the attribution of each atom to the final \n",
    "#prediction, for direction you can choose between \"up\": all positive attributions\n",
    "#\"down\": all negative attributions and recommended \"both\": summing up over all attributions\n",
    "def atom_attributions(mol, indices, shap_vals, featureNames, direction=\"both\"):\n",
    "    \"\"\"\n",
    "    Sum up the attributions to the model predictions (Shap values) for each atom in mol.\n",
    "\n",
    "    Parameters:\n",
    "        mol: rdkit molecule object\n",
    "        indices (dict): a dictionary with keys = feature names and values = list of atom positions in mol (output of calc_indices_for_fingerprints())\n",
    "        shap_vals: a numpy array of dimensions (len(featureNames))\n",
    "        featureNames (np.array): numpy array containing the feature names for shap_vals\n",
    "        direction (str): whether to consider positive (up), negative (down) or both shap_vals contributions\n",
    "\n",
    "    Returns:\n",
    "        np.array: each element = 1 atom of mol and the values are the summed contributions from shap_vals\n",
    "    \"\"\"\n",
    "    ## TO DO: add support for MACCS keys\n",
    "\n",
    "    # get for each fingerprint bit the atom numbers mapping to that bit and initialise weights to 0\n",
    "    curWeights = np.array([0 for i in range(mol.GetNumAtoms())], 'float64')\n",
    "\n",
    "    # step through each index (dictionary)\n",
    "    for j in indices:\n",
    "        # retrieve index to subset into the featureNames and the corresponding shap values array\n",
    "        curFPIndex = [i for i, s in enumerate(featureNames) if s == j]\n",
    "        if len(curFPIndex) == 0:\n",
    "            continue\n",
    "        if len(curFPIndex) > 1:\n",
    "            raise IndexError(\"Found several matches for feature \" + j)\n",
    "        # do something only if (i) the fingerprint bit was in the selected features\n",
    "        # and (ii) only if the retrieved shap value is positive/negative (direction argument)\n",
    "        if (direction == \"up\" and shap_vals[curFPIndex[0]] > 0) or (direction == \"down\" and shap_vals[curFPIndex[0]] < 0) or (direction == \"both\"):\n",
    "            # use the atom numbers ('positions') to update the curWeights array, where\n",
    "            # each atom of the current molecule is represented\n",
    "            # one atom may be part of several fingerprints so the summing has to be done across\n",
    "            # all fingerprints\n",
    "            for pos in indices[j]:\n",
    "                #value = (curWeights[pos] + shap_vals[curFPIndex[0]])\n",
    "                value = (curWeights[pos] + (shap_vals[curFPIndex[0]]/len(indices[j])))\n",
    "                curWeights[pos] = value\n",
    "\n",
    "    return curWeights\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To test the code we define some smiles, this is just a proof of concept, please feel free to add smiles of your own to get abigger dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#######################################################\n",
    "##define some smiles\n",
    "#######################################################\n",
    "\n",
    "smiles = [\"c1cccc[nH+]1\",\n",
    "      \"C[N+](C)(C)C\",\"c1ccccc1[NH3+]\",\n",
    "      \"CC(=O)[O-]\",\"c1ccccc1[O-]\",\n",
    "      \"CCS\",\n",
    "      \"C[N-]S(=O)(=O)C\",\n",
    "      \"C[N-]C=C\",\"C[N-]N=C\",\n",
    "      \"c1ccc[n-]1\",\n",
    "      \"CC[N-]C(=O)CC\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before we can use the smiles some preprocessing steps have to be performed to make them canonical and also to store them as mols for use in Rdkit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "################################################################################\n",
    "##make sure that the smiles are canonical to have full compatibility with CIME\n",
    "################################################################################\n",
    "\n",
    "#canonicalize smiles\n",
    "mols, successful_inds = smiles_to_mols(smiles)\n",
    "# get canonical smiles\n",
    "smiles = [Chem.MolToSmiles(m) for m in mols]\n",
    "\n",
    "#generate some names for the smiles\n",
    "smiles_names = [i for i in range(len(smiles))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Before we continue we store the smiles also as a data frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "##################################################\n",
    "# put the smiles and info into a data frame\n",
    "###################################################\n",
    "sdfExportData = pd.DataFrame({'names': smiles_names,\n",
    "        'smiles': smiles})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we did that we can calculate the logp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "################################################################################\n",
    "##calculate the logp for each of them\n",
    "################################################################################\n",
    "\n",
    "logp = [Descriptors.MolLogP(i) for i in mols]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The basis for our model are count based morgan fingerprints which we calculate and store in a matrix format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "##############################################\n",
    "##calculate the morgan fingerprints\n",
    "################################################################################\n",
    "\n",
    "# calculate the morgan counts fingerprints\n",
    "featuresMorganCounts = [calc_morgan_counts_for_smiles(\n",
    "    mol, radius=2, nBits=2048) for mol in mols]\n",
    "columns = [\"fp_morgan_counts_\" + str(i) for i in range(2048)]\n",
    "featuresMorganCounts = pd.DataFrame(featuresMorganCounts, columns=[\n",
    "                                  \"fp_morgan_counts_\" + str(i) for i in range(2048)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learning rate set to 0.020078\n",
      "0:\tlearn: 0.6529934\ttotal: 140ms\tremaining: 2m 19s\n",
      "1:\tlearn: 0.6501014\ttotal: 141ms\tremaining: 1m 10s\n",
      "2:\tlearn: 0.6468481\ttotal: 141ms\tremaining: 47s\n",
      "3:\tlearn: 0.6441279\ttotal: 142ms\tremaining: 35.3s\n",
      "4:\tlearn: 0.6414791\ttotal: 143ms\tremaining: 28.4s\n",
      "5:\tlearn: 0.6383103\ttotal: 143ms\tremaining: 23.7s\n",
      "6:\tlearn: 0.6355829\ttotal: 144ms\tremaining: 20.4s\n",
      "7:\tlearn: 0.6318449\ttotal: 145ms\tremaining: 17.9s\n",
      "8:\tlearn: 0.6284757\ttotal: 145ms\tremaining: 16s\n",
      "9:\tlearn: 0.6257633\ttotal: 146ms\tremaining: 14.4s\n",
      "10:\tlearn: 0.6229713\ttotal: 146ms\tremaining: 13.2s\n",
      "11:\tlearn: 0.6196277\ttotal: 147ms\tremaining: 12.1s\n",
      "12:\tlearn: 0.6170333\ttotal: 147ms\tremaining: 11.2s\n",
      "13:\tlearn: 0.6137229\ttotal: 148ms\tremaining: 10.4s\n",
      "14:\tlearn: 0.6111082\ttotal: 148ms\tremaining: 9.74s\n",
      "15:\tlearn: 0.6081114\ttotal: 149ms\tremaining: 9.15s\n",
      "16:\tlearn: 0.6054127\ttotal: 149ms\tremaining: 8.64s\n",
      "17:\tlearn: 0.6024950\ttotal: 150ms\tremaining: 8.18s\n",
      "18:\tlearn: 0.5998816\ttotal: 150ms\tremaining: 7.76s\n",
      "19:\tlearn: 0.5974411\ttotal: 151ms\tremaining: 7.38s\n",
      "20:\tlearn: 0.5948571\ttotal: 151ms\tremaining: 7.04s\n",
      "21:\tlearn: 0.5919822\ttotal: 152ms\tremaining: 6.73s\n",
      "22:\tlearn: 0.5884381\ttotal: 152ms\tremaining: 6.46s\n",
      "23:\tlearn: 0.5860311\ttotal: 152ms\tremaining: 6.19s\n",
      "24:\tlearn: 0.5836742\ttotal: 153ms\tremaining: 5.96s\n",
      "25:\tlearn: 0.5810578\ttotal: 153ms\tremaining: 5.75s\n",
      "26:\tlearn: 0.5782653\ttotal: 154ms\tremaining: 5.55s\n",
      "27:\tlearn: 0.5754984\ttotal: 155ms\tremaining: 5.37s\n",
      "28:\tlearn: 0.5727307\ttotal: 155ms\tremaining: 5.19s\n",
      "29:\tlearn: 0.5698933\ttotal: 156ms\tremaining: 5.03s\n",
      "30:\tlearn: 0.5677174\ttotal: 156ms\tremaining: 4.88s\n",
      "31:\tlearn: 0.5649782\ttotal: 157ms\tremaining: 4.74s\n",
      "32:\tlearn: 0.5627036\ttotal: 157ms\tremaining: 4.6s\n",
      "33:\tlearn: 0.5604548\ttotal: 157ms\tremaining: 4.47s\n",
      "34:\tlearn: 0.5578328\ttotal: 158ms\tremaining: 4.36s\n",
      "35:\tlearn: 0.5556106\ttotal: 159ms\tremaining: 4.25s\n",
      "36:\tlearn: 0.5531828\ttotal: 159ms\tremaining: 4.15s\n",
      "37:\tlearn: 0.5502346\ttotal: 160ms\tremaining: 4.05s\n",
      "38:\tlearn: 0.5474375\ttotal: 161ms\tremaining: 3.96s\n",
      "39:\tlearn: 0.5449863\ttotal: 161ms\tremaining: 3.87s\n",
      "40:\tlearn: 0.5427804\ttotal: 162ms\tremaining: 3.78s\n",
      "41:\tlearn: 0.5399984\ttotal: 162ms\tremaining: 3.69s\n",
      "42:\tlearn: 0.5376278\ttotal: 163ms\tremaining: 3.62s\n",
      "43:\tlearn: 0.5353788\ttotal: 163ms\tremaining: 3.54s\n",
      "44:\tlearn: 0.5333483\ttotal: 164ms\tremaining: 3.47s\n",
      "45:\tlearn: 0.5308566\ttotal: 164ms\tremaining: 3.41s\n",
      "46:\tlearn: 0.5278007\ttotal: 165ms\tremaining: 3.34s\n",
      "47:\tlearn: 0.5256878\ttotal: 165ms\tremaining: 3.27s\n",
      "48:\tlearn: 0.5228558\ttotal: 165ms\tremaining: 3.21s\n",
      "49:\tlearn: 0.5205787\ttotal: 166ms\tremaining: 3.16s\n",
      "50:\tlearn: 0.5186624\ttotal: 167ms\tremaining: 3.1s\n",
      "51:\tlearn: 0.5166402\ttotal: 167ms\tremaining: 3.05s\n",
      "52:\tlearn: 0.5141465\ttotal: 168ms\tremaining: 3s\n",
      "53:\tlearn: 0.5116924\ttotal: 169ms\tremaining: 2.96s\n",
      "54:\tlearn: 0.5090220\ttotal: 170ms\tremaining: 2.92s\n",
      "55:\tlearn: 0.5069732\ttotal: 171ms\tremaining: 2.88s\n",
      "56:\tlearn: 0.5050213\ttotal: 171ms\tremaining: 2.83s\n",
      "57:\tlearn: 0.5028863\ttotal: 171ms\tremaining: 2.79s\n",
      "58:\tlearn: 0.5001513\ttotal: 172ms\tremaining: 2.75s\n",
      "59:\tlearn: 0.4979592\ttotal: 173ms\tremaining: 2.7s\n",
      "60:\tlearn: 0.4956096\ttotal: 173ms\tremaining: 2.66s\n",
      "61:\tlearn: 0.4932515\ttotal: 174ms\tremaining: 2.63s\n",
      "62:\tlearn: 0.4912860\ttotal: 174ms\tremaining: 2.59s\n",
      "63:\tlearn: 0.4891070\ttotal: 175ms\tremaining: 2.55s\n",
      "64:\tlearn: 0.4867674\ttotal: 175ms\tremaining: 2.52s\n",
      "65:\tlearn: 0.4842476\ttotal: 176ms\tremaining: 2.49s\n",
      "66:\tlearn: 0.4817336\ttotal: 176ms\tremaining: 2.45s\n",
      "67:\tlearn: 0.4799576\ttotal: 177ms\tremaining: 2.42s\n",
      "68:\tlearn: 0.4773786\ttotal: 177ms\tremaining: 2.39s\n",
      "69:\tlearn: 0.4748531\ttotal: 178ms\tremaining: 2.36s\n",
      "70:\tlearn: 0.4726050\ttotal: 178ms\tremaining: 2.33s\n",
      "71:\tlearn: 0.4704388\ttotal: 179ms\tremaining: 2.3s\n",
      "72:\tlearn: 0.4685530\ttotal: 179ms\tremaining: 2.28s\n",
      "73:\tlearn: 0.4663395\ttotal: 180ms\tremaining: 2.25s\n",
      "74:\tlearn: 0.4643817\ttotal: 180ms\tremaining: 2.23s\n",
      "75:\tlearn: 0.4622157\ttotal: 181ms\tremaining: 2.2s\n",
      "76:\tlearn: 0.4605956\ttotal: 181ms\tremaining: 2.17s\n",
      "77:\tlearn: 0.4583075\ttotal: 182ms\tremaining: 2.15s\n",
      "78:\tlearn: 0.4557590\ttotal: 183ms\tremaining: 2.13s\n",
      "79:\tlearn: 0.4540486\ttotal: 183ms\tremaining: 2.11s\n",
      "80:\tlearn: 0.4524461\ttotal: 183ms\tremaining: 2.08s\n",
      "81:\tlearn: 0.4503276\ttotal: 184ms\tremaining: 2.06s\n",
      "82:\tlearn: 0.4481365\ttotal: 184ms\tremaining: 2.04s\n",
      "83:\tlearn: 0.4460221\ttotal: 185ms\tremaining: 2.02s\n",
      "84:\tlearn: 0.4439559\ttotal: 185ms\tremaining: 1.99s\n",
      "85:\tlearn: 0.4423017\ttotal: 186ms\tremaining: 1.97s\n",
      "86:\tlearn: 0.4406586\ttotal: 186ms\tremaining: 1.95s\n",
      "87:\tlearn: 0.4391412\ttotal: 186ms\tremaining: 1.93s\n",
      "88:\tlearn: 0.4370631\ttotal: 187ms\tremaining: 1.91s\n",
      "89:\tlearn: 0.4354345\ttotal: 187ms\tremaining: 1.89s\n",
      "90:\tlearn: 0.4338327\ttotal: 187ms\tremaining: 1.87s\n",
      "91:\tlearn: 0.4321384\ttotal: 188ms\tremaining: 1.85s\n",
      "92:\tlearn: 0.4300587\ttotal: 189ms\tremaining: 1.84s\n",
      "93:\tlearn: 0.4277400\ttotal: 189ms\tremaining: 1.82s\n",
      "94:\tlearn: 0.4259640\ttotal: 190ms\tremaining: 1.8s\n",
      "95:\tlearn: 0.4240021\ttotal: 190ms\tremaining: 1.79s\n",
      "96:\tlearn: 0.4224572\ttotal: 191ms\tremaining: 1.77s\n",
      "97:\tlearn: 0.4207998\ttotal: 191ms\tremaining: 1.76s\n",
      "98:\tlearn: 0.4186274\ttotal: 191ms\tremaining: 1.74s\n",
      "99:\tlearn: 0.4164557\ttotal: 192ms\tremaining: 1.73s\n",
      "100:\tlearn: 0.4144643\ttotal: 193ms\tremaining: 1.71s\n",
      "101:\tlearn: 0.4125453\ttotal: 193ms\tremaining: 1.7s\n",
      "102:\tlearn: 0.4103794\ttotal: 194ms\tremaining: 1.69s\n",
      "103:\tlearn: 0.4084729\ttotal: 194ms\tremaining: 1.67s\n",
      "104:\tlearn: 0.4066244\ttotal: 195ms\tremaining: 1.66s\n",
      "105:\tlearn: 0.4046540\ttotal: 195ms\tremaining: 1.65s\n",
      "106:\tlearn: 0.4033552\ttotal: 196ms\tremaining: 1.63s\n",
      "107:\tlearn: 0.4013387\ttotal: 196ms\tremaining: 1.62s\n",
      "108:\tlearn: 0.3995518\ttotal: 196ms\tremaining: 1.6s\n",
      "109:\tlearn: 0.3981071\ttotal: 197ms\tremaining: 1.59s\n",
      "110:\tlearn: 0.3966859\ttotal: 197ms\tremaining: 1.58s\n",
      "111:\tlearn: 0.3952767\ttotal: 197ms\tremaining: 1.56s\n",
      "112:\tlearn: 0.3933474\ttotal: 198ms\tremaining: 1.55s\n",
      "113:\tlearn: 0.3915859\ttotal: 198ms\tremaining: 1.54s\n",
      "114:\tlearn: 0.3899760\ttotal: 199ms\tremaining: 1.53s\n",
      "115:\tlearn: 0.3875476\ttotal: 199ms\tremaining: 1.52s\n",
      "116:\tlearn: 0.3862123\ttotal: 200ms\tremaining: 1.51s\n",
      "117:\tlearn: 0.3844776\ttotal: 200ms\tremaining: 1.5s\n",
      "118:\tlearn: 0.3826342\ttotal: 201ms\tremaining: 1.49s\n",
      "119:\tlearn: 0.3808506\ttotal: 201ms\tremaining: 1.48s\n",
      "120:\tlearn: 0.3795079\ttotal: 202ms\tremaining: 1.46s\n",
      "121:\tlearn: 0.3782611\ttotal: 202ms\tremaining: 1.45s\n",
      "122:\tlearn: 0.3765296\ttotal: 202ms\tremaining: 1.44s\n",
      "123:\tlearn: 0.3745037\ttotal: 203ms\tremaining: 1.43s\n",
      "124:\tlearn: 0.3732291\ttotal: 203ms\tremaining: 1.42s\n",
      "125:\tlearn: 0.3714237\ttotal: 204ms\tremaining: 1.41s\n",
      "126:\tlearn: 0.3696288\ttotal: 204ms\tremaining: 1.41s\n",
      "127:\tlearn: 0.3683332\ttotal: 205ms\tremaining: 1.4s\n",
      "128:\tlearn: 0.3667346\ttotal: 205ms\tremaining: 1.39s\n",
      "129:\tlearn: 0.3649583\ttotal: 206ms\tremaining: 1.38s\n",
      "130:\tlearn: 0.3632806\ttotal: 206ms\tremaining: 1.37s\n",
      "131:\tlearn: 0.3620565\ttotal: 207ms\tremaining: 1.36s\n",
      "132:\tlearn: 0.3608034\ttotal: 207ms\tremaining: 1.35s\n",
      "133:\tlearn: 0.3593360\ttotal: 207ms\tremaining: 1.34s\n",
      "134:\tlearn: 0.3573923\ttotal: 208ms\tremaining: 1.33s\n",
      "135:\tlearn: 0.3558775\ttotal: 208ms\tremaining: 1.32s\n",
      "136:\tlearn: 0.3541507\ttotal: 209ms\tremaining: 1.31s\n",
      "137:\tlearn: 0.3525434\ttotal: 209ms\tremaining: 1.31s\n",
      "138:\tlearn: 0.3511986\ttotal: 210ms\tremaining: 1.3s\n",
      "139:\tlearn: 0.3494718\ttotal: 210ms\tremaining: 1.29s\n",
      "140:\tlearn: 0.3478659\ttotal: 211ms\tremaining: 1.28s\n",
      "141:\tlearn: 0.3462300\ttotal: 211ms\tremaining: 1.28s\n",
      "142:\tlearn: 0.3447165\ttotal: 212ms\tremaining: 1.27s\n",
      "143:\tlearn: 0.3432431\ttotal: 212ms\tremaining: 1.26s\n",
      "144:\tlearn: 0.3416159\ttotal: 213ms\tremaining: 1.25s\n",
      "145:\tlearn: 0.3397617\ttotal: 213ms\tremaining: 1.25s\n",
      "146:\tlearn: 0.3383203\ttotal: 214ms\tremaining: 1.24s\n",
      "147:\tlearn: 0.3368776\ttotal: 214ms\tremaining: 1.23s\n",
      "148:\tlearn: 0.3356020\ttotal: 215ms\tremaining: 1.23s\n",
      "149:\tlearn: 0.3341204\ttotal: 215ms\tremaining: 1.22s\n",
      "150:\tlearn: 0.3326136\ttotal: 216ms\tremaining: 1.21s\n",
      "151:\tlearn: 0.3311141\ttotal: 216ms\tremaining: 1.21s\n",
      "152:\tlearn: 0.3299911\ttotal: 217ms\tremaining: 1.2s\n",
      "153:\tlearn: 0.3285731\ttotal: 217ms\tremaining: 1.19s\n",
      "154:\tlearn: 0.3272469\ttotal: 217ms\tremaining: 1.19s\n",
      "155:\tlearn: 0.3257765\ttotal: 218ms\tremaining: 1.18s\n",
      "156:\tlearn: 0.3242298\ttotal: 219ms\tremaining: 1.17s\n",
      "157:\tlearn: 0.3227751\ttotal: 219ms\tremaining: 1.17s\n",
      "158:\tlearn: 0.3212202\ttotal: 220ms\tremaining: 1.16s\n",
      "159:\tlearn: 0.3198153\ttotal: 220ms\tremaining: 1.16s\n",
      "160:\tlearn: 0.3183839\ttotal: 221ms\tremaining: 1.15s\n",
      "161:\tlearn: 0.3172210\ttotal: 221ms\tremaining: 1.14s\n",
      "162:\tlearn: 0.3161659\ttotal: 222ms\tremaining: 1.14s\n",
      "163:\tlearn: 0.3146476\ttotal: 222ms\tremaining: 1.13s\n",
      "164:\tlearn: 0.3132040\ttotal: 223ms\tremaining: 1.13s\n",
      "165:\tlearn: 0.3116808\ttotal: 223ms\tremaining: 1.12s\n",
      "166:\tlearn: 0.3102916\ttotal: 224ms\tremaining: 1.11s\n",
      "167:\tlearn: 0.3090275\ttotal: 224ms\tremaining: 1.11s\n",
      "168:\tlearn: 0.3075626\ttotal: 225ms\tremaining: 1.1s\n",
      "169:\tlearn: 0.3061786\ttotal: 225ms\tremaining: 1.1s\n",
      "170:\tlearn: 0.3046770\ttotal: 226ms\tremaining: 1.09s\n",
      "171:\tlearn: 0.3034452\ttotal: 226ms\tremaining: 1.09s\n",
      "172:\tlearn: 0.3020608\ttotal: 227ms\tremaining: 1.08s\n",
      "173:\tlearn: 0.3007864\ttotal: 227ms\tremaining: 1.08s\n",
      "174:\tlearn: 0.2994469\ttotal: 228ms\tremaining: 1.07s\n",
      "175:\tlearn: 0.2984670\ttotal: 228ms\tremaining: 1.07s\n",
      "176:\tlearn: 0.2970344\ttotal: 229ms\tremaining: 1.06s\n",
      "177:\tlearn: 0.2955766\ttotal: 229ms\tremaining: 1.06s\n",
      "178:\tlearn: 0.2943208\ttotal: 230ms\tremaining: 1.05s\n",
      "179:\tlearn: 0.2927308\ttotal: 231ms\tremaining: 1.05s\n",
      "180:\tlearn: 0.2913516\ttotal: 231ms\tremaining: 1.05s\n",
      "181:\tlearn: 0.2900654\ttotal: 232ms\tremaining: 1.04s\n",
      "182:\tlearn: 0.2886798\ttotal: 232ms\tremaining: 1.04s\n",
      "183:\tlearn: 0.2871546\ttotal: 233ms\tremaining: 1.03s\n",
      "184:\tlearn: 0.2858634\ttotal: 233ms\tremaining: 1.03s\n",
      "185:\tlearn: 0.2847386\ttotal: 234ms\tremaining: 1.02s\n",
      "186:\tlearn: 0.2836213\ttotal: 234ms\tremaining: 1.02s\n",
      "187:\tlearn: 0.2824287\ttotal: 235ms\tremaining: 1.01s\n",
      "188:\tlearn: 0.2810536\ttotal: 235ms\tremaining: 1.01s\n",
      "189:\tlearn: 0.2797246\ttotal: 236ms\tremaining: 1s\n",
      "190:\tlearn: 0.2783861\ttotal: 236ms\tremaining: 1s\n",
      "191:\tlearn: 0.2771397\ttotal: 237ms\tremaining: 997ms\n",
      "192:\tlearn: 0.2757595\ttotal: 237ms\tremaining: 993ms\n",
      "193:\tlearn: 0.2745325\ttotal: 238ms\tremaining: 989ms\n",
      "194:\tlearn: 0.2733314\ttotal: 238ms\tremaining: 984ms\n",
      "195:\tlearn: 0.2721312\ttotal: 239ms\tremaining: 980ms\n",
      "196:\tlearn: 0.2708380\ttotal: 240ms\tremaining: 977ms\n",
      "197:\tlearn: 0.2696570\ttotal: 240ms\tremaining: 972ms\n",
      "198:\tlearn: 0.2684529\ttotal: 240ms\tremaining: 968ms\n",
      "199:\tlearn: 0.2671718\ttotal: 241ms\tremaining: 964ms\n",
      "200:\tlearn: 0.2657608\ttotal: 242ms\tremaining: 961ms\n",
      "201:\tlearn: 0.2645971\ttotal: 242ms\tremaining: 957ms\n",
      "202:\tlearn: 0.2633133\ttotal: 243ms\tremaining: 953ms\n",
      "203:\tlearn: 0.2619528\ttotal: 243ms\tremaining: 950ms\n",
      "204:\tlearn: 0.2606767\ttotal: 244ms\tremaining: 946ms\n",
      "205:\tlearn: 0.2594074\ttotal: 245ms\tremaining: 942ms\n",
      "206:\tlearn: 0.2581931\ttotal: 245ms\tremaining: 939ms\n",
      "207:\tlearn: 0.2570317\ttotal: 246ms\tremaining: 935ms\n",
      "208:\tlearn: 0.2559170\ttotal: 246ms\tremaining: 931ms\n",
      "209:\tlearn: 0.2546934\ttotal: 247ms\tremaining: 928ms\n",
      "210:\tlearn: 0.2534398\ttotal: 247ms\tremaining: 925ms\n",
      "211:\tlearn: 0.2523432\ttotal: 248ms\tremaining: 921ms\n",
      "212:\tlearn: 0.2511116\ttotal: 248ms\tremaining: 917ms\n",
      "213:\tlearn: 0.2499270\ttotal: 249ms\tremaining: 914ms\n",
      "214:\tlearn: 0.2488483\ttotal: 249ms\tremaining: 910ms\n",
      "215:\tlearn: 0.2477758\ttotal: 250ms\tremaining: 906ms\n",
      "216:\tlearn: 0.2465753\ttotal: 250ms\tremaining: 903ms\n",
      "217:\tlearn: 0.2453790\ttotal: 251ms\tremaining: 900ms\n",
      "218:\tlearn: 0.2442350\ttotal: 251ms\tremaining: 897ms\n",
      "219:\tlearn: 0.2430561\ttotal: 252ms\tremaining: 893ms\n",
      "220:\tlearn: 0.2418781\ttotal: 253ms\tremaining: 890ms\n",
      "221:\tlearn: 0.2407521\ttotal: 253ms\tremaining: 887ms\n",
      "222:\tlearn: 0.2395860\ttotal: 254ms\tremaining: 883ms\n",
      "223:\tlearn: 0.2384736\ttotal: 254ms\tremaining: 880ms\n",
      "224:\tlearn: 0.2373689\ttotal: 255ms\tremaining: 877ms\n",
      "225:\tlearn: 0.2362115\ttotal: 255ms\tremaining: 874ms\n",
      "226:\tlearn: 0.2351975\ttotal: 256ms\tremaining: 871ms\n",
      "227:\tlearn: 0.2340800\ttotal: 256ms\tremaining: 868ms\n",
      "228:\tlearn: 0.2328849\ttotal: 257ms\tremaining: 865ms\n",
      "229:\tlearn: 0.2317650\ttotal: 257ms\tremaining: 862ms\n",
      "230:\tlearn: 0.2306405\ttotal: 258ms\tremaining: 859ms\n",
      "231:\tlearn: 0.2295105\ttotal: 258ms\tremaining: 856ms\n",
      "232:\tlearn: 0.2284467\ttotal: 259ms\tremaining: 852ms\n",
      "233:\tlearn: 0.2273673\ttotal: 260ms\tremaining: 850ms\n",
      "234:\tlearn: 0.2263166\ttotal: 260ms\tremaining: 847ms\n",
      "235:\tlearn: 0.2253524\ttotal: 261ms\tremaining: 843ms\n",
      "236:\tlearn: 0.2243115\ttotal: 261ms\tremaining: 841ms\n",
      "237:\tlearn: 0.2233255\ttotal: 262ms\tremaining: 838ms\n",
      "238:\tlearn: 0.2222491\ttotal: 262ms\tremaining: 835ms\n",
      "239:\tlearn: 0.2211784\ttotal: 263ms\tremaining: 832ms\n",
      "240:\tlearn: 0.2201150\ttotal: 263ms\tremaining: 829ms\n",
      "241:\tlearn: 0.2190460\ttotal: 264ms\tremaining: 827ms\n",
      "242:\tlearn: 0.2179876\ttotal: 264ms\tremaining: 824ms\n",
      "243:\tlearn: 0.2169052\ttotal: 265ms\tremaining: 821ms\n",
      "244:\tlearn: 0.2158974\ttotal: 265ms\tremaining: 818ms\n",
      "245:\tlearn: 0.2148554\ttotal: 266ms\tremaining: 815ms\n",
      "246:\tlearn: 0.2138285\ttotal: 267ms\tremaining: 812ms\n",
      "247:\tlearn: 0.2128071\ttotal: 267ms\tremaining: 810ms\n",
      "248:\tlearn: 0.2117910\ttotal: 268ms\tremaining: 807ms\n",
      "249:\tlearn: 0.2108962\ttotal: 268ms\tremaining: 804ms\n",
      "250:\tlearn: 0.2097966\ttotal: 269ms\tremaining: 802ms\n",
      "251:\tlearn: 0.2088324\ttotal: 269ms\tremaining: 799ms\n",
      "252:\tlearn: 0.2078373\ttotal: 270ms\tremaining: 796ms\n",
      "253:\tlearn: 0.2068474\ttotal: 270ms\tremaining: 793ms\n",
      "254:\tlearn: 0.2058987\ttotal: 271ms\tremaining: 791ms\n",
      "255:\tlearn: 0.2049192\ttotal: 271ms\tremaining: 788ms\n",
      "256:\tlearn: 0.2039016\ttotal: 272ms\tremaining: 786ms\n",
      "257:\tlearn: 0.2029320\ttotal: 272ms\tremaining: 783ms\n",
      "258:\tlearn: 0.2020064\ttotal: 273ms\tremaining: 781ms\n",
      "259:\tlearn: 0.2010039\ttotal: 273ms\tremaining: 778ms\n",
      "260:\tlearn: 0.2000854\ttotal: 274ms\tremaining: 776ms\n",
      "261:\tlearn: 0.1990666\ttotal: 274ms\tremaining: 772ms\n",
      "262:\tlearn: 0.1981566\ttotal: 275ms\tremaining: 770ms\n",
      "263:\tlearn: 0.1972173\ttotal: 275ms\tremaining: 768ms\n",
      "264:\tlearn: 0.1962330\ttotal: 276ms\tremaining: 765ms\n",
      "265:\tlearn: 0.1953030\ttotal: 277ms\tremaining: 763ms\n",
      "266:\tlearn: 0.1942543\ttotal: 277ms\tremaining: 761ms\n",
      "267:\tlearn: 0.1932908\ttotal: 278ms\tremaining: 758ms\n",
      "268:\tlearn: 0.1923758\ttotal: 278ms\tremaining: 756ms\n",
      "269:\tlearn: 0.1914656\ttotal: 279ms\tremaining: 754ms\n",
      "270:\tlearn: 0.1905602\ttotal: 279ms\tremaining: 751ms\n",
      "271:\tlearn: 0.1896351\ttotal: 280ms\tremaining: 749ms\n",
      "272:\tlearn: 0.1887393\ttotal: 280ms\tremaining: 747ms\n",
      "273:\tlearn: 0.1878036\ttotal: 281ms\tremaining: 745ms\n",
      "274:\tlearn: 0.1868925\ttotal: 282ms\tremaining: 742ms\n",
      "275:\tlearn: 0.1860053\ttotal: 282ms\tremaining: 740ms\n",
      "276:\tlearn: 0.1851601\ttotal: 283ms\tremaining: 738ms\n",
      "277:\tlearn: 0.1843312\ttotal: 283ms\tremaining: 736ms\n",
      "278:\tlearn: 0.1834179\ttotal: 284ms\tremaining: 733ms\n",
      "279:\tlearn: 0.1825479\ttotal: 284ms\tremaining: 731ms\n",
      "280:\tlearn: 0.1816826\ttotal: 285ms\tremaining: 729ms\n",
      "281:\tlearn: 0.1807828\ttotal: 285ms\tremaining: 727ms\n",
      "282:\tlearn: 0.1798877\ttotal: 286ms\tremaining: 725ms\n",
      "283:\tlearn: 0.1790409\ttotal: 287ms\tremaining: 722ms\n",
      "284:\tlearn: 0.1781548\ttotal: 287ms\tremaining: 720ms\n",
      "285:\tlearn: 0.1772733\ttotal: 288ms\tremaining: 718ms\n",
      "286:\tlearn: 0.1763964\ttotal: 288ms\tremaining: 716ms\n",
      "287:\tlearn: 0.1755612\ttotal: 289ms\tremaining: 714ms\n",
      "288:\tlearn: 0.1746931\ttotal: 289ms\tremaining: 712ms\n",
      "289:\tlearn: 0.1738295\ttotal: 290ms\tremaining: 710ms\n",
      "290:\tlearn: 0.1729705\ttotal: 291ms\tremaining: 708ms\n",
      "291:\tlearn: 0.1721578\ttotal: 291ms\tremaining: 706ms\n",
      "292:\tlearn: 0.1713494\ttotal: 292ms\tremaining: 704ms\n",
      "293:\tlearn: 0.1705454\ttotal: 292ms\tremaining: 701ms\n",
      "294:\tlearn: 0.1697031\ttotal: 293ms\tremaining: 699ms\n",
      "295:\tlearn: 0.1688653\ttotal: 293ms\tremaining: 697ms\n",
      "296:\tlearn: 0.1680319\ttotal: 294ms\tremaining: 695ms\n",
      "297:\tlearn: 0.1672442\ttotal: 294ms\tremaining: 693ms\n",
      "298:\tlearn: 0.1664192\ttotal: 295ms\tremaining: 691ms\n",
      "299:\tlearn: 0.1655985\ttotal: 295ms\tremaining: 689ms\n",
      "300:\tlearn: 0.1647848\ttotal: 296ms\tremaining: 687ms\n",
      "301:\tlearn: 0.1639727\ttotal: 296ms\tremaining: 685ms\n",
      "302:\tlearn: 0.1631648\ttotal: 297ms\tremaining: 683ms\n",
      "303:\tlearn: 0.1623612\ttotal: 298ms\tremaining: 681ms\n",
      "304:\tlearn: 0.1615619\ttotal: 298ms\tremaining: 680ms\n",
      "305:\tlearn: 0.1607667\ttotal: 299ms\tremaining: 678ms\n",
      "306:\tlearn: 0.1599782\ttotal: 299ms\tremaining: 676ms\n",
      "307:\tlearn: 0.1591913\ttotal: 300ms\tremaining: 674ms\n",
      "308:\tlearn: 0.1584086\ttotal: 301ms\tremaining: 672ms\n",
      "309:\tlearn: 0.1576300\ttotal: 301ms\tremaining: 670ms\n",
      "310:\tlearn: 0.1568929\ttotal: 302ms\tremaining: 668ms\n",
      "311:\tlearn: 0.1561222\ttotal: 302ms\tremaining: 666ms\n",
      "312:\tlearn: 0.1553554\ttotal: 303ms\tremaining: 664ms\n",
      "313:\tlearn: 0.1545951\ttotal: 303ms\tremaining: 663ms\n",
      "314:\tlearn: 0.1538386\ttotal: 304ms\tremaining: 661ms\n",
      "315:\tlearn: 0.1530839\ttotal: 304ms\tremaining: 659ms\n",
      "316:\tlearn: 0.1523332\ttotal: 305ms\tremaining: 657ms\n",
      "317:\tlearn: 0.1515864\ttotal: 306ms\tremaining: 655ms\n",
      "318:\tlearn: 0.1508435\ttotal: 306ms\tremaining: 653ms\n",
      "319:\tlearn: 0.1501045\ttotal: 307ms\tremaining: 652ms\n",
      "320:\tlearn: 0.1493715\ttotal: 307ms\tremaining: 650ms\n",
      "321:\tlearn: 0.1486424\ttotal: 308ms\tremaining: 648ms\n",
      "322:\tlearn: 0.1479149\ttotal: 308ms\tremaining: 646ms\n",
      "323:\tlearn: 0.1471914\ttotal: 309ms\tremaining: 645ms\n",
      "324:\tlearn: 0.1464716\ttotal: 310ms\tremaining: 643ms\n",
      "325:\tlearn: 0.1457576\ttotal: 310ms\tremaining: 641ms\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "326:\tlearn: 0.1450785\ttotal: 311ms\tremaining: 640ms\n",
      "327:\tlearn: 0.1443717\ttotal: 311ms\tremaining: 638ms\n",
      "328:\tlearn: 0.1436666\ttotal: 312ms\tremaining: 636ms\n",
      "329:\tlearn: 0.1429653\ttotal: 313ms\tremaining: 635ms\n",
      "330:\tlearn: 0.1422676\ttotal: 313ms\tremaining: 633ms\n",
      "331:\tlearn: 0.1415737\ttotal: 314ms\tremaining: 631ms\n",
      "332:\tlearn: 0.1408834\ttotal: 314ms\tremaining: 630ms\n",
      "333:\tlearn: 0.1401967\ttotal: 315ms\tremaining: 628ms\n",
      "334:\tlearn: 0.1395137\ttotal: 315ms\tremaining: 626ms\n",
      "335:\tlearn: 0.1388360\ttotal: 316ms\tremaining: 624ms\n",
      "336:\tlearn: 0.1381601\ttotal: 317ms\tremaining: 623ms\n",
      "337:\tlearn: 0.1374878\ttotal: 317ms\tremaining: 621ms\n",
      "338:\tlearn: 0.1368207\ttotal: 318ms\tremaining: 619ms\n",
      "339:\tlearn: 0.1361555\ttotal: 318ms\tremaining: 618ms\n",
      "340:\tlearn: 0.1354938\ttotal: 319ms\tremaining: 616ms\n",
      "341:\tlearn: 0.1348355\ttotal: 319ms\tremaining: 614ms\n",
      "342:\tlearn: 0.1341808\ttotal: 320ms\tremaining: 613ms\n",
      "343:\tlearn: 0.1335295\ttotal: 320ms\tremaining: 611ms\n",
      "344:\tlearn: 0.1328817\ttotal: 321ms\tremaining: 610ms\n",
      "345:\tlearn: 0.1322387\ttotal: 322ms\tremaining: 608ms\n",
      "346:\tlearn: 0.1315977\ttotal: 322ms\tremaining: 607ms\n",
      "347:\tlearn: 0.1309600\ttotal: 323ms\tremaining: 605ms\n",
      "348:\tlearn: 0.1303258\ttotal: 324ms\tremaining: 604ms\n",
      "349:\tlearn: 0.1296948\ttotal: 324ms\tremaining: 602ms\n",
      "350:\tlearn: 0.1290946\ttotal: 325ms\tremaining: 601ms\n",
      "351:\tlearn: 0.1284702\ttotal: 326ms\tremaining: 600ms\n",
      "352:\tlearn: 0.1278490\ttotal: 326ms\tremaining: 598ms\n",
      "353:\tlearn: 0.1272311\ttotal: 327ms\tremaining: 597ms\n",
      "354:\tlearn: 0.1266178\ttotal: 328ms\tremaining: 596ms\n",
      "355:\tlearn: 0.1260076\ttotal: 329ms\tremaining: 594ms\n",
      "356:\tlearn: 0.1254007\ttotal: 329ms\tremaining: 593ms\n",
      "357:\tlearn: 0.1247970\ttotal: 330ms\tremaining: 591ms\n",
      "358:\tlearn: 0.1241953\ttotal: 330ms\tremaining: 590ms\n",
      "359:\tlearn: 0.1235967\ttotal: 331ms\tremaining: 588ms\n",
      "360:\tlearn: 0.1230013\ttotal: 332ms\tremaining: 587ms\n",
      "361:\tlearn: 0.1224090\ttotal: 332ms\tremaining: 585ms\n",
      "362:\tlearn: 0.1218211\ttotal: 333ms\tremaining: 584ms\n",
      "363:\tlearn: 0.1212363\ttotal: 333ms\tremaining: 582ms\n",
      "364:\tlearn: 0.1206545\ttotal: 334ms\tremaining: 581ms\n",
      "365:\tlearn: 0.1200747\ttotal: 334ms\tremaining: 579ms\n",
      "366:\tlearn: 0.1194979\ttotal: 335ms\tremaining: 578ms\n",
      "367:\tlearn: 0.1189243\ttotal: 336ms\tremaining: 576ms\n",
      "368:\tlearn: 0.1183537\ttotal: 336ms\tremaining: 575ms\n",
      "369:\tlearn: 0.1177861\ttotal: 337ms\tremaining: 573ms\n",
      "370:\tlearn: 0.1172215\ttotal: 337ms\tremaining: 572ms\n",
      "371:\tlearn: 0.1166610\ttotal: 338ms\tremaining: 571ms\n",
      "372:\tlearn: 0.1161024\ttotal: 339ms\tremaining: 569ms\n",
      "373:\tlearn: 0.1155468\ttotal: 339ms\tremaining: 568ms\n",
      "374:\tlearn: 0.1149942\ttotal: 340ms\tremaining: 566ms\n",
      "375:\tlearn: 0.1144445\ttotal: 340ms\tremaining: 565ms\n",
      "376:\tlearn: 0.1138977\ttotal: 341ms\tremaining: 564ms\n",
      "377:\tlearn: 0.1133539\ttotal: 342ms\tremaining: 562ms\n",
      "378:\tlearn: 0.1128129\ttotal: 342ms\tremaining: 561ms\n",
      "379:\tlearn: 0.1122749\ttotal: 343ms\tremaining: 559ms\n",
      "380:\tlearn: 0.1117397\ttotal: 343ms\tremaining: 558ms\n",
      "381:\tlearn: 0.1112074\ttotal: 344ms\tremaining: 556ms\n",
      "382:\tlearn: 0.1106779\ttotal: 344ms\tremaining: 555ms\n",
      "383:\tlearn: 0.1101513\ttotal: 345ms\tremaining: 554ms\n",
      "384:\tlearn: 0.1096282\ttotal: 346ms\tremaining: 552ms\n",
      "385:\tlearn: 0.1091080\ttotal: 347ms\tremaining: 551ms\n",
      "386:\tlearn: 0.1085897\ttotal: 347ms\tremaining: 550ms\n",
      "387:\tlearn: 0.1080750\ttotal: 348ms\tremaining: 549ms\n",
      "388:\tlearn: 0.1075630\ttotal: 349ms\tremaining: 547ms\n",
      "389:\tlearn: 0.1070530\ttotal: 349ms\tremaining: 546ms\n",
      "390:\tlearn: 0.1065464\ttotal: 350ms\tremaining: 545ms\n",
      "391:\tlearn: 0.1060418\ttotal: 351ms\tremaining: 544ms\n",
      "392:\tlearn: 0.1055400\ttotal: 351ms\tremaining: 542ms\n",
      "393:\tlearn: 0.1050408\ttotal: 352ms\tremaining: 541ms\n",
      "394:\tlearn: 0.1045450\ttotal: 353ms\tremaining: 541ms\n",
      "395:\tlearn: 0.1040512\ttotal: 354ms\tremaining: 540ms\n",
      "396:\tlearn: 0.1035600\ttotal: 354ms\tremaining: 538ms\n",
      "397:\tlearn: 0.1030714\ttotal: 355ms\tremaining: 537ms\n",
      "398:\tlearn: 0.1025862\ttotal: 356ms\tremaining: 536ms\n",
      "399:\tlearn: 0.1021029\ttotal: 356ms\tremaining: 534ms\n",
      "400:\tlearn: 0.1015413\ttotal: 357ms\tremaining: 533ms\n",
      "401:\tlearn: 0.1010639\ttotal: 358ms\tremaining: 532ms\n",
      "402:\tlearn: 0.1005884\ttotal: 358ms\tremaining: 531ms\n",
      "403:\tlearn: 0.1000356\ttotal: 359ms\tremaining: 530ms\n",
      "404:\tlearn: 0.0995679\ttotal: 360ms\tremaining: 528ms\n",
      "405:\tlearn: 0.0990999\ttotal: 360ms\tremaining: 527ms\n",
      "406:\tlearn: 0.0985557\ttotal: 361ms\tremaining: 526ms\n",
      "407:\tlearn: 0.0980928\ttotal: 361ms\tremaining: 524ms\n",
      "408:\tlearn: 0.0975546\ttotal: 362ms\tremaining: 523ms\n",
      "409:\tlearn: 0.0970197\ttotal: 363ms\tremaining: 522ms\n",
      "410:\tlearn: 0.0964881\ttotal: 363ms\tremaining: 521ms\n",
      "411:\tlearn: 0.0960357\ttotal: 364ms\tremaining: 519ms\n",
      "412:\tlearn: 0.0955100\ttotal: 364ms\tremaining: 518ms\n",
      "413:\tlearn: 0.0950620\ttotal: 365ms\tremaining: 517ms\n",
      "414:\tlearn: 0.0946169\ttotal: 366ms\tremaining: 515ms\n",
      "415:\tlearn: 0.0940993\ttotal: 366ms\tremaining: 514ms\n",
      "416:\tlearn: 0.0935849\ttotal: 367ms\tremaining: 513ms\n",
      "417:\tlearn: 0.0931470\ttotal: 367ms\tremaining: 511ms\n",
      "418:\tlearn: 0.0926382\ttotal: 368ms\tremaining: 510ms\n",
      "419:\tlearn: 0.0921326\ttotal: 369ms\tremaining: 509ms\n",
      "420:\tlearn: 0.0916301\ttotal: 369ms\tremaining: 508ms\n",
      "421:\tlearn: 0.0911307\ttotal: 370ms\tremaining: 506ms\n",
      "422:\tlearn: 0.0906344\ttotal: 370ms\tremaining: 505ms\n",
      "423:\tlearn: 0.0901412\ttotal: 371ms\tremaining: 504ms\n",
      "424:\tlearn: 0.0896510\ttotal: 371ms\tremaining: 502ms\n",
      "425:\tlearn: 0.0891638\ttotal: 372ms\tremaining: 501ms\n",
      "426:\tlearn: 0.0886797\ttotal: 372ms\tremaining: 500ms\n",
      "427:\tlearn: 0.0881985\ttotal: 373ms\tremaining: 498ms\n",
      "428:\tlearn: 0.0877203\ttotal: 374ms\tremaining: 497ms\n",
      "429:\tlearn: 0.0872450\ttotal: 374ms\tremaining: 496ms\n",
      "430:\tlearn: 0.0867727\ttotal: 375ms\tremaining: 495ms\n",
      "431:\tlearn: 0.0863032\ttotal: 375ms\tremaining: 493ms\n",
      "432:\tlearn: 0.0858367\ttotal: 376ms\tremaining: 492ms\n",
      "433:\tlearn: 0.0853730\ttotal: 376ms\tremaining: 491ms\n",
      "434:\tlearn: 0.0849121\ttotal: 377ms\tremaining: 490ms\n",
      "435:\tlearn: 0.0844541\ttotal: 378ms\tremaining: 488ms\n",
      "436:\tlearn: 0.0839989\ttotal: 378ms\tremaining: 487ms\n",
      "437:\tlearn: 0.0835465\ttotal: 379ms\tremaining: 486ms\n",
      "438:\tlearn: 0.0830968\ttotal: 379ms\tremaining: 485ms\n",
      "439:\tlearn: 0.0826499\ttotal: 380ms\tremaining: 483ms\n",
      "440:\tlearn: 0.0822057\ttotal: 380ms\tremaining: 482ms\n",
      "441:\tlearn: 0.0817642\ttotal: 381ms\tremaining: 481ms\n",
      "442:\tlearn: 0.0813254\ttotal: 382ms\tremaining: 480ms\n",
      "443:\tlearn: 0.0808893\ttotal: 382ms\tremaining: 479ms\n",
      "444:\tlearn: 0.0804559\ttotal: 383ms\tremaining: 477ms\n",
      "445:\tlearn: 0.0800251\ttotal: 383ms\tremaining: 476ms\n",
      "446:\tlearn: 0.0795969\ttotal: 384ms\tremaining: 475ms\n",
      "447:\tlearn: 0.0791713\ttotal: 384ms\tremaining: 474ms\n",
      "448:\tlearn: 0.0787483\ttotal: 385ms\tremaining: 472ms\n",
      "449:\tlearn: 0.0783279\ttotal: 386ms\tremaining: 471ms\n",
      "450:\tlearn: 0.0779100\ttotal: 386ms\tremaining: 470ms\n",
      "451:\tlearn: 0.0774946\ttotal: 387ms\tremaining: 469ms\n",
      "452:\tlearn: 0.0770818\ttotal: 387ms\tremaining: 468ms\n",
      "453:\tlearn: 0.0766715\ttotal: 388ms\tremaining: 466ms\n",
      "454:\tlearn: 0.0762636\ttotal: 388ms\tremaining: 465ms\n",
      "455:\tlearn: 0.0758582\ttotal: 389ms\tremaining: 464ms\n",
      "456:\tlearn: 0.0754553\ttotal: 389ms\tremaining: 463ms\n",
      "457:\tlearn: 0.0750548\ttotal: 390ms\tremaining: 462ms\n",
      "458:\tlearn: 0.0746567\ttotal: 391ms\tremaining: 460ms\n",
      "459:\tlearn: 0.0742611\ttotal: 391ms\tremaining: 459ms\n",
      "460:\tlearn: 0.0738678\ttotal: 392ms\tremaining: 458ms\n",
      "461:\tlearn: 0.0735217\ttotal: 392ms\tremaining: 457ms\n",
      "462:\tlearn: 0.0731823\ttotal: 393ms\tremaining: 456ms\n",
      "463:\tlearn: 0.0728449\ttotal: 393ms\tremaining: 454ms\n",
      "464:\tlearn: 0.0725043\ttotal: 394ms\tremaining: 453ms\n",
      "465:\tlearn: 0.0721204\ttotal: 394ms\tremaining: 452ms\n",
      "466:\tlearn: 0.0717832\ttotal: 395ms\tremaining: 451ms\n",
      "467:\tlearn: 0.0714477\ttotal: 396ms\tremaining: 450ms\n",
      "468:\tlearn: 0.0710695\ttotal: 396ms\tremaining: 448ms\n",
      "469:\tlearn: 0.0707424\ttotal: 397ms\tremaining: 447ms\n",
      "470:\tlearn: 0.0704121\ttotal: 397ms\tremaining: 446ms\n",
      "471:\tlearn: 0.0700833\ttotal: 398ms\tremaining: 445ms\n",
      "472:\tlearn: 0.0697123\ttotal: 398ms\tremaining: 444ms\n",
      "473:\tlearn: 0.0693869\ttotal: 399ms\tremaining: 443ms\n",
      "474:\tlearn: 0.0690632\ttotal: 399ms\tremaining: 441ms\n",
      "475:\tlearn: 0.0686976\ttotal: 400ms\tremaining: 440ms\n",
      "476:\tlearn: 0.0683822\ttotal: 400ms\tremaining: 439ms\n",
      "477:\tlearn: 0.0680633\ttotal: 401ms\tremaining: 438ms\n",
      "478:\tlearn: 0.0677461\ttotal: 402ms\tremaining: 437ms\n",
      "479:\tlearn: 0.0674304\ttotal: 402ms\tremaining: 436ms\n",
      "480:\tlearn: 0.0671165\ttotal: 403ms\tremaining: 435ms\n",
      "481:\tlearn: 0.0667611\ttotal: 403ms\tremaining: 433ms\n",
      "482:\tlearn: 0.0664078\ttotal: 404ms\tremaining: 432ms\n",
      "483:\tlearn: 0.0660984\ttotal: 404ms\tremaining: 431ms\n",
      "484:\tlearn: 0.0657906\ttotal: 405ms\tremaining: 430ms\n",
      "485:\tlearn: 0.0654426\ttotal: 406ms\tremaining: 429ms\n",
      "486:\tlearn: 0.0651380\ttotal: 406ms\tremaining: 428ms\n",
      "487:\tlearn: 0.0647935\ttotal: 407ms\tremaining: 427ms\n",
      "488:\tlearn: 0.0644918\ttotal: 407ms\tremaining: 426ms\n",
      "489:\tlearn: 0.0641971\ttotal: 408ms\tremaining: 424ms\n",
      "490:\tlearn: 0.0638985\ttotal: 408ms\tremaining: 423ms\n",
      "491:\tlearn: 0.0635606\ttotal: 409ms\tremaining: 422ms\n",
      "492:\tlearn: 0.0632353\ttotal: 409ms\tremaining: 421ms\n",
      "493:\tlearn: 0.0629119\ttotal: 410ms\tremaining: 420ms\n",
      "494:\tlearn: 0.0625904\ttotal: 411ms\tremaining: 419ms\n",
      "495:\tlearn: 0.0622708\ttotal: 411ms\tremaining: 418ms\n",
      "496:\tlearn: 0.0619532\ttotal: 412ms\tremaining: 417ms\n",
      "497:\tlearn: 0.0616374\ttotal: 412ms\tremaining: 416ms\n",
      "498:\tlearn: 0.0613234\ttotal: 413ms\tremaining: 414ms\n",
      "499:\tlearn: 0.0610113\ttotal: 413ms\tremaining: 413ms\n",
      "500:\tlearn: 0.0607010\ttotal: 414ms\tremaining: 412ms\n",
      "501:\tlearn: 0.0603926\ttotal: 414ms\tremaining: 411ms\n",
      "502:\tlearn: 0.0600859\ttotal: 415ms\tremaining: 410ms\n",
      "503:\tlearn: 0.0597811\ttotal: 416ms\tremaining: 409ms\n",
      "504:\tlearn: 0.0594780\ttotal: 416ms\tremaining: 408ms\n",
      "505:\tlearn: 0.0591768\ttotal: 417ms\tremaining: 407ms\n",
      "506:\tlearn: 0.0588772\ttotal: 417ms\tremaining: 406ms\n",
      "507:\tlearn: 0.0585794\ttotal: 418ms\tremaining: 405ms\n",
      "508:\tlearn: 0.0582834\ttotal: 418ms\tremaining: 404ms\n",
      "509:\tlearn: 0.0579891\ttotal: 419ms\tremaining: 403ms\n",
      "510:\tlearn: 0.0576964\ttotal: 420ms\tremaining: 402ms\n",
      "511:\tlearn: 0.0574055\ttotal: 420ms\tremaining: 400ms\n",
      "512:\tlearn: 0.0571163\ttotal: 421ms\tremaining: 399ms\n",
      "513:\tlearn: 0.0568287\ttotal: 421ms\tremaining: 398ms\n",
      "514:\tlearn: 0.0565428\ttotal: 422ms\tremaining: 397ms\n",
      "515:\tlearn: 0.0562586\ttotal: 422ms\tremaining: 396ms\n",
      "516:\tlearn: 0.0559760\ttotal: 423ms\tremaining: 395ms\n",
      "517:\tlearn: 0.0556950\ttotal: 424ms\tremaining: 394ms\n",
      "518:\tlearn: 0.0554157\ttotal: 424ms\tremaining: 393ms\n",
      "519:\tlearn: 0.0551379\ttotal: 425ms\tremaining: 392ms\n",
      "520:\tlearn: 0.0548618\ttotal: 425ms\tremaining: 391ms\n",
      "521:\tlearn: 0.0545872\ttotal: 426ms\tremaining: 390ms\n",
      "522:\tlearn: 0.0543398\ttotal: 426ms\tremaining: 389ms\n",
      "523:\tlearn: 0.0540682\ttotal: 427ms\tremaining: 388ms\n",
      "524:\tlearn: 0.0537981\ttotal: 427ms\tremaining: 387ms\n",
      "525:\tlearn: 0.0535295\ttotal: 428ms\tremaining: 386ms\n",
      "526:\tlearn: 0.0532625\ttotal: 429ms\tremaining: 385ms\n",
      "527:\tlearn: 0.0529970\ttotal: 429ms\tremaining: 384ms\n",
      "528:\tlearn: 0.0527330\ttotal: 430ms\tremaining: 383ms\n",
      "529:\tlearn: 0.0524705\ttotal: 430ms\tremaining: 381ms\n",
      "530:\tlearn: 0.0522095\ttotal: 431ms\tremaining: 380ms\n",
      "531:\tlearn: 0.0519501\ttotal: 431ms\tremaining: 379ms\n",
      "532:\tlearn: 0.0516921\ttotal: 432ms\tremaining: 378ms\n",
      "533:\tlearn: 0.0514355\ttotal: 432ms\tremaining: 377ms\n",
      "534:\tlearn: 0.0511805\ttotal: 433ms\tremaining: 376ms\n",
      "535:\tlearn: 0.0509268\ttotal: 434ms\tremaining: 375ms\n",
      "536:\tlearn: 0.0506747\ttotal: 434ms\tremaining: 374ms\n",
      "537:\tlearn: 0.0504239\ttotal: 435ms\tremaining: 373ms\n",
      "538:\tlearn: 0.0501746\ttotal: 435ms\tremaining: 372ms\n",
      "539:\tlearn: 0.0499267\ttotal: 436ms\tremaining: 371ms\n",
      "540:\tlearn: 0.0496802\ttotal: 436ms\tremaining: 370ms\n",
      "541:\tlearn: 0.0494351\ttotal: 437ms\tremaining: 369ms\n",
      "542:\tlearn: 0.0491914\ttotal: 438ms\tremaining: 368ms\n",
      "543:\tlearn: 0.0489491\ttotal: 438ms\tremaining: 367ms\n",
      "544:\tlearn: 0.0487081\ttotal: 439ms\tremaining: 366ms\n",
      "545:\tlearn: 0.0484686\ttotal: 440ms\tremaining: 366ms\n",
      "546:\tlearn: 0.0482303\ttotal: 440ms\tremaining: 365ms\n",
      "547:\tlearn: 0.0479934\ttotal: 441ms\tremaining: 364ms\n",
      "548:\tlearn: 0.0477579\ttotal: 442ms\tremaining: 363ms\n",
      "549:\tlearn: 0.0475237\ttotal: 442ms\tremaining: 362ms\n",
      "550:\tlearn: 0.0472908\ttotal: 443ms\tremaining: 361ms\n",
      "551:\tlearn: 0.0470592\ttotal: 443ms\tremaining: 360ms\n",
      "552:\tlearn: 0.0468289\ttotal: 444ms\tremaining: 359ms\n",
      "553:\tlearn: 0.0465999\ttotal: 444ms\tremaining: 358ms\n",
      "554:\tlearn: 0.0463722\ttotal: 445ms\tremaining: 357ms\n",
      "555:\tlearn: 0.0461458\ttotal: 446ms\tremaining: 356ms\n",
      "556:\tlearn: 0.0459207\ttotal: 446ms\tremaining: 355ms\n",
      "557:\tlearn: 0.0456968\ttotal: 447ms\tremaining: 354ms\n",
      "558:\tlearn: 0.0454742\ttotal: 447ms\tremaining: 353ms\n",
      "559:\tlearn: 0.0452529\ttotal: 448ms\tremaining: 352ms\n",
      "560:\tlearn: 0.0450328\ttotal: 449ms\tremaining: 351ms\n",
      "561:\tlearn: 0.0448139\ttotal: 449ms\tremaining: 350ms\n",
      "562:\tlearn: 0.0445962\ttotal: 450ms\tremaining: 349ms\n",
      "563:\tlearn: 0.0443798\ttotal: 450ms\tremaining: 348ms\n",
      "564:\tlearn: 0.0441646\ttotal: 451ms\tremaining: 347ms\n",
      "565:\tlearn: 0.0439506\ttotal: 452ms\tremaining: 346ms\n",
      "566:\tlearn: 0.0437378\ttotal: 452ms\tremaining: 345ms\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "567:\tlearn: 0.0435262\ttotal: 453ms\tremaining: 344ms\n",
      "568:\tlearn: 0.0433158\ttotal: 454ms\tremaining: 344ms\n",
      "569:\tlearn: 0.0431066\ttotal: 454ms\tremaining: 343ms\n",
      "570:\tlearn: 0.0428985\ttotal: 455ms\tremaining: 342ms\n",
      "571:\tlearn: 0.0426916\ttotal: 455ms\tremaining: 341ms\n",
      "572:\tlearn: 0.0424858\ttotal: 456ms\tremaining: 340ms\n",
      "573:\tlearn: 0.0422813\ttotal: 457ms\tremaining: 339ms\n",
      "574:\tlearn: 0.0420778\ttotal: 457ms\tremaining: 338ms\n",
      "575:\tlearn: 0.0418755\ttotal: 458ms\tremaining: 337ms\n",
      "576:\tlearn: 0.0416743\ttotal: 458ms\tremaining: 336ms\n",
      "577:\tlearn: 0.0414743\ttotal: 459ms\tremaining: 335ms\n",
      "578:\tlearn: 0.0412754\ttotal: 459ms\tremaining: 334ms\n",
      "579:\tlearn: 0.0410776\ttotal: 460ms\tremaining: 333ms\n",
      "580:\tlearn: 0.0408809\ttotal: 461ms\tremaining: 332ms\n",
      "581:\tlearn: 0.0406584\ttotal: 461ms\tremaining: 331ms\n",
      "582:\tlearn: 0.0404374\ttotal: 462ms\tremaining: 330ms\n",
      "583:\tlearn: 0.0402437\ttotal: 462ms\tremaining: 329ms\n",
      "584:\tlearn: 0.0400249\ttotal: 463ms\tremaining: 328ms\n",
      "585:\tlearn: 0.0398075\ttotal: 464ms\tremaining: 328ms\n",
      "586:\tlearn: 0.0395915\ttotal: 464ms\tremaining: 327ms\n",
      "587:\tlearn: 0.0394017\ttotal: 465ms\tremaining: 326ms\n",
      "588:\tlearn: 0.0391880\ttotal: 466ms\tremaining: 325ms\n",
      "589:\tlearn: 0.0389755\ttotal: 467ms\tremaining: 325ms\n",
      "590:\tlearn: 0.0387644\ttotal: 468ms\tremaining: 324ms\n",
      "591:\tlearn: 0.0385784\ttotal: 468ms\tremaining: 323ms\n",
      "592:\tlearn: 0.0383695\ttotal: 469ms\tremaining: 322ms\n",
      "593:\tlearn: 0.0381619\ttotal: 469ms\tremaining: 321ms\n",
      "594:\tlearn: 0.0379555\ttotal: 470ms\tremaining: 320ms\n",
      "595:\tlearn: 0.0377505\ttotal: 471ms\tremaining: 319ms\n",
      "596:\tlearn: 0.0375467\ttotal: 471ms\tremaining: 318ms\n",
      "597:\tlearn: 0.0373441\ttotal: 472ms\tremaining: 317ms\n",
      "598:\tlearn: 0.0371645\ttotal: 472ms\tremaining: 316ms\n",
      "599:\tlearn: 0.0369641\ttotal: 473ms\tremaining: 315ms\n",
      "600:\tlearn: 0.0367649\ttotal: 474ms\tremaining: 314ms\n",
      "601:\tlearn: 0.0365881\ttotal: 474ms\tremaining: 313ms\n",
      "602:\tlearn: 0.0363910\ttotal: 475ms\tremaining: 313ms\n",
      "603:\tlearn: 0.0362004\ttotal: 475ms\tremaining: 312ms\n",
      "604:\tlearn: 0.0360110\ttotal: 476ms\tremaining: 311ms\n",
      "605:\tlearn: 0.0358377\ttotal: 476ms\tremaining: 310ms\n",
      "606:\tlearn: 0.0356502\ttotal: 477ms\tremaining: 309ms\n",
      "607:\tlearn: 0.0354639\ttotal: 478ms\tremaining: 308ms\n",
      "608:\tlearn: 0.0352787\ttotal: 478ms\tremaining: 307ms\n",
      "609:\tlearn: 0.0351088\ttotal: 479ms\tremaining: 306ms\n",
      "610:\tlearn: 0.0349255\ttotal: 479ms\tremaining: 305ms\n",
      "611:\tlearn: 0.0347433\ttotal: 480ms\tremaining: 304ms\n",
      "612:\tlearn: 0.0345622\ttotal: 480ms\tremaining: 303ms\n",
      "613:\tlearn: 0.0343822\ttotal: 481ms\tremaining: 302ms\n",
      "614:\tlearn: 0.0342165\ttotal: 482ms\tremaining: 301ms\n",
      "615:\tlearn: 0.0340383\ttotal: 482ms\tremaining: 301ms\n",
      "616:\tlearn: 0.0338612\ttotal: 483ms\tremaining: 300ms\n",
      "617:\tlearn: 0.0336980\ttotal: 483ms\tremaining: 299ms\n",
      "618:\tlearn: 0.0335228\ttotal: 484ms\tremaining: 298ms\n",
      "619:\tlearn: 0.0333485\ttotal: 484ms\tremaining: 297ms\n",
      "620:\tlearn: 0.0331753\ttotal: 485ms\tremaining: 296ms\n",
      "621:\tlearn: 0.0330031\ttotal: 485ms\tremaining: 295ms\n",
      "622:\tlearn: 0.0328320\ttotal: 486ms\tremaining: 294ms\n",
      "623:\tlearn: 0.0326618\ttotal: 487ms\tremaining: 293ms\n",
      "624:\tlearn: 0.0325086\ttotal: 487ms\tremaining: 292ms\n",
      "625:\tlearn: 0.0323403\ttotal: 488ms\tremaining: 291ms\n",
      "626:\tlearn: 0.0321729\ttotal: 488ms\tremaining: 291ms\n",
      "627:\tlearn: 0.0320066\ttotal: 489ms\tremaining: 290ms\n",
      "628:\tlearn: 0.0318412\ttotal: 490ms\tremaining: 289ms\n",
      "629:\tlearn: 0.0316768\ttotal: 490ms\tremaining: 288ms\n",
      "630:\tlearn: 0.0315133\ttotal: 491ms\tremaining: 287ms\n",
      "631:\tlearn: 0.0313508\ttotal: 491ms\tremaining: 286ms\n",
      "632:\tlearn: 0.0311893\ttotal: 492ms\tremaining: 285ms\n",
      "633:\tlearn: 0.0310287\ttotal: 492ms\tremaining: 284ms\n",
      "634:\tlearn: 0.0308690\ttotal: 493ms\tremaining: 283ms\n",
      "635:\tlearn: 0.0307103\ttotal: 494ms\tremaining: 282ms\n",
      "636:\tlearn: 0.0305525\ttotal: 494ms\tremaining: 282ms\n",
      "637:\tlearn: 0.0303956\ttotal: 495ms\tremaining: 281ms\n",
      "638:\tlearn: 0.0302396\ttotal: 495ms\tremaining: 280ms\n",
      "639:\tlearn: 0.0300846\ttotal: 496ms\tremaining: 279ms\n",
      "640:\tlearn: 0.0299304\ttotal: 496ms\tremaining: 278ms\n",
      "641:\tlearn: 0.0297772\ttotal: 497ms\tremaining: 277ms\n",
      "642:\tlearn: 0.0296248\ttotal: 497ms\tremaining: 276ms\n",
      "643:\tlearn: 0.0294733\ttotal: 498ms\tremaining: 275ms\n",
      "644:\tlearn: 0.0293227\ttotal: 499ms\tremaining: 274ms\n",
      "645:\tlearn: 0.0291730\ttotal: 499ms\tremaining: 274ms\n",
      "646:\tlearn: 0.0290241\ttotal: 500ms\tremaining: 273ms\n",
      "647:\tlearn: 0.0288761\ttotal: 500ms\tremaining: 272ms\n",
      "648:\tlearn: 0.0287290\ttotal: 501ms\tremaining: 271ms\n",
      "649:\tlearn: 0.0285827\ttotal: 501ms\tremaining: 270ms\n",
      "650:\tlearn: 0.0284372\ttotal: 502ms\tremaining: 269ms\n",
      "651:\tlearn: 0.0282926\ttotal: 502ms\tremaining: 268ms\n",
      "652:\tlearn: 0.0281489\ttotal: 503ms\tremaining: 267ms\n",
      "653:\tlearn: 0.0280059\ttotal: 504ms\tremaining: 266ms\n",
      "654:\tlearn: 0.0278638\ttotal: 504ms\tremaining: 266ms\n",
      "655:\tlearn: 0.0277225\ttotal: 505ms\tremaining: 265ms\n",
      "656:\tlearn: 0.0275820\ttotal: 505ms\tremaining: 264ms\n",
      "657:\tlearn: 0.0274423\ttotal: 506ms\tremaining: 263ms\n",
      "658:\tlearn: 0.0273034\ttotal: 507ms\tremaining: 262ms\n",
      "659:\tlearn: 0.0271653\ttotal: 507ms\tremaining: 261ms\n",
      "660:\tlearn: 0.0270280\ttotal: 508ms\tremaining: 260ms\n",
      "661:\tlearn: 0.0268915\ttotal: 508ms\tremaining: 260ms\n",
      "662:\tlearn: 0.0267558\ttotal: 509ms\tremaining: 259ms\n",
      "663:\tlearn: 0.0266208\ttotal: 509ms\tremaining: 258ms\n",
      "664:\tlearn: 0.0264867\ttotal: 510ms\tremaining: 257ms\n",
      "665:\tlearn: 0.0263533\ttotal: 511ms\tremaining: 256ms\n",
      "666:\tlearn: 0.0262206\ttotal: 511ms\tremaining: 255ms\n",
      "667:\tlearn: 0.0260887\ttotal: 512ms\tremaining: 254ms\n",
      "668:\tlearn: 0.0259576\ttotal: 512ms\tremaining: 253ms\n",
      "669:\tlearn: 0.0258272\ttotal: 513ms\tremaining: 253ms\n",
      "670:\tlearn: 0.0256975\ttotal: 513ms\tremaining: 252ms\n",
      "671:\tlearn: 0.0255686\ttotal: 514ms\tremaining: 251ms\n",
      "672:\tlearn: 0.0254405\ttotal: 515ms\tremaining: 250ms\n",
      "673:\tlearn: 0.0253130\ttotal: 515ms\tremaining: 249ms\n",
      "674:\tlearn: 0.0251863\ttotal: 516ms\tremaining: 248ms\n",
      "675:\tlearn: 0.0250603\ttotal: 516ms\tremaining: 247ms\n",
      "676:\tlearn: 0.0249350\ttotal: 517ms\tremaining: 247ms\n",
      "677:\tlearn: 0.0248105\ttotal: 517ms\tremaining: 246ms\n",
      "678:\tlearn: 0.0246866\ttotal: 518ms\tremaining: 245ms\n",
      "679:\tlearn: 0.0245635\ttotal: 519ms\tremaining: 244ms\n",
      "680:\tlearn: 0.0244410\ttotal: 519ms\tremaining: 243ms\n",
      "681:\tlearn: 0.0243192\ttotal: 520ms\tremaining: 242ms\n",
      "682:\tlearn: 0.0241982\ttotal: 520ms\tremaining: 241ms\n",
      "683:\tlearn: 0.0240778\ttotal: 521ms\tremaining: 241ms\n",
      "684:\tlearn: 0.0239581\ttotal: 521ms\tremaining: 240ms\n",
      "685:\tlearn: 0.0238391\ttotal: 522ms\tremaining: 239ms\n",
      "686:\tlearn: 0.0237207\ttotal: 523ms\tremaining: 238ms\n",
      "687:\tlearn: 0.0236030\ttotal: 523ms\tremaining: 237ms\n",
      "688:\tlearn: 0.0234860\ttotal: 524ms\tremaining: 236ms\n",
      "689:\tlearn: 0.0233697\ttotal: 524ms\tremaining: 236ms\n",
      "690:\tlearn: 0.0232540\ttotal: 525ms\tremaining: 235ms\n",
      "691:\tlearn: 0.0231389\ttotal: 525ms\tremaining: 234ms\n",
      "692:\tlearn: 0.0230245\ttotal: 526ms\tremaining: 233ms\n",
      "693:\tlearn: 0.0229108\ttotal: 527ms\tremaining: 232ms\n",
      "694:\tlearn: 0.0227977\ttotal: 527ms\tremaining: 231ms\n",
      "695:\tlearn: 0.0226852\ttotal: 528ms\tremaining: 230ms\n",
      "696:\tlearn: 0.0225734\ttotal: 528ms\tremaining: 230ms\n",
      "697:\tlearn: 0.0224622\ttotal: 529ms\tremaining: 229ms\n",
      "698:\tlearn: 0.0223516\ttotal: 529ms\tremaining: 228ms\n",
      "699:\tlearn: 0.0222417\ttotal: 530ms\tremaining: 227ms\n",
      "700:\tlearn: 0.0221323\ttotal: 530ms\tremaining: 226ms\n",
      "701:\tlearn: 0.0220236\ttotal: 531ms\tremaining: 225ms\n",
      "702:\tlearn: 0.0219155\ttotal: 532ms\tremaining: 225ms\n",
      "703:\tlearn: 0.0218080\ttotal: 532ms\tremaining: 224ms\n",
      "704:\tlearn: 0.0217011\ttotal: 533ms\tremaining: 223ms\n",
      "705:\tlearn: 0.0215948\ttotal: 533ms\tremaining: 222ms\n",
      "706:\tlearn: 0.0214892\ttotal: 534ms\tremaining: 221ms\n",
      "707:\tlearn: 0.0213841\ttotal: 534ms\tremaining: 220ms\n",
      "708:\tlearn: 0.0212796\ttotal: 535ms\tremaining: 220ms\n",
      "709:\tlearn: 0.0211756\ttotal: 536ms\tremaining: 219ms\n",
      "710:\tlearn: 0.0210723\ttotal: 536ms\tremaining: 218ms\n",
      "711:\tlearn: 0.0209696\ttotal: 537ms\tremaining: 217ms\n",
      "712:\tlearn: 0.0208674\ttotal: 537ms\tremaining: 216ms\n",
      "713:\tlearn: 0.0207658\ttotal: 538ms\tremaining: 215ms\n",
      "714:\tlearn: 0.0206648\ttotal: 538ms\tremaining: 215ms\n",
      "715:\tlearn: 0.0205643\ttotal: 539ms\tremaining: 214ms\n",
      "716:\tlearn: 0.0204644\ttotal: 540ms\tremaining: 213ms\n",
      "717:\tlearn: 0.0203651\ttotal: 540ms\tremaining: 212ms\n",
      "718:\tlearn: 0.0202663\ttotal: 541ms\tremaining: 211ms\n",
      "719:\tlearn: 0.0201681\ttotal: 541ms\tremaining: 210ms\n",
      "720:\tlearn: 0.0200705\ttotal: 542ms\tremaining: 210ms\n",
      "721:\tlearn: 0.0199733\ttotal: 542ms\tremaining: 209ms\n",
      "722:\tlearn: 0.0198768\ttotal: 543ms\tremaining: 208ms\n",
      "723:\tlearn: 0.0197807\ttotal: 544ms\tremaining: 207ms\n",
      "724:\tlearn: 0.0196853\ttotal: 544ms\tremaining: 206ms\n",
      "725:\tlearn: 0.0195903\ttotal: 545ms\tremaining: 206ms\n",
      "726:\tlearn: 0.0194959\ttotal: 545ms\tremaining: 205ms\n",
      "727:\tlearn: 0.0194020\ttotal: 546ms\tremaining: 204ms\n",
      "728:\tlearn: 0.0193086\ttotal: 547ms\tremaining: 203ms\n",
      "729:\tlearn: 0.0192158\ttotal: 548ms\tremaining: 203ms\n",
      "730:\tlearn: 0.0191235\ttotal: 548ms\tremaining: 202ms\n",
      "731:\tlearn: 0.0190317\ttotal: 549ms\tremaining: 201ms\n",
      "732:\tlearn: 0.0189404\ttotal: 550ms\tremaining: 200ms\n",
      "733:\tlearn: 0.0188497\ttotal: 550ms\tremaining: 199ms\n",
      "734:\tlearn: 0.0187594\ttotal: 551ms\tremaining: 199ms\n",
      "735:\tlearn: 0.0186696\ttotal: 551ms\tremaining: 198ms\n",
      "736:\tlearn: 0.0185804\ttotal: 552ms\tremaining: 197ms\n",
      "737:\tlearn: 0.0184917\ttotal: 553ms\tremaining: 196ms\n",
      "738:\tlearn: 0.0184034\ttotal: 553ms\tremaining: 195ms\n",
      "739:\tlearn: 0.0183157\ttotal: 554ms\tremaining: 195ms\n",
      "740:\tlearn: 0.0182284\ttotal: 555ms\tremaining: 194ms\n",
      "741:\tlearn: 0.0181417\ttotal: 555ms\tremaining: 193ms\n",
      "742:\tlearn: 0.0180554\ttotal: 556ms\tremaining: 192ms\n",
      "743:\tlearn: 0.0179696\ttotal: 556ms\tremaining: 191ms\n",
      "744:\tlearn: 0.0178875\ttotal: 557ms\tremaining: 191ms\n",
      "745:\tlearn: 0.0178027\ttotal: 558ms\tremaining: 190ms\n",
      "746:\tlearn: 0.0177182\ttotal: 559ms\tremaining: 189ms\n",
      "747:\tlearn: 0.0176343\ttotal: 559ms\tremaining: 188ms\n",
      "748:\tlearn: 0.0175540\ttotal: 560ms\tremaining: 188ms\n",
      "749:\tlearn: 0.0174723\ttotal: 560ms\tremaining: 187ms\n",
      "750:\tlearn: 0.0173910\ttotal: 561ms\tremaining: 186ms\n",
      "751:\tlearn: 0.0173102\ttotal: 562ms\tremaining: 185ms\n",
      "752:\tlearn: 0.0172298\ttotal: 562ms\tremaining: 184ms\n",
      "753:\tlearn: 0.0171497\ttotal: 563ms\tremaining: 184ms\n",
      "754:\tlearn: 0.0170701\ttotal: 563ms\tremaining: 183ms\n",
      "755:\tlearn: 0.0169909\ttotal: 564ms\tremaining: 182ms\n",
      "756:\tlearn: 0.0169121\ttotal: 565ms\tremaining: 181ms\n",
      "757:\tlearn: 0.0168337\ttotal: 565ms\tremaining: 180ms\n",
      "758:\tlearn: 0.0167557\ttotal: 566ms\tremaining: 180ms\n",
      "759:\tlearn: 0.0166782\ttotal: 566ms\tremaining: 179ms\n",
      "760:\tlearn: 0.0166010\ttotal: 567ms\tremaining: 178ms\n",
      "761:\tlearn: 0.0165242\ttotal: 567ms\tremaining: 177ms\n",
      "762:\tlearn: 0.0164478\ttotal: 568ms\tremaining: 176ms\n",
      "763:\tlearn: 0.0163717\ttotal: 569ms\tremaining: 176ms\n",
      "764:\tlearn: 0.0162961\ttotal: 569ms\tremaining: 175ms\n",
      "765:\tlearn: 0.0162209\ttotal: 570ms\tremaining: 174ms\n",
      "766:\tlearn: 0.0161460\ttotal: 570ms\tremaining: 173ms\n",
      "767:\tlearn: 0.0160715\ttotal: 571ms\tremaining: 172ms\n",
      "768:\tlearn: 0.0159975\ttotal: 571ms\tremaining: 172ms\n",
      "769:\tlearn: 0.0159237\ttotal: 572ms\tremaining: 171ms\n",
      "770:\tlearn: 0.0158504\ttotal: 573ms\tremaining: 170ms\n",
      "771:\tlearn: 0.0157775\ttotal: 573ms\tremaining: 169ms\n",
      "772:\tlearn: 0.0157049\ttotal: 574ms\tremaining: 168ms\n",
      "773:\tlearn: 0.0156327\ttotal: 574ms\tremaining: 168ms\n",
      "774:\tlearn: 0.0155608\ttotal: 575ms\tremaining: 167ms\n",
      "775:\tlearn: 0.0154893\ttotal: 575ms\tremaining: 166ms\n",
      "776:\tlearn: 0.0154182\ttotal: 576ms\tremaining: 165ms\n",
      "777:\tlearn: 0.0153475\ttotal: 577ms\tremaining: 165ms\n",
      "778:\tlearn: 0.0152771\ttotal: 577ms\tremaining: 164ms\n",
      "779:\tlearn: 0.0152071\ttotal: 578ms\tremaining: 163ms\n",
      "780:\tlearn: 0.0151374\ttotal: 578ms\tremaining: 162ms\n",
      "781:\tlearn: 0.0150681\ttotal: 579ms\tremaining: 161ms\n",
      "782:\tlearn: 0.0149992\ttotal: 579ms\tremaining: 161ms\n",
      "783:\tlearn: 0.0149306\ttotal: 580ms\tremaining: 160ms\n",
      "784:\tlearn: 0.0148623\ttotal: 580ms\tremaining: 159ms\n",
      "785:\tlearn: 0.0147944\ttotal: 581ms\tremaining: 158ms\n",
      "786:\tlearn: 0.0147269\ttotal: 582ms\tremaining: 157ms\n",
      "787:\tlearn: 0.0146597\ttotal: 582ms\tremaining: 157ms\n",
      "788:\tlearn: 0.0145928\ttotal: 583ms\tremaining: 156ms\n",
      "789:\tlearn: 0.0145263\ttotal: 583ms\tremaining: 155ms\n",
      "790:\tlearn: 0.0144601\ttotal: 584ms\tremaining: 154ms\n",
      "791:\tlearn: 0.0143943\ttotal: 585ms\tremaining: 154ms\n",
      "792:\tlearn: 0.0143138\ttotal: 585ms\tremaining: 153ms\n",
      "793:\tlearn: 0.0142338\ttotal: 586ms\tremaining: 152ms\n",
      "794:\tlearn: 0.0141543\ttotal: 586ms\tremaining: 151ms\n",
      "795:\tlearn: 0.0140753\ttotal: 587ms\tremaining: 150ms\n",
      "796:\tlearn: 0.0139969\ttotal: 587ms\tremaining: 150ms\n",
      "797:\tlearn: 0.0139189\ttotal: 588ms\tremaining: 149ms\n",
      "798:\tlearn: 0.0138414\ttotal: 589ms\tremaining: 148ms\n",
      "799:\tlearn: 0.0137643\ttotal: 589ms\tremaining: 147ms\n",
      "800:\tlearn: 0.0136878\ttotal: 590ms\tremaining: 146ms\n",
      "801:\tlearn: 0.0136212\ttotal: 590ms\tremaining: 146ms\n",
      "802:\tlearn: 0.0135549\ttotal: 591ms\tremaining: 145ms\n",
      "803:\tlearn: 0.0134796\ttotal: 591ms\tremaining: 144ms\n",
      "804:\tlearn: 0.0134141\ttotal: 592ms\tremaining: 143ms\n",
      "805:\tlearn: 0.0133395\ttotal: 592ms\tremaining: 143ms\n",
      "806:\tlearn: 0.0132804\ttotal: 593ms\tremaining: 142ms\n",
      "807:\tlearn: 0.0132205\ttotal: 593ms\tremaining: 141ms\n",
      "808:\tlearn: 0.0131471\ttotal: 594ms\tremaining: 140ms\n",
      "809:\tlearn: 0.0130888\ttotal: 594ms\tremaining: 139ms\n",
      "810:\tlearn: 0.0130299\ttotal: 595ms\tremaining: 139ms\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "811:\tlearn: 0.0129712\ttotal: 596ms\tremaining: 138ms\n",
      "812:\tlearn: 0.0128991\ttotal: 596ms\tremaining: 137ms\n",
      "813:\tlearn: 0.0128365\ttotal: 597ms\tremaining: 136ms\n",
      "814:\tlearn: 0.0127743\ttotal: 597ms\tremaining: 136ms\n",
      "815:\tlearn: 0.0127083\ttotal: 598ms\tremaining: 135ms\n",
      "816:\tlearn: 0.0126511\ttotal: 599ms\tremaining: 134ms\n",
      "817:\tlearn: 0.0125808\ttotal: 599ms\tremaining: 133ms\n",
      "818:\tlearn: 0.0125198\ttotal: 600ms\tremaining: 133ms\n",
      "819:\tlearn: 0.0124592\ttotal: 600ms\tremaining: 132ms\n",
      "820:\tlearn: 0.0123948\ttotal: 601ms\tremaining: 131ms\n",
      "821:\tlearn: 0.0123392\ttotal: 602ms\tremaining: 130ms\n",
      "822:\tlearn: 0.0122705\ttotal: 602ms\tremaining: 129ms\n",
      "823:\tlearn: 0.0122111\ttotal: 603ms\tremaining: 129ms\n",
      "824:\tlearn: 0.0121521\ttotal: 603ms\tremaining: 128ms\n",
      "825:\tlearn: 0.0120845\ttotal: 604ms\tremaining: 127ms\n",
      "826:\tlearn: 0.0120312\ttotal: 604ms\tremaining: 126ms\n",
      "827:\tlearn: 0.0119772\ttotal: 605ms\tremaining: 126ms\n",
      "828:\tlearn: 0.0119153\ttotal: 606ms\tremaining: 125ms\n",
      "829:\tlearn: 0.0118619\ttotal: 607ms\tremaining: 124ms\n",
      "830:\tlearn: 0.0118046\ttotal: 607ms\tremaining: 123ms\n",
      "831:\tlearn: 0.0117389\ttotal: 608ms\tremaining: 123ms\n",
      "832:\tlearn: 0.0116823\ttotal: 608ms\tremaining: 122ms\n",
      "833:\tlearn: 0.0116219\ttotal: 609ms\tremaining: 121ms\n",
      "834:\tlearn: 0.0115699\ttotal: 609ms\tremaining: 120ms\n",
      "835:\tlearn: 0.0115054\ttotal: 610ms\tremaining: 120ms\n",
      "836:\tlearn: 0.0114500\ttotal: 610ms\tremaining: 119ms\n",
      "837:\tlearn: 0.0113948\ttotal: 611ms\tremaining: 118ms\n",
      "838:\tlearn: 0.0113360\ttotal: 612ms\tremaining: 117ms\n",
      "839:\tlearn: 0.0112852\ttotal: 612ms\tremaining: 117ms\n",
      "840:\tlearn: 0.0112223\ttotal: 613ms\tremaining: 116ms\n",
      "841:\tlearn: 0.0111683\ttotal: 613ms\tremaining: 115ms\n",
      "842:\tlearn: 0.0111061\ttotal: 614ms\tremaining: 114ms\n",
      "843:\tlearn: 0.0110573\ttotal: 614ms\tremaining: 114ms\n",
      "844:\tlearn: 0.0110018\ttotal: 615ms\tremaining: 113ms\n",
      "845:\tlearn: 0.0109466\ttotal: 615ms\tremaining: 112ms\n",
      "846:\tlearn: 0.0108918\ttotal: 616ms\tremaining: 111ms\n",
      "847:\tlearn: 0.0108373\ttotal: 617ms\tremaining: 111ms\n",
      "848:\tlearn: 0.0107831\ttotal: 617ms\tremaining: 110ms\n",
      "849:\tlearn: 0.0107293\ttotal: 618ms\tremaining: 109ms\n",
      "850:\tlearn: 0.0106758\ttotal: 619ms\tremaining: 108ms\n",
      "851:\tlearn: 0.0106227\ttotal: 619ms\tremaining: 108ms\n",
      "852:\tlearn: 0.0105698\ttotal: 620ms\tremaining: 107ms\n",
      "853:\tlearn: 0.0105173\ttotal: 621ms\tremaining: 106ms\n",
      "854:\tlearn: 0.0104652\ttotal: 621ms\tremaining: 105ms\n",
      "855:\tlearn: 0.0104133\ttotal: 622ms\tremaining: 105ms\n",
      "856:\tlearn: 0.0103618\ttotal: 622ms\tremaining: 104ms\n",
      "857:\tlearn: 0.0103105\ttotal: 623ms\tremaining: 103ms\n",
      "858:\tlearn: 0.0102596\ttotal: 623ms\tremaining: 102ms\n",
      "859:\tlearn: 0.0102090\ttotal: 624ms\tremaining: 102ms\n",
      "860:\tlearn: 0.0101587\ttotal: 625ms\tremaining: 101ms\n",
      "861:\tlearn: 0.0101087\ttotal: 625ms\tremaining: 100ms\n",
      "862:\tlearn: 0.0100591\ttotal: 626ms\tremaining: 99.4ms\n",
      "863:\tlearn: 0.0100097\ttotal: 626ms\tremaining: 98.6ms\n",
      "864:\tlearn: 0.0099606\ttotal: 627ms\tremaining: 97.9ms\n",
      "865:\tlearn: 0.0099119\ttotal: 628ms\tremaining: 97.1ms\n",
      "866:\tlearn: 0.0098634\ttotal: 628ms\tremaining: 96.4ms\n",
      "867:\tlearn: 0.0098152\ttotal: 629ms\tremaining: 95.6ms\n",
      "868:\tlearn: 0.0097673\ttotal: 629ms\tremaining: 94.9ms\n",
      "869:\tlearn: 0.0097197\ttotal: 630ms\tremaining: 94.1ms\n",
      "870:\tlearn: 0.0096632\ttotal: 631ms\tremaining: 93.4ms\n",
      "871:\tlearn: 0.0096162\ttotal: 631ms\tremaining: 92.6ms\n",
      "872:\tlearn: 0.0095603\ttotal: 632ms\tremaining: 91.9ms\n",
      "873:\tlearn: 0.0095048\ttotal: 632ms\tremaining: 91.2ms\n",
      "874:\tlearn: 0.0094585\ttotal: 633ms\tremaining: 90.4ms\n",
      "875:\tlearn: 0.0094036\ttotal: 633ms\tremaining: 89.7ms\n",
      "876:\tlearn: 0.0093579\ttotal: 634ms\tremaining: 88.9ms\n",
      "877:\tlearn: 0.0093036\ttotal: 635ms\tremaining: 88.2ms\n",
      "878:\tlearn: 0.0092497\ttotal: 635ms\tremaining: 87.4ms\n",
      "879:\tlearn: 0.0092046\ttotal: 636ms\tremaining: 86.7ms\n",
      "880:\tlearn: 0.0091513\ttotal: 636ms\tremaining: 86ms\n",
      "881:\tlearn: 0.0091068\ttotal: 637ms\tremaining: 85.2ms\n",
      "882:\tlearn: 0.0090541\ttotal: 637ms\tremaining: 84.5ms\n",
      "883:\tlearn: 0.0090017\ttotal: 638ms\tremaining: 83.7ms\n",
      "884:\tlearn: 0.0089579\ttotal: 639ms\tremaining: 83ms\n",
      "885:\tlearn: 0.0089061\ttotal: 639ms\tremaining: 82.2ms\n",
      "886:\tlearn: 0.0088627\ttotal: 640ms\tremaining: 81.5ms\n",
      "887:\tlearn: 0.0088115\ttotal: 640ms\tremaining: 80.8ms\n",
      "888:\tlearn: 0.0087606\ttotal: 641ms\tremaining: 80ms\n",
      "889:\tlearn: 0.0087180\ttotal: 641ms\tremaining: 79.3ms\n",
      "890:\tlearn: 0.0086677\ttotal: 642ms\tremaining: 78.5ms\n",
      "891:\tlearn: 0.0086255\ttotal: 643ms\tremaining: 77.8ms\n",
      "892:\tlearn: 0.0085757\ttotal: 643ms\tremaining: 77.1ms\n",
      "893:\tlearn: 0.0085263\ttotal: 644ms\tremaining: 76.3ms\n",
      "894:\tlearn: 0.0084848\ttotal: 644ms\tremaining: 75.6ms\n",
      "895:\tlearn: 0.0084359\ttotal: 645ms\tremaining: 74.8ms\n",
      "896:\tlearn: 0.0083949\ttotal: 645ms\tremaining: 74.1ms\n",
      "897:\tlearn: 0.0083466\ttotal: 646ms\tremaining: 73.4ms\n",
      "898:\tlearn: 0.0082985\ttotal: 647ms\tremaining: 72.6ms\n",
      "899:\tlearn: 0.0082582\ttotal: 647ms\tremaining: 71.9ms\n",
      "900:\tlearn: 0.0082107\ttotal: 648ms\tremaining: 71.2ms\n",
      "901:\tlearn: 0.0081707\ttotal: 648ms\tremaining: 70.4ms\n",
      "902:\tlearn: 0.0081238\ttotal: 649ms\tremaining: 69.7ms\n",
      "903:\tlearn: 0.0080771\ttotal: 649ms\tremaining: 69ms\n",
      "904:\tlearn: 0.0080378\ttotal: 650ms\tremaining: 68.2ms\n",
      "905:\tlearn: 0.0079917\ttotal: 650ms\tremaining: 67.5ms\n",
      "906:\tlearn: 0.0079555\ttotal: 651ms\tremaining: 66.8ms\n",
      "907:\tlearn: 0.0079098\ttotal: 652ms\tremaining: 66ms\n",
      "908:\tlearn: 0.0078740\ttotal: 652ms\tremaining: 65.3ms\n",
      "909:\tlearn: 0.0078288\ttotal: 653ms\tremaining: 64.6ms\n",
      "910:\tlearn: 0.0077906\ttotal: 653ms\tremaining: 63.8ms\n",
      "911:\tlearn: 0.0077526\ttotal: 654ms\tremaining: 63.1ms\n",
      "912:\tlearn: 0.0077149\ttotal: 655ms\tremaining: 62.4ms\n",
      "913:\tlearn: 0.0076706\ttotal: 655ms\tremaining: 61.6ms\n",
      "914:\tlearn: 0.0076359\ttotal: 656ms\tremaining: 60.9ms\n",
      "915:\tlearn: 0.0075921\ttotal: 656ms\tremaining: 60.2ms\n",
      "916:\tlearn: 0.0075551\ttotal: 657ms\tremaining: 59.5ms\n",
      "917:\tlearn: 0.0075183\ttotal: 657ms\tremaining: 58.7ms\n",
      "918:\tlearn: 0.0074817\ttotal: 658ms\tremaining: 58ms\n",
      "919:\tlearn: 0.0074387\ttotal: 658ms\tremaining: 57.3ms\n",
      "920:\tlearn: 0.0074025\ttotal: 659ms\tremaining: 56.5ms\n",
      "921:\tlearn: 0.0073600\ttotal: 660ms\tremaining: 55.8ms\n",
      "922:\tlearn: 0.0073178\ttotal: 660ms\tremaining: 55.1ms\n",
      "923:\tlearn: 0.0072847\ttotal: 661ms\tremaining: 54.3ms\n",
      "924:\tlearn: 0.0072492\ttotal: 661ms\tremaining: 53.6ms\n",
      "925:\tlearn: 0.0072139\ttotal: 662ms\tremaining: 52.9ms\n",
      "926:\tlearn: 0.0071788\ttotal: 662ms\tremaining: 52.2ms\n",
      "927:\tlearn: 0.0071376\ttotal: 663ms\tremaining: 51.4ms\n",
      "928:\tlearn: 0.0070966\ttotal: 664ms\tremaining: 50.7ms\n",
      "929:\tlearn: 0.0070645\ttotal: 664ms\tremaining: 50ms\n",
      "930:\tlearn: 0.0070301\ttotal: 665ms\tremaining: 49.3ms\n",
      "931:\tlearn: 0.0069897\ttotal: 665ms\tremaining: 48.5ms\n",
      "932:\tlearn: 0.0069591\ttotal: 666ms\tremaining: 47.8ms\n",
      "933:\tlearn: 0.0069276\ttotal: 666ms\tremaining: 47.1ms\n",
      "934:\tlearn: 0.0068879\ttotal: 667ms\tremaining: 46.4ms\n",
      "935:\tlearn: 0.0068567\ttotal: 667ms\tremaining: 45.6ms\n",
      "936:\tlearn: 0.0068174\ttotal: 668ms\tremaining: 44.9ms\n",
      "937:\tlearn: 0.0067842\ttotal: 669ms\tremaining: 44.2ms\n",
      "938:\tlearn: 0.0067512\ttotal: 669ms\tremaining: 43.5ms\n",
      "939:\tlearn: 0.0067184\ttotal: 670ms\tremaining: 42.7ms\n",
      "940:\tlearn: 0.0066881\ttotal: 670ms\tremaining: 42ms\n",
      "941:\tlearn: 0.0066556\ttotal: 671ms\tremaining: 41.3ms\n",
      "942:\tlearn: 0.0066234\ttotal: 671ms\tremaining: 40.6ms\n",
      "943:\tlearn: 0.0065911\ttotal: 672ms\tremaining: 39.9ms\n",
      "944:\tlearn: 0.0065532\ttotal: 672ms\tremaining: 39.1ms\n",
      "945:\tlearn: 0.0065155\ttotal: 673ms\tremaining: 38.4ms\n",
      "946:\tlearn: 0.0064861\ttotal: 674ms\tremaining: 37.7ms\n",
      "947:\tlearn: 0.0064488\ttotal: 674ms\tremaining: 37ms\n",
      "948:\tlearn: 0.0064206\ttotal: 675ms\tremaining: 36.3ms\n",
      "949:\tlearn: 0.0063916\ttotal: 675ms\tremaining: 35.5ms\n",
      "950:\tlearn: 0.0063549\ttotal: 676ms\tremaining: 34.8ms\n",
      "951:\tlearn: 0.0063271\ttotal: 676ms\tremaining: 34.1ms\n",
      "952:\tlearn: 0.0062907\ttotal: 677ms\tremaining: 33.4ms\n",
      "953:\tlearn: 0.0062623\ttotal: 677ms\tremaining: 32.7ms\n",
      "954:\tlearn: 0.0062263\ttotal: 678ms\tremaining: 31.9ms\n",
      "955:\tlearn: 0.0061983\ttotal: 679ms\tremaining: 31.2ms\n",
      "956:\tlearn: 0.0061712\ttotal: 679ms\tremaining: 30.5ms\n",
      "957:\tlearn: 0.0061357\ttotal: 680ms\tremaining: 29.8ms\n",
      "958:\tlearn: 0.0061081\ttotal: 680ms\tremaining: 29.1ms\n",
      "959:\tlearn: 0.0060730\ttotal: 681ms\tremaining: 28.4ms\n",
      "960:\tlearn: 0.0060435\ttotal: 681ms\tremaining: 27.6ms\n",
      "961:\tlearn: 0.0060163\ttotal: 682ms\tremaining: 26.9ms\n",
      "962:\tlearn: 0.0059871\ttotal: 682ms\tremaining: 26.2ms\n",
      "963:\tlearn: 0.0059527\ttotal: 683ms\tremaining: 25.5ms\n",
      "964:\tlearn: 0.0059239\ttotal: 684ms\tremaining: 24.8ms\n",
      "965:\tlearn: 0.0058972\ttotal: 684ms\tremaining: 24.1ms\n",
      "966:\tlearn: 0.0058687\ttotal: 685ms\tremaining: 23.4ms\n",
      "967:\tlearn: 0.0058404\ttotal: 685ms\tremaining: 22.7ms\n",
      "968:\tlearn: 0.0058119\ttotal: 686ms\tremaining: 21.9ms\n",
      "969:\tlearn: 0.0057784\ttotal: 686ms\tremaining: 21.2ms\n",
      "970:\tlearn: 0.0057451\ttotal: 687ms\tremaining: 20.5ms\n",
      "971:\tlearn: 0.0057193\ttotal: 688ms\tremaining: 19.8ms\n",
      "972:\tlearn: 0.0056943\ttotal: 688ms\tremaining: 19.1ms\n",
      "973:\tlearn: 0.0056615\ttotal: 689ms\tremaining: 18.4ms\n",
      "974:\tlearn: 0.0056361\ttotal: 689ms\tremaining: 17.7ms\n",
      "975:\tlearn: 0.0056036\ttotal: 690ms\tremaining: 17ms\n",
      "976:\tlearn: 0.0055791\ttotal: 690ms\tremaining: 16.3ms\n",
      "977:\tlearn: 0.0055470\ttotal: 691ms\tremaining: 15.5ms\n",
      "978:\tlearn: 0.0055221\ttotal: 691ms\tremaining: 14.8ms\n",
      "979:\tlearn: 0.0054903\ttotal: 692ms\tremaining: 14.1ms\n",
      "980:\tlearn: 0.0054657\ttotal: 693ms\tremaining: 13.4ms\n",
      "981:\tlearn: 0.0054418\ttotal: 693ms\tremaining: 12.7ms\n",
      "982:\tlearn: 0.0054105\ttotal: 694ms\tremaining: 12ms\n",
      "983:\tlearn: 0.0053862\ttotal: 694ms\tremaining: 11.3ms\n",
      "984:\tlearn: 0.0053552\ttotal: 695ms\tremaining: 10.6ms\n",
      "985:\tlearn: 0.0053293\ttotal: 695ms\tremaining: 9.87ms\n",
      "986:\tlearn: 0.0053054\ttotal: 696ms\tremaining: 9.16ms\n",
      "987:\tlearn: 0.0052749\ttotal: 696ms\tremaining: 8.46ms\n",
      "988:\tlearn: 0.0052494\ttotal: 697ms\tremaining: 7.75ms\n",
      "989:\tlearn: 0.0052241\ttotal: 697ms\tremaining: 7.04ms\n",
      "990:\tlearn: 0.0052007\ttotal: 698ms\tremaining: 6.34ms\n",
      "991:\tlearn: 0.0051756\ttotal: 699ms\tremaining: 5.63ms\n",
      "992:\tlearn: 0.0051507\ttotal: 699ms\tremaining: 4.93ms\n",
      "993:\tlearn: 0.0051255\ttotal: 700ms\tremaining: 4.22ms\n",
      "994:\tlearn: 0.0050959\ttotal: 700ms\tremaining: 3.52ms\n",
      "995:\tlearn: 0.0050665\ttotal: 701ms\tremaining: 2.81ms\n",
      "996:\tlearn: 0.0050439\ttotal: 702ms\tremaining: 2.11ms\n",
      "997:\tlearn: 0.0050148\ttotal: 702ms\tremaining: 1.41ms\n",
      "998:\tlearn: 0.0049929\ttotal: 703ms\tremaining: 703us\n",
      "999:\tlearn: 0.0049641\ttotal: 703ms\tremaining: 0us\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostRegressor at 0x2069a7ca220>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "##################################################\n",
    "##train a model on the values\n",
    "##################################################\n",
    "\n",
    "model = catboost.CatBoostRegressor()\n",
    "# Fit model\n",
    "model.fit(featuresMorganCounts, logp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we trained a model we can gat the shap values for each fingerprint bit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "##################################################\n",
    "##calculate the shap values\n",
    "##################################################\n",
    "\n",
    "explainer = shap.TreeExplainer(model)\n",
    "shap_vals = explainer.shap_values(featuresMorganCounts)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As we now have the shap values we can use them to calculate a distance matrix based on spearman correlation. Use the distance to for UMAP and store the emebdding vectors in the sdfExportData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "using precomputed metric; inverse_transform will be unavailable\n",
      "n_neighbors is larger than the dataset size; truncating to X.shape[0] - 1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished UMAP\n"
     ]
    }
   ],
   "source": [
    "\n",
    "#######################################################################\n",
    "## Calculate umap based on the shap values and add to the data frame ##\n",
    "#######################################################################\n",
    "\n",
    "#We use spearman correlation as a proximity measure, scale it to a range\n",
    "#between 0 and 1 and transform it to a distance to be used in umap\n",
    "umap_dist = 1 - (pd.DataFrame(shap_vals).T.corr(method=\"spearman\") + 1) / 2\n",
    "\n",
    "my_umap = umap.UMAP(metric='precomputed')\n",
    "\n",
    "my_umap_fit = my_umap.fit_transform(umap_dist)\n",
    "\n",
    "print(\"Finished UMAP\")\n",
    "\n",
    "#get the coordinates\n",
    "x = my_umap_fit[:, 0]\n",
    "y = my_umap_fit[:, 1]\n",
    "\n",
    "#add the coordinates to the data object\n",
    "sdfExportData[\"x\"] = x\n",
    "sdfExportData[\"y\"] = y\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now lets add some dummy groupLabels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "groups = np.zeros((len(sdfExportData),))\n",
    "groups[3:] = 1\n",
    "sdfExportData[\"groupLabel\"] = groups"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now lets use the shap values to assign atom attributions based on the model for logp prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################\n",
    "#now assign weights based on the model\n",
    "###################################################\n",
    "\n",
    "#first we have to calculate the atom indices associated with each fingerprint bit\n",
    "indices =  [calc_indices_for_fingerprints(smi,radius = 2,nBits=2048) for smi in smiles]\n",
    "\n",
    "#we also store the column names of the fingerprint matrix\n",
    "featureNames = featuresMorganCounts.columns\n",
    "\n",
    "#set direction to \"up\" if you only want the positive attributions\n",
    "#to \"down\" for only the positive attributions and \"both\" summing up over positive and\n",
    "#negative attributions\n",
    "modelAttributions = [list(atom_attributions(mols[i], indices[i], shap_vals[i],featureNames , direction=\"both\")) for i in range(len(smiles))]\n",
    "\n",
    "####################################################################################\n",
    "##Add the modelAttributions to the logp as one string for each compound\n",
    "####################################################################################\n",
    "\n",
    "sdfExportData[\"atom.Contribution.rep_modelAttributions\"] = [' '.join([str(round(i, 4)) for i in modelAttributions[j]]) for\n",
    "                                                           j in range(len(modelAttributions))]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To compare the model attribution against them we also calculate the crippen contributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "######################################################\n",
    "##Calculate the Crippen Contribs to the logp values\n",
    "######################################################\n",
    "\n",
    "logp_contribs = [[x for x, y in rdMolDescriptors._CalcCrippenContribs(mols[i])] for i in range(len(mols))]\n",
    "\n",
    "sdfExportData[\"atom.Contribution.rep_CrippenContribs\"] = [' '.join([str(round(i, 4)) for i in logp_contribs[j]]) for j\n",
    "                                                          in range(len(logp_contribs))]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As both attributions (Crippen and Model) were stored in the sdfExportData we prepare the data frame fro export to sdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#######################################################################################\n",
    "#add the chemical info to the data frame\n",
    "#######################################################################################\n",
    "# smilesCol: in which column the smiles are located\n",
    "PandasTools.AddMoleculeColumnToFrame(sdfExportData, smilesCol='smiles', molCol='ROMol', includeFingerprints=False)\n",
    "# to avoid duplicate smiles column\n",
    "del sdfExportData['smiles']\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we can write the data frame as an sdf file for use in CIME"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################################################################\n",
    "###Export as an sdf file you can read into CIME\n",
    "########################################################################\n",
    "\n",
    "PandasTools.WriteSDF(sdfExportData,\"SDF_OUTPUT.sdf\",\n",
    "                     properties=list(sdfExportData.columns), idName=\"RowID\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}